move abx plugins inside vendor dir

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/__init__.py

@@ -0,0 +1,35 @@
+__package__ = 'abx_plugin_singlefile'
+__label__ = 'Singlefile'
+__homepage__ = 'https://github.com/gildas-lormeau/singlefile'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import SINGLEFILE_CONFIG
+    
+    return {
+        'SINGLEFILE_CONFIG': SINGLEFILE_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import SINGLEFILE_BINARY
+    
+    return {
+        'singlefile': SINGLEFILE_BINARY,
+    }
+
+@abx.hookimpl
+def get_EXTRACTORS():
+    from .extractors import SINGLEFILE_EXTRACTOR
+    
+    return {
+        'singlefile': SINGLEFILE_EXTRACTOR,
+    }
+
+# @abx.hookimpl
+# def get_INSTALLED_APPS():
+#     # needed to load ./models.py
+#     return [__package__]

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/binaries.py

@@ -0,0 +1,45 @@
+from typing import List
+
+from pydantic import InstanceOf
+from pydantic_pkgr import Binary, BinProvider, BinaryOverrides, BinName, bin_abspath
+
+from abx_plugin_default_binproviders import env
+from abx_plugin_npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
+
+from .config import SINGLEFILE_CONFIG
+
+
+SINGLEFILE_MIN_VERSION = '1.1.54'
+SINGLEFILE_MAX_VERSION = '1.1.60'
+
+
+class SinglefileBinary(Binary):
+    name: BinName = SINGLEFILE_CONFIG.SINGLEFILE_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
+
+    overrides: BinaryOverrides = {
+        LIB_NPM_BINPROVIDER.name: {
+            "abspath": lambda:
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=LIB_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file", PATH=LIB_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file-node.js", PATH=LIB_NPM_BINPROVIDER.PATH),
+            "packages": [f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
+        },
+        SYS_NPM_BINPROVIDER.name: {
+            "abspath": lambda:
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=SYS_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file", PATH=SYS_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file-node.js", PATH=SYS_NPM_BINPROVIDER.PATH),
+            "packages": [f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
+            "install": lambda: None,
+        },
+        env.name: {
+            'abspath': lambda:
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=env.PATH)
+                or bin_abspath('single-file', PATH=env.PATH)
+                or bin_abspath('single-file-node.js', PATH=env.PATH),
+        },
+    }
+
+
+SINGLEFILE_BINARY = SinglefileBinary()

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/config.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+from typing import List, Optional
+
+from pydantic import Field
+
+from abx_spec_config.base_configset import BaseConfigSet
+
+from archivebox.config.common import ARCHIVING_CONFIG
+
+
+class SinglefileConfig(BaseConfigSet):
+    SAVE_SINGLEFILE: bool = True
+
+    SINGLEFILE_USER_AGENT: str              = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
+    SINGLEFILE_TIMEOUT: int                 = Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
+    SINGLEFILE_CHECK_SSL_VALIDITY: bool     = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
+    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=lambda: ARCHIVING_CONFIG.COOKIES_FILE)
+
+    SINGLEFILE_BINARY: str = Field(default='single-file')
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+
+
+SINGLEFILE_CONFIG = SinglefileConfig()

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/extractors.py

@@ -0,0 +1,18 @@
+__package__ = 'abx_plugin_singlefile'
+
+# from pathlib import Path
+
+# from pydantic_pkgr import BinName
+
+# from .binaries import SINGLEFILE_BINARY
+
+
+# class SinglefileExtractor(BaseExtractor):
+#     name: str = 'singlefile'
+#     binary: BinName = SINGLEFILE_BINARY.name
+
+#     def get_output_path(self, snapshot) -> Path:
+#         return Path(snapshot.link_dir) / 'singlefile.html'
+
+
+# SINGLEFILE_EXTRACTOR = SinglefileExtractor()

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/models.py

@@ -0,0 +1,14 @@
+from django.db import models
+
+from core.models import ArchiveResult
+
+class SinglefileResultManager(models.Manager):
+    def get_queryset(self):
+        return super().get_queryset().filter(extractor='singlefile')
+
+
+class SinglefileResult(ArchiveResult):
+    objects = SinglefileResultManager()
+
+    class Meta:
+        proxy = True

archivebox/vendor/abx-plugin-singlefile/abx_plugin_singlefile/singlefile.py

@@ -0,0 +1,103 @@
+__package__ = 'archivebox.extractors'
+
+from pathlib import Path
+
+from typing import Optional
+import json
+
+from ..index.schema import Link, ArchiveResult, ArchiveError
+from archivebox.misc.system import run, chmod_file
+from archivebox.misc.util import enforce_types, is_static_file, dedupe
+from ..logging_util import TimedProgress
+
+from plugins_extractor.chrome.config import CHROME_CONFIG
+from plugins_extractor.chrome.binaries import CHROME_BINARY
+from plugins_extractor.singlefile.config import SINGLEFILE_CONFIG
+from plugins_extractor.singlefile.binaries import SINGLEFILE_BINARY
+
+
+def get_output_path():
+    return 'singlefile.html'
+
+
+@enforce_types
+def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    
+    if is_static_file(link.url):
+        return False
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / get_output_path()).exists():
+        return False
+
+    return CHROME_CONFIG.USE_CHROME and SINGLEFILE_CONFIG.SAVE_SINGLEFILE
+
+
+@enforce_types
+def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=60) -> ArchiveResult:
+    """download full site using single-file"""
+
+    CHROME_BIN = CHROME_BINARY.load()
+    assert CHROME_BIN.abspath and CHROME_BIN.version
+    SINGLEFILE_BIN = SINGLEFILE_BINARY.load()
+    assert SINGLEFILE_BIN.abspath and SINGLEFILE_BIN.version
+
+    out_dir = out_dir or Path(link.link_dir)
+    output = get_output_path()
+
+    browser_args = CHROME_CONFIG.chrome_args(CHROME_TIMEOUT=0)
+
+    # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
+    options = [
+        '--browser-executable-path={}'.format(CHROME_BIN.abspath),
+        *(["--browser-cookies-file={}".format(SINGLEFILE_CONFIG.SINGLEFILE_COOKIES_FILE)] if SINGLEFILE_CONFIG.SINGLEFILE_COOKIES_FILE else []),
+        '--browser-args={}'.format(json.dumps(browser_args)),
+        *SINGLEFILE_CONFIG.SINGLEFILE_EXTRA_ARGS,
+    ]
+    cmd = [
+        str(SINGLEFILE_BIN.abspath),
+        *dedupe(options),
+        link.url,
+        output,
+    ]
+
+    status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    result = None
+    try:
+        result = run(cmd, cwd=str(out_dir), timeout=timeout, text=True, capture_output=True)
+
+        # parse out number of files downloaded from last line of stderr:
+        #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
+        output_tail = [
+            line.strip()
+            for line in (result.stdout + result.stderr).rsplit('\n', 5)[-5:]
+            if line.strip()
+        ]
+        hints = (
+            'Got single-file response code: {}.'.format(result.returncode),
+            *output_tail,
+        )
+
+        # Check for common failure cases
+        if (result.returncode > 0) or not (out_dir / output).is_file():
+            raise ArchiveError(f'SingleFile was not able to archive the page (status={result.returncode})', hints)
+        chmod_file(output, cwd=str(out_dir))
+    except (Exception, OSError) as err:
+        status = 'failed'
+        # TODO: Make this prettier. This is necessary to run the command (escape JSON internal quotes).
+        cmd[2] = cmd[2].replace('"', "\\\"")
+        if result:
+            err.hints = (result.stdout + result.stderr).split('\n')
+        output = err
+    finally:
+        timer.end()
+
+    return ArchiveResult(
+        cmd=cmd,
+        pwd=str(out_dir),
+        cmd_version=str(SINGLEFILE_BIN.version),
+        output=output,
+        status=status,
+        **timer.stats,
+    )