move abx plugins inside vendor dir

archivebox/vendor/abx-plugin-git/abx_plugin_git/__init__.py

@@ -0,0 +1,29 @@
+__package__ = 'abx_plugin_git'
+__label__ = 'Git'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import GIT_CONFIG
+    
+    return {
+        'GIT_CONFIG': GIT_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import GIT_BINARY
+    
+    return {
+        'git': GIT_BINARY,
+    }
+
+@abx.hookimpl
+def get_EXTRACTORS():
+    from .extractors import GIT_EXTRACTOR
+    
+    return {
+        'git': GIT_EXTRACTOR,
+    }

archivebox/vendor/abx-plugin-git/abx_plugin_git/binaries.py

@@ -0,0 +1,18 @@
+__package__ = 'abx_plugin_git'
+
+from typing import List
+
+from pydantic import InstanceOf
+from pydantic_pkgr import BinProvider, BinName, Binary
+
+from abx_plugin_default_binproviders import apt, brew, env
+
+from .config import GIT_CONFIG
+
+
+
+class GitBinary(Binary):
+    name: BinName = GIT_CONFIG.GIT_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+GIT_BINARY = GitBinary()

archivebox/vendor/abx-plugin-git/abx_plugin_git/config.py

@@ -0,0 +1,28 @@
+__package__ = 'abx_plugin_git'
+
+from typing import List
+
+from pydantic import Field
+
+from abx_spec_config.base_configset import BaseConfigSet
+
+from archivebox.config.common import ARCHIVING_CONFIG
+
+
+class GitConfig(BaseConfigSet):
+
+    SAVE_GIT: bool = True
+    
+    GIT_DOMAINS: str = Field(default='github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht')
+    
+    GIT_BINARY: str = Field(default='git')
+    GIT_ARGS: List[str] = [
+        '--recursive',
+    ]
+    GIT_EXTRA_ARGS: List[str] = []
+    
+    GIT_TIMEOUT: int =  Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
+    GIT_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
+    
+
+GIT_CONFIG = GitConfig()

archivebox/vendor/abx-plugin-git/abx_plugin_git/extractors.py

@@ -0,0 +1,15 @@
+__package__ = 'abx_plugin_git'
+
+# from pathlib import Path
+
+# from .binaries import GIT_BINARY
+
+
+# class GitExtractor(BaseExtractor):
+#     name: ExtractorName = 'git'
+#     binary: str = GIT_BINARY.name
+
+#     def get_output_path(self, snapshot) -> Path | None:
+#         return snapshot.as_link() / 'git'
+
+# GIT_EXTRACTOR = GitExtractor()

archivebox/vendor/abx-plugin-git/abx_plugin_git/git.py

@@ -0,0 +1,100 @@
+__package__ = 'archivebox.extractors'
+
+
+from pathlib import Path
+from typing import Optional
+
+from archivebox.misc.system import run, chmod_file
+from archivebox.misc.util import (
+    enforce_types,
+    is_static_file,
+    domain,
+    extension,
+    without_query,
+    without_fragment,
+)
+from ..logging_util import TimedProgress
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+
+from abx_plugin_git.config import GIT_CONFIG
+from abx_plugin_git.binaries import GIT_BINARY
+
+
+def get_output_path():
+    return 'git/'
+
+def get_embed_path(archiveresult=None):
+    if not archiveresult:
+        return get_output_path()
+
+    try:
+        return get_output_path() + list((archiveresult.snapshot_dir / get_output_path()).glob('*'))[0].name + '/'
+    except IndexError:
+        pass
+
+    return get_output_path()
+
+@enforce_types
+def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
+        return False
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / get_output_path()).exists():
+        return False
+
+    is_clonable_url = (
+        (domain(link.url) in GIT_CONFIG.GIT_DOMAINS)
+        or (extension(link.url) == 'git')
+    )
+    if not is_clonable_url:
+        return False
+
+    return GIT_CONFIG.SAVE_GIT
+
+
+@enforce_types
+def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=GIT_CONFIG.GIT_TIMEOUT) -> ArchiveResult:
+    """download full site using git"""
+    
+    git_binary = GIT_BINARY.load()
+    assert git_binary.abspath and git_binary.version
+
+    out_dir = out_dir or Path(link.link_dir)
+    output: ArchiveOutput = get_output_path()
+    output_path = out_dir / output
+    output_path.mkdir(exist_ok=True)
+    cmd = [
+        str(git_binary.abspath),
+        'clone',
+        *GIT_CONFIG.GIT_ARGS,
+        *([] if GIT_CONFIG.GIT_CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']),
+        without_query(without_fragment(link.url)),
+    ]
+    status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    try:
+        result = run(cmd, cwd=str(output_path), timeout=timeout + 1)
+        if result.returncode == 128:
+            # ignore failed re-download when the folder already exists
+            pass
+        elif result.returncode > 0:
+            hints = 'Got git response code: {}.'.format(result.returncode)
+            raise ArchiveError('Failed to save git clone', hints)
+
+        chmod_file(output, cwd=str(out_dir))
+
+    except Exception as err:
+        status = 'failed'
+        output = err
+    finally:
+        timer.end()
+
+    return ArchiveResult(
+        cmd=cmd,
+        pwd=str(out_dir),
+        cmd_version=str(git_binary.version),
+        output=output,
+        status=status,
+        **timer.stats,
+    )