From f12bfeb3229345b2d4cd7c1670ba050ca1111e7c Mon Sep 17 00:00:00 2001
From: Cristian <cristian@swapps.com>
Date: Wed, 8 Jul 2020 08:17:47 -0500
Subject: [PATCH] refactor: Change add() to receive url and depth instead of
import_str and import_path
---
archivebox/cli/archivebox_add.py | 12 ++----------
archivebox/core/views.py | 8 +++-----
archivebox/main.py | 25 ++++++++++---------------
3 files changed, 15 insertions(+), 30 deletions(-)
diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py
index c692750b..8f491d42 100644
--- a/archivebox/cli/archivebox_add.py
+++ b/archivebox/cli/archivebox_add.py
@@ -68,20 +68,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
import_path = command.import_path
add(
- import_str=import_path,
- import_path=None,
+ url=import_path,
+ depth=command.depth,
update_all=command.update_all,
index_only=command.index_only,
out_dir=pwd or OUTPUT_DIR,
)
- if command.depth == 1:
- add(
- import_str=None,
- import_path=import_path,
- update_all=command.update_all,
- index_only=command.index_only,
- out_dir=pwd or OUTPUT_DIR,
- )
if __name__ == '__main__':
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 0c5efff2..a721b992 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -66,12 +66,10 @@ class AddLinks(View):
if form.is_valid():
url = form.cleaned_data["url"]
print(f'[+] Adding URL: {url}')
- if form.cleaned_data["source"] == "url":
- key = "import_str"
- else:
- key = "import_path"
+ depth = 0 if form.cleaned_data["source"] == "url" else 1
input_kwargs = {
- key: url,
+ "url": url,
+ "depth": depth,
"update_all": False,
"out_dir": OUTPUT_DIR,
}
diff --git a/archivebox/main.py b/archivebox/main.py
index 3f05a385..a96c4250 100644
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -496,8 +496,8 @@ def status(out_dir: str=OUTPUT_DIR) -> None:
@enforce_types
-def add(import_str: Optional[str]=None,
- import_path: Optional[str]=None,
+def add(url: str,
+ depth: int=0,
update_all: bool=not ONLY_NEW,
index_only: bool=False,
out_dir: str=OUTPUT_DIR) -> List[Link]:
@@ -505,17 +505,9 @@ def add(import_str: Optional[str]=None,
check_data_folder(out_dir=out_dir)
- if (import_str and import_path) or (not import_str and not import_path):
- stderr(
- '[X] You should pass an import path or a page url as an argument\n',
- color='red',
- )
- raise SystemExit(2)
- elif import_str:
- import_path = save_stdin_to_sources(import_str, out_dir=out_dir)
- elif import_path:
- import_path = save_file_to_sources(import_path, out_dir=out_dir)
-
+ base_path = save_stdin_to_sources(url, out_dir=out_dir)
+ if depth == 1:
+ depth_path = save_file_to_sources(url, out_dir=out_dir)
check_dependencies()
# Step 1: Load list of links from the existing index
@@ -523,8 +515,11 @@ def add(import_str: Optional[str]=None,
all_links: List[Link] = []
new_links: List[Link] = []
all_links = load_main_index(out_dir=out_dir)
- if import_path:
- all_links, new_links = import_new_links(all_links, import_path, out_dir=out_dir)
+ all_links, new_links = import_new_links(all_links, base_path, out_dir=out_dir)
+ if depth == 1:
+ all_links, new_links_depth = import_new_links(all_links, depth_path, out_dir=out_dir)
+ new_links = new_links + new_links_depth
+
# Step 2: Write updated index with deduped old and new links back to disk
write_main_index(links=all_links, out_dir=out_dir)