mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-30 22:45:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Initial implementation

Browse source
This commit is contained in:
JDC 2020-11-17 18:42:57 -05:00 • committed by Nick Sweeting
parent 7bc13204e6
commit b1f70b2197
7 changed files with 73 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
archivebox/extractors/__init__.py
View file

@ -23,6 +23,7 @@ from ..logging_util import (
log_archive_method_started,
log_archive_method_finished,
)
from ..search import write_search_index
from .title import should_save_title, save_title
from .favicon import should_save_favicon, save_favicon
@ -107,6 +108,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
link.history[method_name].append(result)
stats[result.status] += 1
write_search_index(link=link, texts=result.index_texts)
log_archive_method_finished(result)
if not skip_index:
ArchiveResult.objects.create(snapshot=snapshot, extractor=method_name, cmd=result.cmd, cmd_version=result.cmd_version,

7
archivebox/extractors/readability.py
View file

@ -71,6 +71,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
CURL_BINARY,
link.url
]
readability_content = None
timer = TimedProgress(timeout, prefix=' ')
try:
document = get_html(link, out_dir)
@ -86,8 +87,9 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
result = run(cmd, cwd=out_dir, timeout=timeout)
result_json = json.loads(result.stdout)
output_folder.mkdir(exist_ok=True)
readability_content = result_json.pop("textContent")
atomic_write(str(output_folder / "content.html"), result_json.pop("content"))
atomic_write(str(output_folder / "content.txt"), result_json.pop("textContent"))
atomic_write(str(output_folder / "content.txt"), readability_content)
atomic_write(str(output_folder / "article.json"), result_json)
# parse out number of files downloaded from last line of stderr:
@ -117,5 +119,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
cmd_version=READABILITY_VERSION,
output=output,
status=status,
**timer.stats,
index_texts= [readability_content] if readability_content else [],
**timer.stats,
)