mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 14:44:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Split output of readability into multiple files

Browse source
This commit is contained in:
Cristian 2020-08-10 13:15:28 -05:00
parent 7e2b249388
commit b33c66a9f7
1 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

16
archivebox/extractors/readability.py
View file

@ -1,6 +1,7 @@
__package__ = 'archivebox.extractors' __package__ = 'archivebox.extractors'
from pathlib import Path from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional from typing import Optional
import json import json
@ -35,14 +36,15 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
"""download reader friendly version using @mozilla/readability""" """download reader friendly version using @mozilla/readability"""
out_dir = out_dir or link.link_dir out_dir = out_dir or link.link_dir
output = str(Path(out_dir).absolute() / "readability.json") output_folder = Path(out_dir).absolute() / "readability"
document = download_url(link.url) document = download_url(link.url)
temp_doc = NamedTemporaryFile()
temp_doc.write(document.encode("utf-8"))
# SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
cmd = [ cmd = [
READABILITY_BINARY, READABILITY_BINARY,
document temp_doc.name
] ]
status = 'succeeded' status = 'succeeded'
@ -50,7 +52,10 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
try: try:
result = run(cmd, cwd=out_dir, timeout=timeout) result = run(cmd, cwd=out_dir, timeout=timeout)
result_json = json.loads(result.stdout) result_json = json.loads(result.stdout)
atomic_write(output, result_json) output_folder.mkdir(exist_ok=True)
atomic_write(str(output_folder / "content.html"), result_json.pop("content"))
atomic_write(str(output_folder / "content.txt"), result_json.pop("textContent"))
atomic_write(str(output_folder / "article.json"), result_json)
# parse out number of files downloaded from last line of stderr: # parse out number of files downloaded from last line of stderr:
# "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)" # "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
@ -72,12 +77,13 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
output = err output = err
finally: finally:
timer.end() timer.end()
temp_doc.close()
return ArchiveResult( return ArchiveResult(
cmd=cmd, cmd=cmd,
pwd=out_dir, pwd=out_dir,
cmd_version=READABILITY_VERSION, cmd_version=READABILITY_VERSION,
output=output, output=str(output_folder),
status=status, status=status,
**timer.stats, **timer.stats,
) )