mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-16 16:14:28 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: readability uses snapshot instead of link

Browse source
This commit is contained in:
Cristian 2020-12-29 13:55:08 -05:00
parent 6230984cb3
commit 5cf9ca0e2c
3 changed files with 64 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

24
archivebox/extractors/readability.py
View file

@ -6,6 +6,8 @@ from tempfile import NamedTemporaryFile
from typing import Optional
import json
from django.db.models import Model
from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, atomic_write
from ..util import (
@ -24,12 +26,12 @@ from ..config import (
from ..logging_util import TimedProgress
@enforce_types
def get_html(link: Link, path: Path) -> str:
def get_html(snapshot: Model, path: Path) -> str:
"""
Try to find wget, singlefile and then dom files.
If none is found, download the url again.
"""
canonical = link.canonical_outputs()
canonical = snapshot.canonical_outputs()
abs_path = path.absolute()
sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
document = None
@ -41,25 +43,25 @@ def get_html(link: Link, path: Path) -> str:
except (FileNotFoundError, TypeError):
continue
if document is None:
return download_url(link.url)
return download_url(snapshot.url)
else:
return document
@enforce_types
def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
out_dir = out_dir or link.link_dir
if is_static_file(link.url):
def should_save_readability(snapshot: Model, out_dir: Optional[str]=None) -> bool:
out_dir = out_dir or snapshot.link_dir
if is_static_file(snapshot.url):
return False
output = Path(out_dir or link.link_dir) / 'readability'
output = Path(out_dir or snapshot.snapshot_dir) / 'readability'
return SAVE_READABILITY and READABILITY_VERSION and (not output.exists())
@enforce_types
def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
"""download reader friendly version using @mozilla/readability"""
out_dir = Path(out_dir or link.link_dir)
out_dir = Path(out_dir or snapshot.snapshot_dir)
output_folder = out_dir.absolute() / "readability"
output = str(output_folder)
@ -69,12 +71,12 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
# fake command to show the user so they have something to try debugging if get_html fails
cmd = [
CURL_BINARY,
link.url
snapshot.url
]
readability_content = None
timer = TimedProgress(timeout, prefix=' ')
try:
document = get_html(link, out_dir)
document = get_html(snapshot, out_dir)
temp_doc = NamedTemporaryFile(delete=False)
temp_doc.write(document.encode("utf-8"))
temp_doc.close()