mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

tests: Add readability tests

Browse source
This commit is contained in:
Cristian 2020-08-11 11:15:15 -05:00
parent 8aa7b34de7
commit 2a68af1b94
2 changed files with 36 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
archivebox/extractors/readability.py
View file

@ -37,7 +37,7 @@ def get_html(link: Link, path: Path) -> str:
with open(abs_path / source, "r") as f:
document = f.read()
break
except FileNotFoundError:
except (FileNotFoundError, TypeError):
continue
if document is None:
return download_url(link.url)
@ -51,6 +51,7 @@ def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
return False
output = Path(out_dir or link.link_dir) / 'readability.json'
print(output, SAVE_READABILITY)
return SAVE_READABILITY and (not output.exists())
@ -63,8 +64,9 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
output = str(output_folder)
document = get_html(link, out_dir)
temp_doc = NamedTemporaryFile()
temp_doc = NamedTemporaryFile(delete=False)
temp_doc.write(document.encode("utf-8"))
temp_doc.close()
# Readability Docs: https://github.com/mozilla/readability
cmd = [
READABILITY_BINARY,
@ -101,7 +103,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
output = err
finally:
timer.end()
temp_doc.close()
return ArchiveResult(
cmd=cmd,