mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-20 09:55:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Add htmltotext extractor

Browse source 
Saves HTML text nodes and selected element attributes in
`htmltotext.txt` for each Snapshot. Primarily intended to be used
for search indexing.
This commit is contained in:
Ross Williams 2023-10-23 21:42:25 -04:00
parent 6555719489
commit 310b4d1242
9 changed files with 203 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tests/test_extractors.py
View file

@ -39,6 +39,14 @@ def test_mercury_works(tmp_path, process, disable_extractors_dict):
output_file = archived_item_path / "mercury" / "content.html"
assert output_file.exists()
def test_htmltotext_works(tmp_path, process, disable_extractors_dict):
disable_extractors_dict.update({"SAVE_HTMLTOTEXT": "true"})
add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],
capture_output=True, env=disable_extractors_dict)
archived_item_path = list(tmp_path.glob("archive/**/*"))[0]
output_file = archived_item_path / "htmltotext.txt"
assert output_file.exists()
def test_readability_works_with_wget(tmp_path, process, disable_extractors_dict):
disable_extractors_dict.update({"USE_READABILITY": "true", "USE_WGET": "true"})
add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],