load EXTRACTORS dynamically using importlib.import_module

This commit is contained in:
Nick Sweeting 2024-05-11 22:28:59 -07:00
parent c7f55fc3ba
commit 457c42bf84
No known key found for this signature in database
18 changed files with 198 additions and 40 deletions

View file

@ -19,6 +19,12 @@ from ..util import (
)
from .title import get_html
def get_output_path():
return "htmltotext.txt"
class HTMLTextExtractor(HTMLParser):
TEXT_ATTRS = [
"alt", "cite", "href", "label",
@ -109,7 +115,7 @@ def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite:
return False
out_dir = out_dir or Path(link.link_dir)
if not overwrite and (out_dir / 'htmltotext.txt').exists():
if not overwrite and (out_dir / get_output_path()).exists():
return False
return SAVE_HTMLTOTEXT
@ -120,7 +126,7 @@ def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
"""extract search-indexing-friendly text from an HTML document"""
out_dir = Path(out_dir or link.link_dir)
output = "htmltotext.txt"
output = get_output_path()
cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
timer = TimedProgress(timeout, prefix=' ')