mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-09 12:21:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

move pdf, screenshot, dom, singlefile, and ytdlp extractor config to new plugin system

Browse source
This commit is contained in:
Nick Sweeting 2024-09-25 00:42:26 -07:00
parent a2a586e369
commit a5ffd4e9d3
No known key found for this signature in database
11 changed files with 333 additions and 353 deletions
Download patch file Download diff file Expand all files Collapse all files

29
archivebox/extractors/dom.py
View file

@ -8,13 +8,6 @@ from ..system import run, chmod_file, atomic_write
from ..util import (
enforce_types,
is_static_file,
chrome_args,
chrome_cleanup,
)
from ..config import (
TIMEOUT,
SAVE_DOM,
CHROME_VERSION,
)
from ..logging_util import TimedProgress
@ -25,6 +18,8 @@ def get_output_path():
@enforce_types
def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
from plugins_extractor.chrome.apps import CHROME_CONFIG
if is_static_file(link.url):
return False
@ -33,42 +28,48 @@ def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
if (out_dir / get_output_path()).stat().st_size > 1:
return False
return SAVE_DOM
return CHROME_CONFIG.SAVE_DOM
@enforce_types
def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=60) -> ArchiveResult:
"""print HTML of site to file using chrome --dump-html"""
from plugins_extractor.chrome.apps import CHROME_CONFIG, CHROME_BINARY
CHROME_BIN = CHROME_BINARY.load()
assert CHROME_BIN.abspath and CHROME_BIN.version
out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = get_output_path()
output_path = out_dir / output
cmd = [
*chrome_args(),
str(CHROME_BIN.abspath),
*CHROME_CONFIG.chrome_args(),
'--dump-dom',
link.url
]
status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ')
try:
result = run(cmd, cwd=str(out_dir), timeout=timeout)
result = run(cmd, cwd=str(out_dir), timeout=timeout, text=True)
atomic_write(output_path, result.stdout)
if result.returncode:
hints = result.stderr.decode()
hints = result.stderr
raise ArchiveError('Failed to save DOM', hints)
chmod_file(output, cwd=str(out_dir))
except Exception as err:
status = 'failed'
output = err
chrome_cleanup()
CHROME_BINARY.chrome_cleanup_lockfile()
finally:
timer.end()
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=CHROME_VERSION,
cmd_version=str(CHROME_BIN.version),
output=output,
status=status,
**timer.stats,

54
archivebox/extractors/media.py
View file

@ -5,20 +5,7 @@ from typing import Optional
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from ..system import run, chmod_file
from ..util import (
enforce_types,
is_static_file,
dedupe,
)
from ..config import (
MEDIA_TIMEOUT,
SAVE_MEDIA,
YOUTUBEDL_ARGS,
YOUTUBEDL_EXTRA_ARGS,
YOUTUBEDL_BINARY,
YOUTUBEDL_VERSION,
CHECK_SSL_VALIDITY
)
from ..util import enforce_types, is_static_file, dedupe
from ..logging_util import TimedProgress
@ -38,6 +25,8 @@ def get_embed_path(archiveresult=None):
@enforce_types
def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
from plugins_extractor.ytdlp.apps import YTDLP_CONFIG
if is_static_file(link.url):
return False
@ -45,45 +34,52 @@ def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optio
if not overwrite and (out_dir / get_output_path()).exists():
return False
return SAVE_MEDIA
return YTDLP_CONFIG.USE_YTDLP
@enforce_types
def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=0) -> ArchiveResult:
"""Download playlists or individual video, audio, and subtitles using youtube-dl or yt-dlp"""
# from plugins_extractor.chrome.apps import CHROME_CONFIG
from plugins_extractor.ytdlp.apps import YTDLP_BINARY, YTDLP_CONFIG
YTDLP_BIN = YTDLP_BINARY.load()
assert YTDLP_BIN.abspath and YTDLP_BIN.version
timeout = timeout or YTDLP_CONFIG.YTDLP_TIMEOUT
out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = get_output_path()
output_path = out_dir / output
output_path.mkdir(exist_ok=True)
# later options take precedence
options = [
*YOUTUBEDL_ARGS,
*YOUTUBEDL_EXTRA_ARGS,
*([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
*YTDLP_CONFIG.YTDLP_EXTRA_ARGS,
*([] if YTDLP_CONFIG.YTDLP_CHECK_SSL_VALIDITY else ['--no-check-certificate']),
# TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR}
]
cmd = [
YOUTUBEDL_BINARY,
str(YTDLP_BIN.abspath),
*dedupe(options),
link.url,
]
status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ')
try:
result = run(cmd, cwd=str(output_path), timeout=timeout + 1)
result = run(cmd, cwd=str(output_path), timeout=timeout + 1, text=True)
chmod_file(output, cwd=str(out_dir))
if result.returncode:
if (b'ERROR: Unsupported URL' in result.stderr
or b'HTTP Error 404' in result.stderr
or b'HTTP Error 403' in result.stderr
or b'URL could be a direct video link' in result.stderr
or b'Unable to extract container ID' in result.stderr):
if ('ERROR: Unsupported URL' in result.stderr
or 'HTTP Error 404' in result.stderr
or 'HTTP Error 403' in result.stderr
or 'URL could be a direct video link' in result.stderr
or 'Unable to extract container ID' in result.stderr):
# These happen too frequently on non-media pages to warrant printing to console
pass
else:
hints = (
'Got youtube-dl (or yt-dlp) response code: {}.'.format(result.returncode),
*result.stderr.decode().split('\n'),
'Got yt-dlp response code: {}.'.format(result.returncode),
*result.stderr.split('\n'),
)
raise ArchiveError('Failed to save media', hints)
except Exception as err:
@ -117,7 +113,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=YOUTUBEDL_VERSION,
cmd_version=str(YTDLP_BIN.version),
output=output,
status=status,
index_texts=index_texts,

29
archivebox/extractors/pdf.py
View file

@ -8,13 +8,6 @@ from ..system import run, chmod_file
from ..util import (
enforce_types,
is_static_file,
chrome_args,
chrome_cleanup,
)
from ..config import (
TIMEOUT,
SAVE_PDF,
CHROME_VERSION,
)
from ..logging_util import TimedProgress
@ -25,6 +18,8 @@ def get_output_path():
@enforce_types
def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
from plugins_extractor.chrome.apps import CHROME_CONFIG
if is_static_file(link.url):
return False
@ -32,34 +27,40 @@ def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
if not overwrite and (out_dir / get_output_path()).exists():
return False
return SAVE_PDF
return CHROME_CONFIG.SAVE_PDF
@enforce_types
def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=60) -> ArchiveResult:
"""print PDF of site to file using chrome --headless"""
from plugins_extractor.chrome.apps import CHROME_CONFIG, CHROME_BINARY
CHROME_BIN = CHROME_BINARY.load()
assert CHROME_BIN.abspath and CHROME_BIN.version
out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = get_output_path()
cmd = [
*chrome_args(),
str(CHROME_BIN.abspath),
*CHROME_CONFIG.chrome_args(),
'--print-to-pdf',
link.url,
]
status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ')
try:
result = run(cmd, cwd=str(out_dir), timeout=timeout)
result = run(cmd, cwd=str(out_dir), timeout=timeout, text=True)
if result.returncode:
hints = (result.stderr or result.stdout).decode()
hints = (result.stderr or result.stdout)
raise ArchiveError('Failed to save PDF', hints)
chmod_file(get_output_path(), cwd=str(out_dir))
except Exception as err:
status = 'failed'
output = err
chrome_cleanup()
CHROME_BINARY.chrome_cleanup_lockfile()
finally:
timer.end()
@ -67,7 +68,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=CHROME_VERSION,
cmd_version=str(CHROME_BINARY.version),
output=output,
status=status,
**timer.stats,

33
archivebox/extractors/screenshot.py
View file

@ -5,17 +5,7 @@ from typing import Optional
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from ..system import run, chmod_file
from ..util import (
enforce_types,
is_static_file,
chrome_args,
chrome_cleanup,
)
from ..config import (
TIMEOUT,
SAVE_SCREENSHOT,
CHROME_VERSION,
)
from ..util import enforce_types, is_static_file
from ..logging_util import TimedProgress
@ -25,6 +15,8 @@ def get_output_path():
@enforce_types
def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
from plugins_extractor.chrome.apps import CHROME_CONFIG
if is_static_file(link.url):
return False
@ -32,40 +24,45 @@ def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite:
if not overwrite and (out_dir / get_output_path()).exists():
return False
return SAVE_SCREENSHOT
return CHROME_CONFIG.SAVE_SCREENSHOT
@enforce_types
def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=60) -> ArchiveResult:
"""take screenshot of site using chrome --headless"""
from plugins_extractor.chrome.apps import CHROME_CONFIG, CHROME_BINARY
CHROME_BIN = CHROME_BINARY.load()
assert CHROME_BIN.abspath and CHROME_BIN.version
out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = get_output_path()
cmd = [
*chrome_args(),
str(CHROME_BIN.abspath),
*CHROME_CONFIG.chrome_args(),
'--screenshot',
link.url,
]
status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ')
try:
result = run(cmd, cwd=str(out_dir), timeout=timeout)
result = run(cmd, cwd=str(out_dir), timeout=timeout, text=True)
if result.returncode:
hints = (result.stderr or result.stdout).decode()
hints = (result.stderr or result.stdout)
raise ArchiveError('Failed to save screenshot', hints)
chmod_file(output, cwd=str(out_dir))
except Exception as err:
status = 'failed'
output = err
chrome_cleanup()
CHROME_BINARY.chrome_cleanup_lockfile()
finally:
timer.end()
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=CHROME_VERSION,
cmd_version=str(CHROME_BIN.version),
output=output,
status=status,
**timer.stats,

56
archivebox/extractors/singlefile.py
View file

@ -7,22 +7,7 @@ import json
from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, chmod_file
from ..util import (
enforce_types,
is_static_file,
chrome_args,
dedupe,
)
from ..config import (
TIMEOUT,
SAVE_SINGLEFILE,
DEPENDENCIES,
SINGLEFILE_VERSION,
SINGLEFILE_ARGS,
SINGLEFILE_EXTRA_ARGS,
CHROME_BINARY,
COOKIES_FILE,
)
from ..util import enforce_types, is_static_file, dedupe
from ..logging_util import TimedProgress
@ -32,6 +17,8 @@ def get_output_path():
@enforce_types
def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
from plugins_extractor.singlefile.apps import SINGLEFILE_CONFIG
if is_static_file(link.url):
return False
@ -39,30 +26,35 @@ def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite:
if not overwrite and (out_dir / get_output_path()).exists():
return False
return SAVE_SINGLEFILE
return SINGLEFILE_CONFIG.SAVE_SINGLEFILE
@enforce_types
def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=60) -> ArchiveResult:
"""download full site using single-file"""
from plugins_extractor.chrome.apps import CHROME_CONFIG, CHROME_BINARY
from plugins_extractor.singlefile.apps import SINGLEFILE_CONFIG, SINGLEFILE_BINARY
CHROME_BIN = CHROME_BINARY.load()
assert CHROME_BIN.abspath and CHROME_BIN.version
SINGLEFILE_BIN = SINGLEFILE_BINARY.load()
assert SINGLEFILE_BIN.abspath and SINGLEFILE_BIN.version
out_dir = out_dir or Path(link.link_dir)
output = get_output_path()
browser_args = chrome_args(CHROME_TIMEOUT=0)
browser_args = CHROME_CONFIG.chrome_args(CHROME_TIMEOUT=0)
# SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:]))
# later options take precedence
options = [
'--browser-executable-path={}'.format(CHROME_BINARY),
*(["--browser-cookies-file={}".format(COOKIES_FILE)] if COOKIES_FILE else []),
browser_args,
*SINGLEFILE_ARGS,
*SINGLEFILE_EXTRA_ARGS,
'--browser-executable-path={}'.format(CHROME_BIN.abspath),
*(["--browser-cookies-file={}".format(SINGLEFILE_CONFIG.SINGLEFILE_COOKIES_FILE)] if SINGLEFILE_CONFIG.SINGLEFILE_COOKIES_FILE else []),
'--browser-args={}'.format(json.dumps(browser_args)),
*SINGLEFILE_CONFIG.SINGLEFILE_EXTRA_ARGS,
]
cmd = [
DEPENDENCIES['SINGLEFILE_BINARY']['path'],
str(SINGLEFILE_BIN.abspath),
*dedupe(options),
link.url,
output,
@ -72,13 +64,13 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
timer = TimedProgress(timeout, prefix=' ')
result = None
try:
result = run(cmd, cwd=str(out_dir), timeout=timeout)
result = run(cmd, cwd=str(out_dir), timeout=timeout, text=True, capture_output=True)
# parse out number of files downloaded from last line of stderr:
# "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
output_tail = [
line.strip()
for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
for line in (result.stdout + result.stderr).rsplit('\n', 5)[-5:]
if line.strip()
]
hints = (
@ -93,9 +85,9 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
except (Exception, OSError) as err:
status = 'failed'
# TODO: Make this prettier. This is necessary to run the command (escape JSON internal quotes).
cmd[2] = browser_args.replace('"', "\\\"")
cmd[2] = cmd[2].replace('"', "\\\"")
if result:
err.hints = (result.stdout + result.stderr).decode().split('\n')
err.hints = (result.stdout + result.stderr).split('\n')
output = err
finally:
timer.end()
@ -103,7 +95,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=SINGLEFILE_VERSION,
cmd_version=str(SINGLEFILE_BIN.version),
output=output,
status=status,
**timer.stats,