mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

use yt-dlp for media archiving instead of youtube-dl

Browse source
This commit is contained in:
Nick Sweeting 2022-04-21 07:09:17 -07:00
parent eb81d41f84
commit 57df65f28f
3 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Dockerfile
View file

@ -81,7 +81,8 @@ RUN apt-get update -qq \
build-essential python-dev python3-dev \ build-essential python-dev python3-dev \
&& echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \ && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
&& python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \ && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
&& pip install --quiet -r /tmp/requirements.txt \ && pip install -r /tmp/requirements.txt \
&& pip install --upgrade youtube-dl yt-dlp \
&& apt-get purge -y build-essential python-dev python3-dev \ && apt-get purge -y build-essential python-dev python3-dev \
&& apt-get autoremove -y \ && apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
@ -104,7 +105,7 @@ RUN pip install -e .
WORKDIR "$DATA_DIR" WORKDIR "$DATA_DIR"
ENV IN_DOCKER=True \ ENV IN_DOCKER=True \
CHROME_SANDBOX=False \ CHROME_SANDBOX=False \
CHROME_BINARY="chromium" \ CHROME_BINARY="/usr/bin/chromium-browser" \
USE_SINGLEFILE=True \ USE_SINGLEFILE=True \
SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \ SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
USE_READABILITY=True \ USE_READABILITY=True \

3
archivebox/config.py
View file

@ -142,6 +142,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'--yes-playlist', '--yes-playlist',
'--continue', '--continue',
'--ignore-errors', '--ignore-errors',
'--no-abort-on-error',
'--geo-bypass', '--geo-bypass',
'--add-metadata', '--add-metadata',
'--max-filesize={}'.format(c['MEDIA_MAX_SIZE']), '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']),
@ -929,7 +930,7 @@ def get_chrome_info(config: ConfigDict) -> ConfigValue:
'TIMEOUT': config['TIMEOUT'], 'TIMEOUT': config['TIMEOUT'],
'RESOLUTION': config['RESOLUTION'], 'RESOLUTION': config['RESOLUTION'],
'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'], 'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'],
'CHROME_BINARY': config['CHROME_BINARY'], 'CHROME_BINARY': bin_path(config['CHROME_BINARY']),
'CHROME_HEADLESS': config['CHROME_HEADLESS'], 'CHROME_HEADLESS': config['CHROME_HEADLESS'],
'CHROME_SANDBOX': config['CHROME_SANDBOX'], 'CHROME_SANDBOX': config['CHROME_SANDBOX'],
'CHROME_USER_AGENT': config['CHROME_USER_AGENT'], 'CHROME_USER_AGENT': config['CHROME_USER_AGENT'],

1
archivebox/extractors/media.py
View file

@ -43,6 +43,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
YOUTUBEDL_BINARY, YOUTUBEDL_BINARY,
*YOUTUBEDL_ARGS, *YOUTUBEDL_ARGS,
*([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']), *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
# TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR}
link.url, link.url,
] ]
status = 'succeeded' status = 'succeeded'