From e2a5e0136c2b6ce994f96ac2fe52eb9b39dce75a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 25 Jun 2020 17:46:11 -0400 Subject: [PATCH] update docker setup and dependencies --- Dockerfile | 135 +++++++++++++++++++++------------------------ docker-compose.yml | 13 +++-- setup.py | 12 ++-- 3 files changed, 77 insertions(+), 83 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6996b706..af75b709 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,84 +1,77 @@ -# This Dockerfile for ArchiveBox installs the following in a container: -# - curl, wget, python3, youtube-dl, google-chrome-beta -# - ArchiveBox +# This is the Dockerfile for ArchiveBox, it includes the following major pieces: +# git, curl, wget, python3, youtube-dl, google-chrome-stable, ArchiveBox # Usage: -# docker build github.com/pirate/ArchiveBox -t archivebox -# echo 'https://example.com' | docker run -i --mount type=bind,source=./data,target=/data archivebox /bin/archive -# docker run --mount type=bind,source=./data,target=/data archivebox /bin/archive 'https://example.com/some/rss/feed.xml' +# docker build . -t archivebox:latest +# docker run -v=./data:/data archivebox:latest init +# docker run -v=./data:/data archivebox:latest add 'https://example.com' # Documentation: # https://github.com/pirate/ArchiveBox/wiki/Docker#docker -FROM node:13-slim -LABEL maintainer="Nick Sweeting " - -RUN apt-get update \ - && apt-get install -yq --no-install-recommends \ - jq git zlib1g-dev wget curl youtube-dl gnupg2 libgconf-2-4 python3 python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Install latest chrome package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ - && apt-get update \ - && apt-get install -y google-chrome-beta fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ - --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /src/*.deb - -# It's a good idea to use dumb-init to help prevent zombie chrome processes. -ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init -RUN chmod +x /usr/local/bin/dumb-init - -# Uncomment to skip the chromium download when installing puppeteer. If you do, -# you'll need to launch puppeteer with: -# browser.launch({executablePath: 'google-chrome-beta'}) -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# Install puppeteer so it's available in the container. -RUN npm install puppeteer - -# Add user so we don't need --no-sandbox. -RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \ - && mkdir -p /home/pptruser/Downloads \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /node_modules - -WORKDIR /home/pptruser/app - -RUN python3 -m pip install --upgrade pip setuptools && python3 -m pip install virtualenv \ - && python3 -m virtualenv ".docker-venv" -ENV PATH="/home/pttruser/app/.docker-venv/bin:${PATH}" -COPY ./Pipfile.lock "/home/pttruser/app/Pipfile.lock" -RUN jq -r \ - '.default,.develop | to_entries[] | .key + .value.version' \ - "/home/pttruser/app/Pipfile.lock" \ - | /home/pttruser/app/.docker-venv/bin/python -m pip install --no-cache-dir -r /dev/stdin \ - && rm "/home/pttruser/app/Pipfile.lock" - -# Install the ArchiveBox repository and pip requirements -# RUN git clone https://github.com/pirate/ArchiveBox /home/pptruser/app \ -ADD . /home/pptruser/app -RUN mkdir -p /data \ - && chown -R pptruser:pptruser /data \ - && ln -s /data /home/pptruser/app/archivebox/output \ - && ln -s /home/pptruser/app/bin/* /bin/ \ - && ln -s /home/pptruser/app/bin/archivebox /bin/archive \ - && chown -R pptruser:pptruser /home/pptruser/app/archivebox - -VOLUME /data -EXPOSE 8000 +FROM python:3.8-slim-buster +LABEL name="archivebox" \ + maintainer="Nick Sweeting " \ + version="0.4.3" \ + description="All-in-one personal internet archiving container" ENV LANG=C.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=C.UTF-8 \ PYTHONIOENCODING=UTF-8 \ - CHROME_SANDBOX=False \ - CHROME_BINARY=google-chrome-beta \ - OUTPUT_DIR=/data + PYTHONUNBUFFERED=1 \ + APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \ + CODE_PATH=/app \ + VENV_PATH=/venv \ + DATA_PATH=/data + +# Install latest chrome package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends \ + apt-transport-https ca-certificates apt-utils gnupg gnupg2 libgconf-2-4 zlib1g-dev dumb-init \ + wget curl youtube-dl jq git ffmpeg avconv \ + && curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && echo "deb https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \ + && apt-get update \ + && apt-get install -qq -y --no-install-recommends \ + google-chrome-stable \ + fontconfig \ + fonts-ipafont-gothic \ + fonts-wqy-zenhei \ + fonts-thai-tlwg \ + fonts-kacst \ + fonts-symbola \ + fonts-noto \ + fonts-freefont-ttf \ + && rm -rf /var/lib/apt/lists/* + +# Add user so we don't need --no-sandbox to run chrome +RUN groupadd -r archivebox && useradd -r -g archivebox -G audio,video archivebox \ + && mkdir -p /home/archivebox/Downloads \ + && chown -R archivebox:archivebox /home/archivebox + +WORKDIR "$CODE_PATH" +ADD . "$CODE_PATH" +VOLUME "$CODE_PATH" +RUN chown -R archivebox:archivebox "$CODE_PATH" + +ENV PATH="$VENV_PATH/bin:${PATH}" +RUN python --version \ + && python -m venv "$VENV_PATH" \ + && pip install --upgrade pip \ + && pip install -e . \ + && chown -R archivebox:archivebox "$VENV_PATH" + +WORKDIR "$DATA_PATH" +VOLUME "$DATA_PATH" +RUN chown -R archivebox:archivebox "$DATA_PATH" # Run everything from here on out as non-privileged user -USER pptruser -WORKDIR /home/pptruser/app +USER archivebox +ENV CHROME_BINARY=google-chrome \ + CHROME_SANDBOX=False \ + OUTPUT_DIR="$DATA_PATH" + +RUN archivebox version ENTRYPOINT ["dumb-init", "--"] -CMD ["/bin/archivebox"] +CMD ["archivebox"] diff --git a/docker-compose.yml b/docker-compose.yml index 631ebcf7..d1f52961 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,25 +8,26 @@ # Documentation: # https://github.com/pirate/ArchiveBox/wiki/Docker#docker-compose -version: '3' +version: '3.7' services: archivebox: build: . + image: archivebox:latest + command: archivebox server stdin_open: true tty: true - # env_file: path/to/your/ArchiveBox.conf environment: - - USE_COLOR=False + - USE_COLOR=True - SHOW_PROGRESS=False volumes: - ./data:/data - command: bash -c 'echo "https://github.com/pirate/ArchiveBox" | /bin/archive; tail -f /dev/null' nginx: - image: 'nginx' + image: nginx:alpine ports: - - '8098:80' + - 443:443 + - 80:80 volumes: - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf - ./data:/var/www diff --git a/setup.py b/setup.py index f23ae7b5..f335bb59 100644 --- a/setup.py +++ b/setup.py @@ -14,13 +14,13 @@ with open(os.path.join(PYTHON_DIR, 'VERSION'), 'r') as f: setuptools.setup( name="archivebox", version=VERSION, + license='MIT', author="Nick Sweeting", author_email="git@nicksweeting.com", description="The self-hosted internet archive.", long_description=README, long_description_content_type="text/markdown", url="https://github.com/pirate/ArchiveBox", - license='MIT', project_urls={ 'Donate': 'https://github.com/pirate/ArchiveBox/wiki/Donations', 'Changelog': 'https://github.com/pirate/ArchiveBox/wiki/Changelog', @@ -31,14 +31,14 @@ setuptools.setup( 'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki', }, packages=setuptools.find_packages(), - python_requires='>=3.6', + python_requires='>=3.7', install_requires=[ "dataclasses==0.6", - "mypy-extensions==0.4.1", + "mypy-extensions==0.4.3", "base32-crockford==0.3.0", - "django==2.2", - "django-extensions==2.1.6", - "python-crontab==2.3.6", + "django==3.0.7", + "django-extensions==2.2.9", + "python-crontab==2.5.1", "youtube-dl", "ipython",