mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-06-05 01:03:51 -04:00
update build and release scripts to use uv
This commit is contained in:
Nick Sweeting
parent
ce2e19a429
commit
e29aff12bf
10 changed files with 1292 additions and 2094 deletions
|
|
@ -9,7 +9,7 @@ set -o errexit
|
|||
set -o errtrace
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
IFS=$' '
|
||||
|
||||
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
|
||||
cd "$REPO_DIR"
|
||||
|
|
@ -18,15 +18,34 @@ which docker > /dev/null || exit 1
|
|||
which jq > /dev/null || exit 1
|
||||
# which pdm > /dev/null || exit 1
|
||||
|
||||
SUPPORTED_PLATFORMS="linux/amd64,linux/arm64"
|
||||
|
||||
TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
|
||||
declare -a TAG_NAMES="$*"
|
||||
BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
|
||||
VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
|
||||
SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
|
||||
GIT_SHA=sha-"$(git rev-parse --short HEAD)"
|
||||
SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
|
||||
SELECTED_PLATFORMS="linux/amd64,linux/arm64"
|
||||
|
||||
echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$SELECTED_PLATFORMS"
|
||||
# if not already in TAG_NAMES, add GIT_SHA and BRANCH_NAME
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$GIT_SHA"; then
|
||||
TAG_NAMES+=("$GIT_SHA")
|
||||
fi
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$BRANCH_NAME"; then
|
||||
TAG_NAMES+=("$BRANCH_NAME")
|
||||
fi
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$VERSION"; then
|
||||
TAG_NAMES+=("$VERSION")
|
||||
fi
|
||||
|
||||
echo "[+] Building Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION tags=${TAG_NAMES[*]}"
|
||||
|
||||
declare -a FULL_TAG_NAMES
|
||||
# for each tag in TAG_NAMES, add archivebox/archivebox:tag and nikisweeting/archivebox:tag to FULL_TAG_NAMES
|
||||
for TAG_NAME in "${TAG_NAMES[@]}"; do
|
||||
[[ "$TAG_NAME" == "" ]] && continue
|
||||
FULL_TAG_NAMES+=("-t archivebox/archivebox:$TAG_NAME")
|
||||
FULL_TAG_NAMES+=("-t nikisweeting/archivebox:$TAG_NAME")
|
||||
FULL_TAG_NAMES+=("-t ghcr.io/archivebox/archivebox:$TAG_NAME")
|
||||
done
|
||||
echo "${FULL_TAG_NAMES[@]}"
|
||||
|
||||
function check_platforms() {
|
||||
INSTALLED_PLATFORMS="$(docker buildx inspect | grep 'Platforms:' )"
|
||||
|
|
@ -72,30 +91,13 @@ check_platforms || (recreate_builder && check_platforms) || exit 1
|
|||
|
||||
|
||||
# Make sure pyproject.toml, pdm{.dev}.lock, requirements{-dev}.txt, package{-lock}.json are all up-to-date
|
||||
echo "[!] Make sure you've run ./bin/lock_pkgs.sh recently!"
|
||||
sleep 1
|
||||
# bash ./bin/lock_pkgs.sh
|
||||
# echo "[!] Make sure you've run ./bin/lock_pkgs.sh recently!"
|
||||
bash ./bin/lock_pkgs.sh
|
||||
|
||||
|
||||
echo "[+] Building archivebox:$VERSION docker image..."
|
||||
# docker builder prune
|
||||
# docker build . --no-cache -t archivebox-dev \
|
||||
# replace --load with --push to deploy
|
||||
docker buildx build --platform "$SELECTED_PLATFORMS" --load . \
|
||||
-t archivebox/archivebox:$TAG_NAME \
|
||||
-t archivebox/archivebox:$GIT_SHA \
|
||||
-t nikisweeting/archivebox:$TAG_NAME \
|
||||
-t nikisweeting/archivebox:$GIT_SHA \
|
||||
-t ghcr.io/archivebox/archivebox:$TAG_NAME \
|
||||
-t ghcr.io/archivebox/archivebox:$GIT_SHA
|
||||
# -t archivebox/archivebox \
|
||||
# -t archivebox/archivebox:$VERSION \
|
||||
# -t archivebox/archivebox:$SHORT_VERSION \
|
||||
# -t archivebox/archivebox:latest \
|
||||
# -t nikisweeting/archivebox \
|
||||
# -t nikisweeting/archivebox:$VERSION \
|
||||
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
||||
# -t nikisweeting/archivebox:latest \
|
||||
# -t ghcr.io/archivebox/archivebox:$VERSION \
|
||||
# -t ghcr.io/archivebox/archivebox:$SHORT_VERSION \
|
||||
# -t ghcr.io/archivebox/archivebox:latest
|
||||
# shellcheck disable=SC2068
|
||||
docker buildx build --platform "$SELECTED_PLATFORMS" --load . ${FULL_TAG_NAMES[@]}
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ git pull
|
|||
cd "$REPO_DIR"
|
||||
|
||||
echo "[+] Building docs"
|
||||
sphinx-apidoc -o docs archivebox
|
||||
cd "$REPO_DIR/docs"
|
||||
make clean
|
||||
make html
|
||||
# open docs/_build/html/index.html to see the output
|
||||
cd "$REPO_DIR"
|
||||
|
|
|
|||
|
|
@ -11,21 +11,15 @@ set -o pipefail
|
|||
IFS=$'\n'
|
||||
|
||||
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
|
||||
|
||||
if [[ -f "$REPO_DIR/.venv/bin/activate" ]]; then
|
||||
source "$REPO_DIR/.venv/bin/activate"
|
||||
else
|
||||
echo "[!] Warning: No virtualenv presesnt in $REPO_DIR/.venv, creating one now..."
|
||||
python3 -m venv --system-site-packages --symlinks $REPO_DIR/.venv
|
||||
fi
|
||||
cd "$REPO_DIR"
|
||||
|
||||
# Generate pdm.lock, requirements.txt, and package-lock.json
|
||||
bash ./bin/lock_pkgs.sh
|
||||
source .venv/bin/activate
|
||||
|
||||
echo "[+] Building sdist, bdist_wheel, and egg_info"
|
||||
rm -Rf build dist
|
||||
pdm build
|
||||
uv build
|
||||
cp dist/* ./pip_dist/
|
||||
|
||||
echo
|
||||
|
|
|
|||
|
|
@ -32,52 +32,40 @@ echo
|
|||
echo "[*] Cleaning up old lockfiles and build files"
|
||||
deactivate 2>/dev/null || true
|
||||
rm -Rf build dist
|
||||
rm -f pdm.lock
|
||||
rm -f pdm.dev.lock
|
||||
rm -f uv.lock
|
||||
rm -f requirements.txt
|
||||
rm -f requirements-dev.txt
|
||||
rm -f package-lock.json
|
||||
rm -f archivebox/package.json
|
||||
rm -f archivebox/package-lock.json
|
||||
rm -Rf ./.venv
|
||||
rm -Rf ./node_modules
|
||||
rm -Rf ./archivebox/node_modules
|
||||
# rm -Rf ./.venv
|
||||
# rm -Rf ./node_modules
|
||||
# rm -Rf ./archivebox/node_modules
|
||||
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[+] Generating dev & prod requirements.txt & pdm.lock from pyproject.toml..."
|
||||
pip install --upgrade pip setuptools
|
||||
pdm self update >/dev/null 2>&1 || true
|
||||
pdm venv create 3.12
|
||||
uv venv --python 3.12
|
||||
source .venv/bin/activate
|
||||
echo
|
||||
echo "pyproject.toml: archivebox $(grep 'version = ' pyproject.toml | awk '{print $3}' | jq -r)"
|
||||
echo "$(which python): $(python --version | head -n 1)"
|
||||
echo "$(which pdm): $(pdm --version | head -n 1)"
|
||||
pdm info --env
|
||||
pdm info
|
||||
echo "$(which uv): $(uv --version | head -n 1)"
|
||||
|
||||
echo
|
||||
# https://pdm-project.org/latest/usage/lockfile/
|
||||
# prod
|
||||
pdm lock --group=':all' --production --lockfile pdm.lock --python="==3.12.*" --platform=linux
|
||||
pdm lock --group=':all' --production --lockfile pdm.lock --python="==3.12.*" --platform=macos --append
|
||||
pdm sync --group=':all' --production --lockfile pdm.lock --clean
|
||||
pdm export --group=':all' --production --lockfile pdm.lock --without-hashes -o requirements.txt
|
||||
# cp ./pdm.lock ./pip_dist/
|
||||
# cp ./requirements.txt ./pip_dist/
|
||||
|
||||
# dev
|
||||
pdm lock --group=':all' --dev --lockfile pdm.dev.lock --python="==3.12.*" --platform=linux
|
||||
pdm lock --group=':all' --dev --lockfile pdm.dev.lock --python="==3.12.*" --platform=macos --append
|
||||
pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean
|
||||
pdm export --group=':all' --dev --lockfile pdm.dev.lock --without-hashes -o requirements-dev.txt
|
||||
# cp ./pdm.dev.lock ./pip_dist/
|
||||
# cp ./requirements-dev.txt ./pip_dist/
|
||||
uv lock
|
||||
uv pip compile pyproject.toml --all-extras -o requirements.txt >/dev/null
|
||||
uv sync --all-extras --frozen 2>/dev/null
|
||||
|
||||
echo
|
||||
echo "[+] Generating package-lock.json from package.json..."
|
||||
npm install -g npm
|
||||
npm config set fund false --location=global &
|
||||
npm config set fund false &
|
||||
npm config set audit false --location=global &
|
||||
npm config set audit false &
|
||||
echo
|
||||
echo "package.json: archivebox $(jq -r '.version' package.json)"
|
||||
echo
|
||||
|
|
@ -85,7 +73,7 @@ echo "$(which node): $(node --version | head -n 1)"
|
|||
echo "$(which npm): $(npm --version | head -n 1)"
|
||||
|
||||
echo
|
||||
npm install --package-lock-only
|
||||
npm install --package-lock-only --prefer-offline
|
||||
cp package.json archivebox/package.json
|
||||
cp package-lock.json archivebox/package-lock.json
|
||||
|
||||
|
|
@ -93,10 +81,8 @@ echo
|
|||
echo "[√] Finished. Don't forget to commit the new lockfiles:"
|
||||
echo
|
||||
ls "pyproject.toml" | cat
|
||||
ls "pdm.lock" | cat
|
||||
ls "pdm.dev.lock" | cat
|
||||
ls "requirements.txt" | cat
|
||||
ls "requirements-dev.txt" | cat
|
||||
ls "uv.lock" | cat
|
||||
echo
|
||||
ls "package.json" | cat
|
||||
ls "package-lock.json" | cat
|
||||
|
|
|
|||
|
|
@ -8,43 +8,47 @@ set -o errexit
|
|||
set -o errtrace
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
IFS=$' '
|
||||
|
||||
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
|
||||
cd "$REPO_DIR"
|
||||
|
||||
SUPPORTED_PLATFORMS="linux/amd64,linux/arm64" # no longer supported: linux/arm/v7
|
||||
|
||||
TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
|
||||
declare -a TAG_NAMES="$*"
|
||||
BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
|
||||
VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
|
||||
SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
|
||||
GIT_SHA=sha-"$(git rev-parse --short HEAD)"
|
||||
SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
|
||||
SELECTED_PLATFORMS="linux/amd64,linux/arm64"
|
||||
|
||||
# if not already in TAG_NAMES, add GIT_SHA and BRANCH_NAME
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$GIT_SHA"; then
|
||||
TAG_NAMES+=("$GIT_SHA")
|
||||
fi
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$BRANCH_NAME"; then
|
||||
TAG_NAMES+=("$BRANCH_NAME")
|
||||
fi
|
||||
if ! echo "${TAG_NAMES[@]}" | grep -q "$VERSION"; then
|
||||
TAG_NAMES+=("$VERSION")
|
||||
fi
|
||||
|
||||
echo "[+] Building + releasing Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION tags=${TAG_NAMES[*]}"
|
||||
|
||||
declare -a FULL_TAG_NAMES
|
||||
# for each tag in TAG_NAMES, add archivebox/archivebox:tag and nikisweeting/archivebox:tag to FULL_TAG_NAMES
|
||||
for TAG_NAME in "${TAG_NAMES[@]}"; do
|
||||
[[ "$TAG_NAME" == "" ]] && continue
|
||||
FULL_TAG_NAMES+=("-t archivebox/archivebox:$TAG_NAME")
|
||||
FULL_TAG_NAMES+=("-t nikisweeting/archivebox:$TAG_NAME")
|
||||
FULL_TAG_NAMES+=("-t ghcr.io/archivebox/archivebox:$TAG_NAME")
|
||||
done
|
||||
echo "${FULL_TAG_NAMES[@]}"
|
||||
|
||||
|
||||
./bin/lock_pkgs.sh
|
||||
|
||||
# echo "[*] Logging in to Docker Hub & Github Container Registry"
|
||||
# docker login --username=nikisweeting
|
||||
# docker login ghcr.io --username=pirate
|
||||
|
||||
# echo "[^] Building docker image"
|
||||
# ./bin/build_docker.sh "$TAG_NAME" "$SELECTED_PLATFORMS"
|
||||
|
||||
echo "[^] Uploading docker image"
|
||||
docker buildx build --platform "$SELECTED_PLATFORMS" --push . \
|
||||
-t archivebox/archivebox:"$TAG_NAME" \
|
||||
-t archivebox/archivebox:"$GIT_SHA" \
|
||||
-t nikisweeting/archivebox:"$TAG_NAME" \
|
||||
-t nikisweeting/archivebox:"$GIT_SHA" \
|
||||
-t ghcr.io/archivebox/archivebox:"$TAG_NAME" \
|
||||
-t ghcr.io/archivebox/archivebox:"$GIT_SHA"
|
||||
# -t archivebox/archivebox \
|
||||
# -t archivebox/archivebox:$VERSION \
|
||||
# -t archivebox/archivebox:$SHORT_VERSION \
|
||||
# -t archivebox/archivebox:latest \
|
||||
# -t nikisweeting/archivebox \
|
||||
# -t nikisweeting/archivebox:$VERSION \
|
||||
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
||||
# -t nikisweeting/archivebox:latest \
|
||||
# -t ghcr.io/archivebox/archivebox:$VERSION \
|
||||
# -t ghcr.io/archivebox/archivebox:$SHORT_VERSION \
|
||||
|
||||
# shellcheck disable=SC2068
|
||||
docker buildx build --platform "$SELECTED_PLATFORMS" --push . ${FULL_TAG_NAMES[@]}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,5 @@ REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && p
|
|||
cd "$REPO_DIR"
|
||||
source "$REPO_DIR/.venv/bin/activate"
|
||||
|
||||
echo "[^] Publishing to Test PyPI..."
|
||||
pdm publish --repository testpypi
|
||||
|
||||
echo "[^] Publishing to PyPI..."
|
||||
pdm publish --no-build
|
||||
uv publish
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue