mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-06-07 09:54:57 -04:00
fix(export_browser_history): tilde doesn't expand in quotes (#1661)
Some checks failed
CodeQL / Analyze (python) (push) Has been cancelled
Build Debian package / build (push) Has been cancelled
Deploy static content to Pages / deploy (push) Has been cancelled
Build Homebrew package / build (push) Has been cancelled
Build GitHub Pages website / build (push) Has been cancelled
Run linters / lint (push) Has been cancelled
Build Pip package / build (push) Has been cancelled
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Has been cancelled
Run tests / docker_tests (push) Has been cancelled
Build GitHub Pages website / deploy (push) Has been cancelled
Some checks failed
CodeQL / Analyze (python) (push) Has been cancelled
Build Debian package / build (push) Has been cancelled
Deploy static content to Pages / deploy (push) Has been cancelled
Build Homebrew package / build (push) Has been cancelled
Build GitHub Pages website / build (push) Has been cancelled
Run linters / lint (push) Has been cancelled
Build Pip package / build (push) Has been cancelled
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Has been cancelled
Run tests / docker_tests (push) Has been cancelled
Build GitHub Pages website / deploy (push) Has been cancelled
<!-- IMPORTANT: Do not submit PRs with only formatting / PEP8 / line length changes. --> # Summary Patch submitted by @pcrockett # Related issues - Fixes https://github.com/ArchiveBox/ArchiveBox/issues/1657#issue-2856003985 # Changes these areas - [x] Bugfixes - [ ] Feature behavior - [ ] Command line interface - [ ] Configuration options - [ ] Internal architecture - [ ] Snapshot data layout on disk
This commit is contained in:
commit
d93f32ab24
1 changed files with 94 additions and 19 deletions
|
@ -1,4 +1,5 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
# Helper script to export browser history and bookmarks to a format ArchiveBox can ingest.
|
# Helper script to export browser history and bookmarks to a format ArchiveBox can ingest.
|
||||||
# Usage:
|
# Usage:
|
||||||
# curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/export_browser_history.sh'
|
# curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/export_browser_history.sh'
|
||||||
|
@ -7,43 +8,107 @@
|
||||||
# bash export_browser_history.sh --safari
|
# bash export_browser_history.sh --safari
|
||||||
# ls
|
# ls
|
||||||
# chrome_history.json
|
# chrome_history.json
|
||||||
|
# chrome_bookmarks.json
|
||||||
# firefox_history.json
|
# firefox_history.json
|
||||||
# firefox_bookmarks.json
|
# firefox_bookmarks.json
|
||||||
# safari_history.json
|
# safari_history.json
|
||||||
|
#
|
||||||
|
# Assumptions:
|
||||||
|
#
|
||||||
|
# * you're running this on macOS or Linux
|
||||||
|
# * you're running a reasonably modern version of Bash
|
||||||
|
# * macOS users: `brew install bash`
|
||||||
|
#
|
||||||
|
# Dependencies:
|
||||||
|
#
|
||||||
|
# * sqlite
|
||||||
|
# * jq (for chrome bookmarks)
|
||||||
|
#
|
||||||
|
|
||||||
|
set -eo pipefail
|
||||||
|
|
||||||
|
BROWSER_TO_EXPORT="${1?Please specify --chrome, --firefox, or --safari}"
|
||||||
OUTPUT_DIR="$(pwd)"
|
OUTPUT_DIR="$(pwd)"
|
||||||
|
|
||||||
if [[ "$1" == "--chrome" ]]; then
|
is_linux() {
|
||||||
# Google Chrome / Chromium
|
[[ "$(uname -s)" == "Linux" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
find_firefox_places_db() {
|
||||||
|
# shellcheck disable=SC2012 # `ls` with path expansion is good enough, don't need `find`
|
||||||
|
if is_linux; then
|
||||||
|
ls ~/.mozilla/firefox/*.default*/places.sqlite | head -n 1
|
||||||
|
else
|
||||||
|
ls ~/Library/Application\ Support/Firefox/Profiles/*.default*/places.sqlite | head -n 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
find_chrome_history_db() {
|
||||||
|
if is_linux; then
|
||||||
|
local config_home="${XDG_CONFIG_HOME:-${HOME}/.config}"
|
||||||
|
for path in \
|
||||||
|
"${config_home}/chromium/Default/History" \
|
||||||
|
"${config_home}/google-chrome/Default/History";
|
||||||
|
do
|
||||||
|
if [ -f "${path}" ]; then
|
||||||
|
echo "${path}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Unable to find Chrome history database. You can supply it manually as a second parameter." >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo ~/Library/Application\ Support/Google/Chrome/Default/History
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export_chrome() {
|
||||||
if [[ -e "$2" ]]; then
|
if [[ -e "$2" ]]; then
|
||||||
cp "$2" "$OUTPUT_DIR/chrome_history.db.tmp"
|
cp "$2" "$OUTPUT_DIR/chrome_history.db.tmp"
|
||||||
else
|
else
|
||||||
default=$(ls ~/Library/Application\ Support/Google/Chrome/Default/History)
|
default="$(find_chrome_history_db)"
|
||||||
echo "Defaulting to history db: $default"
|
echo "Defaulting to history db: $default"
|
||||||
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
||||||
cp "$default" "$OUTPUT_DIR/chrome_history.db.tmp"
|
cp "$default" "$OUTPUT_DIR/chrome_history.db.tmp"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sqlite3 "$OUTPUT_DIR/chrome_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', last_visit_time, 'description', title, 'href', url)) || \"]\" FROM urls;" > "$OUTPUT_DIR/chrome_history.json"
|
sqlite3 "$OUTPUT_DIR/chrome_history.db.tmp" "
|
||||||
jq < "$(dirname "${2:-$default}")"/Bookmarks '.roots.other.children[] | {href: .url, description: .name, timestamp: .date_added}' > "$OUTPUT_DIR/chrome_bookmarks.json"
|
SELECT '[' || group_concat(
|
||||||
|
json_object('timestamp', last_visit_time, 'description', title, 'href', url)
|
||||||
|
) || ']'
|
||||||
|
FROM urls;" > "$OUTPUT_DIR/chrome_history.json"
|
||||||
|
|
||||||
|
jq '.roots.other.children[] | {href: .url, description: .name, timestamp: .date_added}' \
|
||||||
|
< "$(dirname "${2:-$default}")"/Bookmarks \
|
||||||
|
> "$OUTPUT_DIR/chrome_bookmarks.json"
|
||||||
|
|
||||||
rm "$OUTPUT_DIR"/chrome_history.db.*
|
rm "$OUTPUT_DIR"/chrome_history.db.*
|
||||||
echo "Chrome history exported to:"
|
echo "Chrome history exported to:"
|
||||||
echo " $OUTPUT_DIR/chrome_history.json"
|
echo " $OUTPUT_DIR/chrome_history.json"
|
||||||
fi
|
echo " $OUTPUT_DIR/chrome_bookmarks.json"
|
||||||
|
}
|
||||||
|
|
||||||
if [[ "$1" == "--firefox" ]]; then
|
export_firefox() {
|
||||||
# Firefox
|
|
||||||
if [[ -e "$2" ]]; then
|
if [[ -e "$2" ]]; then
|
||||||
cp "$2" "$OUTPUT_DIR/firefox_history.db.tmp"
|
cp "$2" "$OUTPUT_DIR/firefox_history.db.tmp"
|
||||||
else
|
else
|
||||||
default=$(ls ~/Library/Application\ Support/Firefox/Profiles/*.default/places.sqlite)
|
default="$(find_firefox_places_db)"
|
||||||
echo "Defaulting to history db: $default"
|
echo "Defaulting to history db: $default"
|
||||||
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
||||||
cp "$default" "$OUTPUT_DIR/firefox_history.db.tmp"
|
cp "$default" "$OUTPUT_DIR/firefox_history.db.tmp"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', last_visit_date, 'description', title, 'href', url)) || \"]\" FROM moz_places;" > "$OUTPUT_DIR/firefox_history.json"
|
sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "
|
||||||
|
SELECT
|
||||||
|
'[' || group_concat(
|
||||||
|
json_object(
|
||||||
|
'timestamp', last_visit_date,
|
||||||
|
'description', title,
|
||||||
|
'href', url
|
||||||
|
)
|
||||||
|
) || ']'
|
||||||
|
FROM moz_places;" > "$OUTPUT_DIR/firefox_history.json"
|
||||||
|
|
||||||
sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "
|
sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "
|
||||||
with recursive tags AS (
|
with recursive tags AS (
|
||||||
|
@ -67,14 +132,13 @@ if [[ "$1" == "--firefox" ]]; then
|
||||||
echo "Firefox history exported to:"
|
echo "Firefox history exported to:"
|
||||||
echo " $OUTPUT_DIR/firefox_history.json"
|
echo " $OUTPUT_DIR/firefox_history.json"
|
||||||
echo " $OUTPUT_DIR/firefox_bookmarks.json"
|
echo " $OUTPUT_DIR/firefox_bookmarks.json"
|
||||||
fi
|
}
|
||||||
|
|
||||||
if [[ "$1" == "--safari" ]]; then
|
export_safari() {
|
||||||
# Safari
|
|
||||||
if [[ -e "$2" ]]; then
|
if [[ -e "$2" ]]; then
|
||||||
cp "$2" "$OUTPUT_DIR/safari_history.db.tmp"
|
cp "$2" "$OUTPUT_DIR/safari_history.db.tmp"
|
||||||
else
|
else
|
||||||
default="~/Library/Safari/History.db"
|
default=~"/Library/Safari/History.db"
|
||||||
echo "Defaulting to history db: $default"
|
echo "Defaulting to history db: $default"
|
||||||
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
|
||||||
cp "$default" "$OUTPUT_DIR/safari_history.db.tmp"
|
cp "$default" "$OUTPUT_DIR/safari_history.db.tmp"
|
||||||
|
@ -85,4 +149,15 @@ if [[ "$1" == "--safari" ]]; then
|
||||||
rm "$OUTPUT_DIR"/safari_history.db.*
|
rm "$OUTPUT_DIR"/safari_history.db.*
|
||||||
echo "Safari history exported to:"
|
echo "Safari history exported to:"
|
||||||
echo " $OUTPUT_DIR/safari_history.json"
|
echo " $OUTPUT_DIR/safari_history.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "$BROWSER_TO_EXPORT" == "--chrome" ]]; then
|
||||||
|
export_chrome "$@"
|
||||||
|
elif [[ "$BROWSER_TO_EXPORT" == "--firefox" ]]; then
|
||||||
|
export_firefox "$@"
|
||||||
|
elif [[ "$BROWSER_TO_EXPORT" == "--safari" ]]; then
|
||||||
|
export_safari "$@"
|
||||||
|
else
|
||||||
|
echo "Unrecognized argument: $1" >&2
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue