mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-17 00:24:26 -04:00
Merge branch 'dev' into feat/search-fields
This commit is contained in:
Nick Sweeting
GitHub
commit
75018ed10b
73 changed files with 3132 additions and 335 deletions
|
|
@ -35,3 +35,5 @@ docker/
|
||||||
data/
|
data/
|
||||||
data*/
|
data*/
|
||||||
output/
|
output/
|
||||||
|
index.sqlite3
|
||||||
|
index.sqlite3-wal
|
||||||
|
|
|
||||||
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
**/*.lock
|
||||||
|
**/*-lock.json
|
||||||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -13,7 +13,6 @@ venv/
|
||||||
node_modules/
|
node_modules/
|
||||||
|
|
||||||
# Ignore dev lockfiles (should always be built fresh)
|
# Ignore dev lockfiles (should always be built fresh)
|
||||||
pdm.lock
|
|
||||||
pdm.dev.lock
|
pdm.dev.lock
|
||||||
requirements-dev.txt
|
requirements-dev.txt
|
||||||
|
|
||||||
|
|
@ -30,6 +29,9 @@ data/
|
||||||
data*/
|
data*/
|
||||||
output/
|
output/
|
||||||
index.sqlite3
|
index.sqlite3
|
||||||
|
*.sqlite*
|
||||||
|
data.*
|
||||||
|
|
||||||
# vim
|
# vim
|
||||||
*.sw?
|
*.sw?
|
||||||
|
.vscode
|
||||||
|
|
|
||||||
26
README.md
26
README.md
|
|
@ -154,7 +154,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
|
||||||
|
|
||||||
> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.* (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))
|
> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.* (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))
|
||||||
> We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more
|
> We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more
|
||||||
> *ArchiveBox has 🏛️ 501(c)(3) [nonprofit status](https://hackclub.com/hcb/) and all our work supports open-source development.*
|
> *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.*
|
||||||
|
|
||||||
<br/>
|
<br/>
|
||||||
|
|
||||||
|
|
@ -291,7 +291,8 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||||
<details>
|
<details>
|
||||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
|
<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
|
||||||
<br/>
|
<br/>
|
||||||
<ol>
|
See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
|
||||||
|
<!--<ol>
|
||||||
<li>Add the ArchiveBox repository to your sources.<br/>
|
<li>Add the ArchiveBox repository to your sources.<br/>
|
||||||
<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
|
<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
|
||||||
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
|
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
|
||||||
|
|
@ -310,7 +311,7 @@ archivebox version # make sure all dependencies are inst
|
||||||
<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
|
<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
|
||||||
archivebox init --setup
|
archivebox init --setup
|
||||||
</code></pre>
|
</code></pre>
|
||||||
<i>Note: If you encounter issues or want more granular instructions, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki.</i><br/><br/>
|
<br/>
|
||||||
</li>
|
</li>
|
||||||
<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
|
<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
|
||||||
<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
|
<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
|
||||||
|
|
@ -320,9 +321,8 @@ archivebox help
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</li>
|
</li>
|
||||||
</ol>
|
</ol>
|
||||||
|
|
||||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||||
<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
|
<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
|
||||||
<br/><br/>
|
<br/><br/>
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
|
|
@ -407,10 +407,12 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
|
||||||
> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
|
> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
|
||||||
|
|
||||||
<ul>
|
<ul>
|
||||||
<li>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></li>
|
<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
|
||||||
<li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
|
<li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
|
||||||
<li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
|
<li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
|
||||||
<li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
|
<li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
|
||||||
|
<li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
|
||||||
|
<li><a href="https://portainer-templates.as93.net/archivebox">Portainer</a></li>
|
||||||
<li><a href="https://github.com/ArchiveBox/ArchiveBox/pull/922/files#diff-00f0606e18b2618c3cc1667ca7c2b703b537af690ca71eba1330633587dcb1ee">AppImage</a></li>
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/pull/922/files#diff-00f0606e18b2618c3cc1667ca7c2b703b537af690ca71eba1330633587dcb1ee">AppImage</a></li>
|
||||||
<li><a href="https://runtipi.io/docs/apps-available#:~:text=for%20AI%20Chats.-,ArchiveBox,Open%20source%20self%2Dhosted%20web%20archiving.,-Atuin%20Server">Runtipi</a></li>
|
<li><a href="https://runtipi.io/docs/apps-available#:~:text=for%20AI%20Chats.-,ArchiveBox,Open%20source%20self%2Dhosted%20web%20archiving.,-Atuin%20Server">Runtipi</a></li>
|
||||||
<li><a href="https://github.com/ArchiveBox/ArchiveBox/issues/986">Umbrel</a> (need contributors...)</li>
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/issues/986">Umbrel</a> (need contributors...)</li>
|
||||||
|
|
@ -567,7 +569,7 @@ ls ./archive/*/index.html # or inspect snapshot data directly on the filesystem
|
||||||
<br/>
|
<br/>
|
||||||
|
|
||||||
<details>
|
<details>
|
||||||
<summary><b>🖥 Web UI Usage</b></summary>
|
<summary><b>🖥 Web UI & API Usage</b></summary>
|
||||||
<pre lang="bash"><code style="white-space: pre-line">
|
<pre lang="bash"><code style="white-space: pre-line">
|
||||||
# Start the server on bare metal (pip/apt/brew/etc):
|
# Start the server on bare metal (pip/apt/brew/etc):
|
||||||
archivebox manage createsuperuser # create a new admin user via CLI
|
archivebox manage createsuperuser # create a new admin user via CLI
|
||||||
|
|
@ -756,8 +758,8 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
|
||||||
# e.g. archivebox config --set TIMEOUT=120
|
# e.g. archivebox config --set TIMEOUT=120
|
||||||
# or docker compose run archivebox config --set TIMEOUT=120
|
# or docker compose run archivebox config --set TIMEOUT=120
|
||||||
<br/>
|
<br/>
|
||||||
TIMEOUT=120 # default: 60 add more seconds on slower networks
|
TIMEOUT=240 # default: 60 add more seconds on slower networks
|
||||||
CHECK_SSL_VALIDITY=True # default: False True = allow saving URLs w/ bad SSL
|
CHECK_SSL_VALIDITY=False # default: True False = allow saving URLs w/ bad SSL
|
||||||
SAVE_ARCHIVE_DOT_ORG=False # default: True False = disable Archive.org saving
|
SAVE_ARCHIVE_DOT_ORG=False # default: True False = disable Archive.org saving
|
||||||
MAX_MEDIA_SIZE=1500m # default: 750m raise/lower youtubedl output size
|
MAX_MEDIA_SIZE=1500m # default: 750m raise/lower youtubedl output size
|
||||||
<br/>
|
<br/>
|
||||||
|
|
@ -776,7 +778,7 @@ CURL_USER_AGENT="Mozilla/5.0 ..."
|
||||||
|
|
||||||
To achieve high-fidelity archives in as many situations as possible, ArchiveBox depends on a variety of 3rd-party libraries and tools that specialize in extracting different types of content.
|
To achieve high-fidelity archives in as many situations as possible, ArchiveBox depends on a variety of 3rd-party libraries and tools that specialize in extracting different types of content.
|
||||||
|
|
||||||
> Under-the-hood, ArchiveBox uses [Django](https://www.djangoproject.com/start/overview/) to power its [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage) and [SQlite](https://www.sqlite.org/locrsf.html) + the filesystem to provide [fast & durable metadata storage](https://www.sqlite.org/locrsf.html) w/ [determinisitc upgrades](https://stackoverflow.com/a/39976321/2156113).
|
> Under-the-hood, ArchiveBox uses [Django](https://www.djangoproject.com/start/overview/) to power its [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Django Ninja](https://django-ninja.dev/) for the REST API, and [SQlite](https://www.sqlite.org/locrsf.html) + the filesystem to provide [fast & durable metadata storage](https://www.sqlite.org/locrsf.html) w/ [deterministic upgrades](https://stackoverflow.com/a/39976321/2156113).
|
||||||
|
|
||||||
ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install), [`wget`, `yt-dlp`, `readability`, etc.](#dependencies) internally, and its operation can be [tuned, secured, and extended](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) as-needed for many different applications.
|
ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install), [`wget`, `yt-dlp`, `readability`, etc.](#dependencies) internally, and its operation can be [tuned, secured, and extended](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) as-needed for many different applications.
|
||||||
|
|
||||||
|
|
@ -785,7 +787,7 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
|
||||||
<summary><i>Expand to learn more about ArchiveBox's internals & dependencies...</i></summary><br/>
|
<summary><i>Expand to learn more about ArchiveBox's internals & dependencies...</i></summary><br/>
|
||||||
|
|
||||||
<blockquote>
|
<blockquote>
|
||||||
<p><em>TIP: For better security, easier updating, and to avoid polluting your host system with extra dependencies,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> with everything pre-installed for the best experience.</em></p>
|
<p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
|
||||||
</blockquote>
|
</blockquote>
|
||||||
|
|
||||||
These optional dependencies used for archiving sites include:
|
These optional dependencies used for archiving sites include:
|
||||||
|
|
@ -1608,7 +1610,7 @@ Extractors take the URL of a page to archive, write their output to the filesyst
|
||||||
<a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a>
|
<a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a>
|
||||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
|
||||||
<br/>
|
<br/>
|
||||||
<sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
|
<sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
|
||||||
<br/><br/>
|
<br/><br/>
|
||||||
<a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>
|
<a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>
|
||||||
<a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>
|
<a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,11 @@ ABID_RAND_LEN = 6
|
||||||
|
|
||||||
DEFAULT_ABID_PREFIX = 'obj_'
|
DEFAULT_ABID_PREFIX = 'obj_'
|
||||||
|
|
||||||
|
# allows people to keep their uris secret on a per-instance basis by changing the salt.
|
||||||
|
# the default means everyone can share the same namespace for URI hashes,
|
||||||
|
# meaning anyone who has a URI and wants to check if you have it can guess the ABID
|
||||||
|
DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
|
||||||
|
|
||||||
|
|
||||||
class ABID(NamedTuple):
|
class ABID(NamedTuple):
|
||||||
"""
|
"""
|
||||||
|
|
@ -32,6 +37,8 @@ class ABID(NamedTuple):
|
||||||
subtype: str # e.g. 01
|
subtype: str # e.g. 01
|
||||||
rand: str # e.g. ZYEBQE
|
rand: str # e.g. ZYEBQE
|
||||||
|
|
||||||
|
# salt: str = DEFAULT_ABID_URI_SALT
|
||||||
|
|
||||||
def __getattr__(self, attr: str) -> Any:
|
def __getattr__(self, attr: str) -> Any:
|
||||||
return getattr(self.ulid, attr)
|
return getattr(self.ulid, attr)
|
||||||
|
|
||||||
|
|
@ -68,6 +75,10 @@ class ABID(NamedTuple):
|
||||||
rand=suffix[20:26].upper(),
|
rand=suffix[20:26].upper(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def uri_salt(self) -> str:
|
||||||
|
return DEFAULT_ABID_URI_SALT
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def suffix(self):
|
def suffix(self):
|
||||||
return ''.join((self.ts, self.uri, self.subtype, self.rand))
|
return ''.join((self.ts, self.uri, self.subtype, self.rand))
|
||||||
|
|
@ -97,7 +108,7 @@ class ABID(NamedTuple):
|
||||||
####################################################
|
####################################################
|
||||||
|
|
||||||
|
|
||||||
def uri_hash(uri: Union[str, bytes]) -> str:
|
def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
||||||
"""
|
"""
|
||||||
'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
|
'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
|
||||||
"""
|
"""
|
||||||
|
|
@ -115,7 +126,7 @@ def uri_hash(uri: Union[str, bytes]) -> str:
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
uri_bytes = uri_str.encode('utf-8')
|
uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
|
||||||
|
|
||||||
return hashlib.sha256(uri_bytes).hexdigest().upper()
|
return hashlib.sha256(uri_bytes).hexdigest().upper()
|
||||||
|
|
||||||
|
|
@ -130,12 +141,12 @@ def abid_part_from_prefix(prefix: Optional[str]) -> str:
|
||||||
assert len(prefix) == 3
|
assert len(prefix) == 3
|
||||||
return prefix + '_'
|
return prefix + '_'
|
||||||
|
|
||||||
def abid_part_from_uri(uri: str) -> str:
|
def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
||||||
"""
|
"""
|
||||||
'E4A5CCD9' # takes first 8 characters of sha256(url)
|
'E4A5CCD9' # takes first 8 characters of sha256(url)
|
||||||
"""
|
"""
|
||||||
uri = str(uri)
|
uri = str(uri)
|
||||||
return uri_hash(uri)[:ABID_URI_LEN]
|
return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
|
||||||
|
|
||||||
def abid_part_from_ts(ts: Optional[datetime]) -> str:
|
def abid_part_from_ts(ts: Optional[datetime]) -> str:
|
||||||
"""
|
"""
|
||||||
|
|
@ -175,7 +186,7 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
|
||||||
return str(rand)[-ABID_RAND_LEN:].upper()
|
return str(rand)[-ABID_RAND_LEN:].upper()
|
||||||
|
|
||||||
|
|
||||||
def abid_from_values(prefix, ts, uri, subtype, rand) -> ABID:
|
def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
|
||||||
"""
|
"""
|
||||||
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||||
"""
|
"""
|
||||||
|
|
@ -183,7 +194,7 @@ def abid_from_values(prefix, ts, uri, subtype, rand) -> ABID:
|
||||||
abid = ABID(
|
abid = ABID(
|
||||||
prefix=abid_part_from_prefix(prefix),
|
prefix=abid_part_from_prefix(prefix),
|
||||||
ts=abid_part_from_ts(ts),
|
ts=abid_part_from_ts(ts),
|
||||||
uri=abid_part_from_uri(uri),
|
uri=abid_part_from_uri(uri, salt=salt),
|
||||||
subtype=abid_part_from_subtype(subtype),
|
subtype=abid_part_from_subtype(subtype),
|
||||||
rand=abid_part_from_rand(rand),
|
rand=abid_part_from_rand(rand),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ from .abid import (
|
||||||
ABID_RAND_LEN,
|
ABID_RAND_LEN,
|
||||||
ABID_SUFFIX_LEN,
|
ABID_SUFFIX_LEN,
|
||||||
DEFAULT_ABID_PREFIX,
|
DEFAULT_ABID_PREFIX,
|
||||||
|
DEFAULT_ABID_URI_SALT,
|
||||||
abid_part_from_prefix,
|
abid_part_from_prefix,
|
||||||
abid_from_values
|
abid_from_values
|
||||||
)
|
)
|
||||||
|
|
@ -69,8 +70,8 @@ class ABIDModel(models.Model):
|
||||||
abid_subtype_src = 'None' # e.g. 'self.extractor'
|
abid_subtype_src = 'None' # e.g. 'self.extractor'
|
||||||
abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id'
|
abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id'
|
||||||
|
|
||||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
|
# id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
|
||||||
uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
# uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
|
||||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||||
|
|
@ -132,6 +133,7 @@ class ABIDModel(models.Model):
|
||||||
uri=uri,
|
uri=uri,
|
||||||
subtype=subtype,
|
subtype=subtype,
|
||||||
rand=rand,
|
rand=rand,
|
||||||
|
salt=DEFAULT_ABID_URI_SALT,
|
||||||
)
|
)
|
||||||
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
||||||
return abid
|
return abid
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,7 @@ class APIToken(ABIDModel):
|
||||||
return {
|
return {
|
||||||
"TYPE": "APIToken",
|
"TYPE": "APIToken",
|
||||||
"uuid": str(self.id),
|
"uuid": str(self.id),
|
||||||
|
"ulid": str(self.ulid),
|
||||||
"abid": str(self.get_abid()),
|
"abid": str(self.get_abid()),
|
||||||
"user_id": str(self.user.id),
|
"user_id": str(self.user.id),
|
||||||
"user_username": self.user.username,
|
"user_username": self.user.username,
|
||||||
|
|
@ -64,6 +65,10 @@ class APIToken(ABIDModel):
|
||||||
"expires": self.expires_as_iso8601,
|
"expires": self.expires_as_iso8601,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ulid(self):
|
||||||
|
return self.get_abid().ulid
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def expires_as_iso8601(self):
|
def expires_as_iso8601(self):
|
||||||
"""Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
|
"""Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@ api = NinjaAPIWithIOCapture(
|
||||||
version='1.0.0',
|
version='1.0.0',
|
||||||
csrf=False,
|
csrf=False,
|
||||||
auth=API_AUTH_METHODS,
|
auth=API_AUTH_METHODS,
|
||||||
urls_namespace="api",
|
urls_namespace="api-1",
|
||||||
docs=Swagger(settings={"persistAuthorization": True}),
|
docs=Swagger(settings={"persistAuthorization": True}),
|
||||||
# docs_decorator=login_required,
|
# docs_decorator=login_required,
|
||||||
# renderer=ORJSONRenderer(),
|
# renderer=ORJSONRenderer(),
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,17 @@
|
||||||
__package__ = 'archivebox.api'
|
__package__ = 'archivebox.api'
|
||||||
|
|
||||||
|
import math
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from typing import List, Optional
|
from typing import List, Optional, Union, Any
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from django.db.models import Q
|
from django.db.models import Q
|
||||||
from django.shortcuts import get_object_or_404
|
from django.shortcuts import get_object_or_404
|
||||||
|
from django.core.exceptions import ValidationError
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
|
||||||
from ninja import Router, Schema, FilterSchema, Field, Query
|
from ninja import Router, Schema, FilterSchema, Field, Query
|
||||||
from ninja.pagination import paginate
|
from ninja.pagination import paginate, PaginationBase
|
||||||
|
|
||||||
from core.models import Snapshot, ArchiveResult, Tag
|
from core.models import Snapshot, ArchiveResult, Tag
|
||||||
from abid_utils.abid import ABID
|
from abid_utils.abid import ABID
|
||||||
|
|
@ -17,23 +20,61 @@ router = Router(tags=['Core Models'])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CustomPagination(PaginationBase):
|
||||||
|
class Input(Schema):
|
||||||
|
limit: int = 200
|
||||||
|
offset: int = 0
|
||||||
|
page: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class Output(Schema):
|
||||||
|
total_items: int
|
||||||
|
total_pages: int
|
||||||
|
page: int
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
num_items: int
|
||||||
|
items: List[Any]
|
||||||
|
|
||||||
|
def paginate_queryset(self, queryset, pagination: Input, **params):
|
||||||
|
limit = min(pagination.limit, 500)
|
||||||
|
offset = pagination.offset or (pagination.page * limit)
|
||||||
|
total = queryset.count()
|
||||||
|
total_pages = math.ceil(total / limit)
|
||||||
|
current_page = math.ceil(offset / (limit + 1))
|
||||||
|
items = queryset[offset : offset + limit]
|
||||||
|
return {
|
||||||
|
'total_items': total,
|
||||||
|
'total_pages': total_pages,
|
||||||
|
'page': current_page,
|
||||||
|
'limit': limit,
|
||||||
|
'offset': offset,
|
||||||
|
'num_items': len(items),
|
||||||
|
'items': items,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
### ArchiveResult #########################################################################
|
### ArchiveResult #########################################################################
|
||||||
|
|
||||||
class ArchiveResultSchema(Schema):
|
class ArchiveResultSchema(Schema):
|
||||||
|
TYPE: str = 'core.models.ArchiveResult'
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
old_id: int
|
||||||
abid: str
|
abid: str
|
||||||
uuid: UUID
|
|
||||||
pk: str
|
|
||||||
modified: datetime
|
modified: datetime
|
||||||
created: datetime
|
created: datetime
|
||||||
created_by_id: str
|
created_by_id: str
|
||||||
|
created_by_username: str
|
||||||
|
|
||||||
snapshot_abid: str
|
snapshot_abid: str
|
||||||
|
snapshot_timestamp: str
|
||||||
snapshot_url: str
|
snapshot_url: str
|
||||||
snapshot_tags: str
|
snapshot_tags: str
|
||||||
|
|
||||||
extractor: str
|
extractor: str
|
||||||
cmd_version: str
|
cmd_version: Optional[str]
|
||||||
cmd: List[str]
|
cmd: List[str]
|
||||||
pwd: str
|
pwd: str
|
||||||
status: str
|
status: str
|
||||||
|
|
@ -43,6 +84,11 @@ class ArchiveResultSchema(Schema):
|
||||||
def resolve_created_by_id(obj):
|
def resolve_created_by_id(obj):
|
||||||
return str(obj.created_by_id)
|
return str(obj.created_by_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_created_by_username(obj):
|
||||||
|
User = get_user_model()
|
||||||
|
return User.objects.get(id=obj.created_by_id).username
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def resolve_pk(obj):
|
def resolve_pk(obj):
|
||||||
return str(obj.pk)
|
return str(obj.pk)
|
||||||
|
|
@ -59,6 +105,10 @@ class ArchiveResultSchema(Schema):
|
||||||
def resolve_created(obj):
|
def resolve_created(obj):
|
||||||
return obj.start_ts
|
return obj.start_ts
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_snapshot_timestamp(obj):
|
||||||
|
return obj.snapshot.timestamp
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def resolve_snapshot_url(obj):
|
def resolve_snapshot_url(obj):
|
||||||
return obj.snapshot.url
|
return obj.snapshot.url
|
||||||
|
|
@ -73,11 +123,10 @@ class ArchiveResultSchema(Schema):
|
||||||
|
|
||||||
|
|
||||||
class ArchiveResultFilterSchema(FilterSchema):
|
class ArchiveResultFilterSchema(FilterSchema):
|
||||||
uuid: Optional[UUID] = Field(None, q='uuid')
|
id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||||
# abid: Optional[str] = Field(None, q='abid')
|
|
||||||
|
|
||||||
search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains'])
|
search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||||
snapshot_uuid: Optional[UUID] = Field(None, q='snapshot_uuid__icontains')
|
snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||||
snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
|
snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
|
||||||
snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
|
snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
|
||||||
|
|
||||||
|
|
@ -93,19 +142,19 @@ class ArchiveResultFilterSchema(FilterSchema):
|
||||||
created__lt: Optional[datetime] = Field(None, q='updated__lt')
|
created__lt: Optional[datetime] = Field(None, q='updated__lt')
|
||||||
|
|
||||||
|
|
||||||
@router.get("/archiveresults", response=List[ArchiveResultSchema])
|
@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
|
||||||
@paginate
|
@paginate(CustomPagination)
|
||||||
def list_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
|
def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
|
||||||
"""List all ArchiveResult entries matching these filters."""
|
"""List all ArchiveResult entries matching these filters."""
|
||||||
qs = ArchiveResult.objects.all()
|
qs = ArchiveResult.objects.all()
|
||||||
results = filters.filter(qs)
|
results = filters.filter(qs).distinct()
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
|
@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
|
||||||
def get_archiveresult(request, archiveresult_id: str):
|
def get_archiveresult(request, archiveresult_id: str):
|
||||||
"""Get a specific ArchiveResult by abid, uuid, or pk."""
|
"""Get a specific ArchiveResult by pk, abid, or old_id."""
|
||||||
return ArchiveResult.objects.get(Q(pk__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(uuid__icontains=archiveresult_id))
|
return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
|
||||||
|
|
||||||
|
|
||||||
# @router.post("/archiveresult", response=ArchiveResultSchema)
|
# @router.post("/archiveresult", response=ArchiveResultSchema)
|
||||||
|
|
@ -137,12 +186,16 @@ def get_archiveresult(request, archiveresult_id: str):
|
||||||
|
|
||||||
|
|
||||||
class SnapshotSchema(Schema):
|
class SnapshotSchema(Schema):
|
||||||
|
TYPE: str = 'core.models.Snapshot'
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
old_id: UUID
|
||||||
abid: str
|
abid: str
|
||||||
uuid: UUID
|
|
||||||
pk: str
|
|
||||||
modified: datetime
|
modified: datetime
|
||||||
created: datetime
|
created: datetime
|
||||||
created_by_id: str
|
created_by_id: str
|
||||||
|
created_by_username: str
|
||||||
|
|
||||||
url: str
|
url: str
|
||||||
tags: str
|
tags: str
|
||||||
|
|
@ -161,6 +214,11 @@ class SnapshotSchema(Schema):
|
||||||
def resolve_created_by_id(obj):
|
def resolve_created_by_id(obj):
|
||||||
return str(obj.created_by_id)
|
return str(obj.created_by_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_created_by_username(obj):
|
||||||
|
User = get_user_model()
|
||||||
|
return User.objects.get(id=obj.created_by_id).username
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def resolve_pk(obj):
|
def resolve_pk(obj):
|
||||||
return str(obj.pk)
|
return str(obj.pk)
|
||||||
|
|
@ -189,10 +247,14 @@ class SnapshotSchema(Schema):
|
||||||
|
|
||||||
|
|
||||||
class SnapshotFilterSchema(FilterSchema):
|
class SnapshotFilterSchema(FilterSchema):
|
||||||
|
id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
|
||||||
|
|
||||||
|
old_id: Optional[str] = Field(None, q='old_id__icontains')
|
||||||
abid: Optional[str] = Field(None, q='abid__icontains')
|
abid: Optional[str] = Field(None, q='abid__icontains')
|
||||||
uuid: Optional[str] = Field(None, q='uuid__icontains')
|
|
||||||
pk: Optional[str] = Field(None, q='pk__icontains')
|
created_by_id: str = Field(None, q='created_by_id')
|
||||||
created_by_id: str = Field(None, q='created_by_id__icontains')
|
created_by_username: str = Field(None, q='created_by__username__icontains')
|
||||||
|
|
||||||
created__gte: datetime = Field(None, q='created__gte')
|
created__gte: datetime = Field(None, q='created__gte')
|
||||||
created__lt: datetime = Field(None, q='created__lt')
|
created__lt: datetime = Field(None, q='created__lt')
|
||||||
created: datetime = Field(None, q='created')
|
created: datetime = Field(None, q='created')
|
||||||
|
|
@ -200,7 +262,7 @@ class SnapshotFilterSchema(FilterSchema):
|
||||||
modified__gte: datetime = Field(None, q='modified__gte')
|
modified__gte: datetime = Field(None, q='modified__gte')
|
||||||
modified__lt: datetime = Field(None, q='modified__lt')
|
modified__lt: datetime = Field(None, q='modified__lt')
|
||||||
|
|
||||||
search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'abid__icontains', 'uuid__icontains'])
|
search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
|
||||||
url: Optional[str] = Field(None, q='url')
|
url: Optional[str] = Field(None, q='url')
|
||||||
tag: Optional[str] = Field(None, q='tags__name')
|
tag: Optional[str] = Field(None, q='tags__name')
|
||||||
title: Optional[str] = Field(None, q='title__icontains')
|
title: Optional[str] = Field(None, q='title__icontains')
|
||||||
|
|
@ -211,35 +273,33 @@ class SnapshotFilterSchema(FilterSchema):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/snapshots", response=List[SnapshotSchema])
|
@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
|
||||||
@paginate
|
@paginate(CustomPagination)
|
||||||
def list_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=True):
|
def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
|
||||||
"""List all Snapshot entries matching these filters."""
|
"""List all Snapshot entries matching these filters."""
|
||||||
request.with_archiveresults = with_archiveresults
|
request.with_archiveresults = with_archiveresults
|
||||||
|
|
||||||
qs = Snapshot.objects.all()
|
qs = Snapshot.objects.all()
|
||||||
results = filters.filter(qs)
|
results = filters.filter(qs).distinct()
|
||||||
return results
|
return results
|
||||||
|
|
||||||
@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema)
|
@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
|
||||||
def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
|
def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
|
||||||
"""Get a specific Snapshot by abid, uuid, or pk."""
|
"""Get a specific Snapshot by abid, uuid, or pk."""
|
||||||
request.with_archiveresults = with_archiveresults
|
request.with_archiveresults = with_archiveresults
|
||||||
snapshot = None
|
snapshot = None
|
||||||
try:
|
try:
|
||||||
snapshot = Snapshot.objects.get(Q(uuid__startswith=snapshot_id) | Q(abid__startswith=snapshot_id)| Q(pk__startswith=snapshot_id))
|
snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
|
||||||
except Snapshot.DoesNotExist:
|
except Snapshot.DoesNotExist:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
snapshot = snapshot or Snapshot.objects.get()
|
snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
|
||||||
except Snapshot.DoesNotExist:
|
except Snapshot.DoesNotExist:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
if not snapshot:
|
||||||
snapshot = snapshot or Snapshot.objects.get(Q(uuid__icontains=snapshot_id) | Q(abid__icontains=snapshot_id))
|
raise Snapshot.DoesNotExist
|
||||||
except Snapshot.DoesNotExist:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return snapshot
|
return snapshot
|
||||||
|
|
||||||
|
|
@ -271,21 +331,94 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
|
||||||
|
|
||||||
|
|
||||||
class TagSchema(Schema):
|
class TagSchema(Schema):
|
||||||
abid: Optional[UUID] = Field(None, q='abid')
|
TYPE: str = 'core.models.Tag'
|
||||||
uuid: Optional[UUID] = Field(None, q='uuid')
|
|
||||||
pk: Optional[UUID] = Field(None, q='pk')
|
id: UUID
|
||||||
|
old_id: str
|
||||||
|
abid: str
|
||||||
|
|
||||||
modified: datetime
|
modified: datetime
|
||||||
created: datetime
|
created: datetime
|
||||||
created_by_id: str
|
created_by_id: str
|
||||||
|
created_by_username: str
|
||||||
|
|
||||||
name: str
|
name: str
|
||||||
slug: str
|
slug: str
|
||||||
|
num_snapshots: int
|
||||||
|
snapshots: List[SnapshotSchema]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_old_id(obj):
|
||||||
|
return str(obj.old_id)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def resolve_created_by_id(obj):
|
def resolve_created_by_id(obj):
|
||||||
return str(obj.created_by_id)
|
return str(obj.created_by_id)
|
||||||
|
|
||||||
@router.get("/tags", response=List[TagSchema])
|
@staticmethod
|
||||||
def list_tags(request):
|
def resolve_created_by_username(obj):
|
||||||
return Tag.objects.all()
|
User = get_user_model()
|
||||||
|
return User.objects.get(id=obj.created_by_id).username
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_num_snapshots(obj, context):
|
||||||
|
return obj.snapshot_set.all().distinct().count()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def resolve_snapshots(obj, context):
|
||||||
|
if context['request'].with_snapshots:
|
||||||
|
return obj.snapshot_set.all().distinct()
|
||||||
|
return Snapshot.objects.none()
|
||||||
|
|
||||||
|
@router.get("/tags", response=List[TagSchema], url_name="get_tags")
|
||||||
|
@paginate(CustomPagination)
|
||||||
|
def get_tags(request):
|
||||||
|
request.with_snapshots = False
|
||||||
|
request.with_archiveresults = False
|
||||||
|
return Tag.objects.all().distinct()
|
||||||
|
|
||||||
|
@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
|
||||||
|
def get_tag(request, tag_id: str, with_snapshots: bool=True):
|
||||||
|
request.with_snapshots = with_snapshots
|
||||||
|
request.with_archiveresults = False
|
||||||
|
tag = None
|
||||||
|
try:
|
||||||
|
tag = tag or Tag.objects.get(old_id__icontains=tag_id)
|
||||||
|
except (Tag.DoesNotExist, ValidationError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
tag = Tag.objects.get(abid__icontains=tag_id)
|
||||||
|
except (Tag.DoesNotExist, ValidationError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
tag = tag or Tag.objects.get(id__icontains=tag_id)
|
||||||
|
except (Tag.DoesNotExist, ValidationError):
|
||||||
|
pass
|
||||||
|
return tag
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
|
||||||
|
def get_any(request, abid: str):
|
||||||
|
request.with_snapshots = False
|
||||||
|
request.with_archiveresults = False
|
||||||
|
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
response = response or get_snapshot(request, abid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = response or get_archiveresult(request, abid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = response or get_tag(request, abid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
|
||||||
|
|
@ -1036,6 +1036,11 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
|
||||||
'enabled': True,
|
'enabled': True,
|
||||||
'is_valid': config['SOURCES_DIR'].exists(),
|
'is_valid': config['SOURCES_DIR'].exists(),
|
||||||
},
|
},
|
||||||
|
'PERSONAS_DIR': {
|
||||||
|
'path': config['PERSONAS_DIR'].resolve(),
|
||||||
|
'enabled': True,
|
||||||
|
'is_valid': config['PERSONAS_DIR'].exists(),
|
||||||
|
},
|
||||||
'LOGS_DIR': {
|
'LOGS_DIR': {
|
||||||
'path': config['LOGS_DIR'].resolve(),
|
'path': config['LOGS_DIR'].resolve(),
|
||||||
'enabled': True,
|
'enabled': True,
|
||||||
|
|
@ -1051,11 +1056,6 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
|
||||||
'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
|
'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
|
||||||
'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
|
'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
|
||||||
},
|
},
|
||||||
'PERSONAS_DIR': {
|
|
||||||
'path': config['PERSONAS_DIR'].resolve(),
|
|
||||||
'enabled': True,
|
|
||||||
'is_valid': config['PERSONAS_DIR'].exists(),
|
|
||||||
},
|
|
||||||
# managed by bin/docker_entrypoint.sh and python-crontab:
|
# managed by bin/docker_entrypoint.sh and python-crontab:
|
||||||
# 'CRONTABS_DIR': {
|
# 'CRONTABS_DIR': {
|
||||||
# 'path': config['CRONTABS_DIR'].resolve(),
|
# 'path': config['CRONTABS_DIR'].resolve(),
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,19 @@
|
||||||
__package__ = 'archivebox.core'
|
__package__ = 'archivebox.core'
|
||||||
|
|
||||||
|
import json
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from contextlib import redirect_stdout
|
from contextlib import redirect_stdout
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.db.models import Count
|
from django.db.models import Count, Q
|
||||||
from django.urls import path
|
from django.urls import path, reverse
|
||||||
from django.utils.html import format_html
|
from django.utils.html import format_html
|
||||||
from django.utils.safestring import mark_safe
|
from django.utils.safestring import mark_safe
|
||||||
from django.shortcuts import render, redirect
|
from django.shortcuts import render, redirect
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.exceptions import ValidationError
|
||||||
from django import forms
|
from django import forms
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -20,7 +22,7 @@ from signal_webhooks.admin import WebhookAdmin, get_webhook_model
|
||||||
|
|
||||||
from ..util import htmldecode, urldecode, ansi_to_html
|
from ..util import htmldecode, urldecode, ansi_to_html
|
||||||
|
|
||||||
from core.models import Snapshot, ArchiveResult, Tag
|
from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
|
||||||
from core.forms import AddLinkForm
|
from core.forms import AddLinkForm
|
||||||
|
|
||||||
from core.mixins import SearchResultsAdminMixin
|
from core.mixins import SearchResultsAdminMixin
|
||||||
|
|
@ -124,31 +126,55 @@ archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archiveb
|
||||||
|
|
||||||
|
|
||||||
class ArchiveResultInline(admin.TabularInline):
|
class ArchiveResultInline(admin.TabularInline):
|
||||||
|
name = 'Archive Results Log'
|
||||||
model = ArchiveResult
|
model = ArchiveResult
|
||||||
|
# fk_name = 'snapshot'
|
||||||
|
extra = 1
|
||||||
|
readonly_fields = ('result_id', 'start_ts', 'end_ts', 'extractor', 'command', 'cmd_version')
|
||||||
|
fields = ('id', *readonly_fields, 'status', 'output')
|
||||||
|
show_change_link = True
|
||||||
|
# # classes = ['collapse']
|
||||||
|
# # list_display_links = ['abid']
|
||||||
|
|
||||||
|
def result_id(self, obj):
|
||||||
|
return format_html('<a href="{}"><small><code>[{}]</code></small></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
|
||||||
|
|
||||||
|
def command(self, obj):
|
||||||
|
return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
|
||||||
|
|
||||||
|
|
||||||
class TagInline(admin.TabularInline):
|
class TagInline(admin.TabularInline):
|
||||||
model = Snapshot.tags.through
|
model = Tag.snapshot_set.through
|
||||||
|
# fk_name = 'snapshot'
|
||||||
|
fields = ('id', 'tag')
|
||||||
|
extra = 1
|
||||||
|
# min_num = 1
|
||||||
|
max_num = 1000
|
||||||
|
autocomplete_fields = (
|
||||||
|
'tag',
|
||||||
|
)
|
||||||
|
|
||||||
from django.contrib.admin.helpers import ActionForm
|
from django.contrib.admin.helpers import ActionForm
|
||||||
from django.contrib.admin.widgets import AutocompleteSelectMultiple
|
from django.contrib.admin.widgets import FilteredSelectMultiple
|
||||||
|
|
||||||
class AutocompleteTags:
|
# class AutocompleteTags:
|
||||||
model = Tag
|
# model = Tag
|
||||||
search_fields = ['name']
|
# search_fields = ['name']
|
||||||
name = 'tags'
|
# name = 'name'
|
||||||
remote_field = TagInline
|
# # source_field = 'name'
|
||||||
|
# remote_field = Tag._meta.get_field('name')
|
||||||
|
|
||||||
class AutocompleteTagsAdminStub:
|
# class AutocompleteTagsAdminStub:
|
||||||
name = 'admin'
|
# name = 'admin'
|
||||||
|
|
||||||
|
|
||||||
class SnapshotActionForm(ActionForm):
|
class SnapshotActionForm(ActionForm):
|
||||||
tags = forms.ModelMultipleChoiceField(
|
tags = forms.ModelMultipleChoiceField(
|
||||||
queryset=Tag.objects.all(),
|
queryset=Tag.objects.all(),
|
||||||
required=False,
|
required=False,
|
||||||
widget=AutocompleteSelectMultiple(
|
widget=FilteredSelectMultiple(
|
||||||
AutocompleteTags(),
|
'core_tag__name',
|
||||||
AutocompleteTagsAdminStub(),
|
False,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -168,48 +194,92 @@ def get_abid_info(self, obj):
|
||||||
return format_html(
|
return format_html(
|
||||||
# URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
|
# URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
|
||||||
'''
|
'''
|
||||||
ABID: <code style="font-size: 16px; user-select: all"><b>{}</b></code><br/>
|
<a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
|
||||||
TS: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
|
<br/><hr/>
|
||||||
URI: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
|
<div style="opacity: 0.8">
|
||||||
SUBTYPE: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
|
TS: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
|
||||||
RAND: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/><br/>
|
URI: <code style="font-size: 10px; user-select: all"><b>{}</b></code> (<span style="display:inline-block; vertical-align: -4px; user-select: all; width: 230px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</span>)<br/>
|
||||||
ABID AS UUID: <code style="font-size: 10px; user-select: all">{}</code> <br/><br/>
|
SUBTYPE: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})
|
||||||
|
RAND: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})
|
||||||
.uuid: <code style="font-size: 10px; user-select: all">{}</code> <br/>
|
SALT: <code style="font-size: 10px; user-select: all"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code>
|
||||||
.id: <code style="font-size: 10px; user-select: all">{}</code> <br/>
|
<br/><hr/>
|
||||||
.pk: <code style="font-size: 10px; user-select: all">{}</code> <br/><br/>
|
<small style="opacity: 0.8">.abid: <code style="font-size: 10px; user-select: all">{}</code></small><br/>
|
||||||
|
<small style="opacity: 0.8">.abid.uuid: <code style="font-size: 10px; user-select: all">{}</code></small><br/>
|
||||||
|
<small style="opacity: 0.8">.id: <code style="font-size: 10px; user-select: all">{}</code></small><br/>
|
||||||
|
<small style="opacity: 0.5">.old_id: <code style="font-size: 10px; user-select: all">{}</code></small><br/>
|
||||||
|
</div>
|
||||||
''',
|
''',
|
||||||
obj.abid,
|
obj.api_url, obj.api_url, obj.api_docs_url,
|
||||||
obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
|
obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
|
||||||
obj.ABID.uri, str(obj.abid_values['uri']),
|
obj.ABID.uri, str(obj.abid_values['uri']),
|
||||||
obj.ABID.subtype, str(obj.abid_values['subtype']),
|
obj.ABID.subtype, str(obj.abid_values['subtype']),
|
||||||
obj.ABID.rand, str(obj.abid_values['rand'])[-7:],
|
obj.ABID.rand, str(obj.abid_values['rand'])[-7:],
|
||||||
obj.ABID.uuid,
|
obj.ABID.uri_salt,
|
||||||
obj.uuid,
|
str(obj.abid),
|
||||||
|
str(obj.ABID.uuid),
|
||||||
obj.id,
|
obj.id,
|
||||||
obj.pk,
|
getattr(obj, 'old_id', ''),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@admin.register(Snapshot, site=archivebox_admin)
|
@admin.register(Snapshot, site=archivebox_admin)
|
||||||
class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
||||||
|
class Meta:
|
||||||
|
model = Snapshot
|
||||||
|
|
||||||
list_display = ('added', 'title_str', 'files', 'size', 'url_str')
|
list_display = ('added', 'title_str', 'files', 'size', 'url_str')
|
||||||
|
# list_editable = ('title',)
|
||||||
sort_fields = ('title_str', 'url_str', 'added', 'files')
|
sort_fields = ('title_str', 'url_str', 'added', 'files')
|
||||||
readonly_fields = ('admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'identifiers')
|
readonly_fields = ('tags', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
|
||||||
search_fields = ('id', 'url', 'abid', 'uuid', 'timestamp', 'title', 'tags__name')
|
search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
|
||||||
fields = ('url', 'timestamp', 'created_by', 'tags', 'title', *readonly_fields)
|
list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags')
|
||||||
list_filter = ('added', 'updated', 'tags', 'archiveresult__status', 'created_by')
|
fields = ('url', 'created_by', 'title', *readonly_fields)
|
||||||
ordering = ['-added']
|
ordering = ['-added']
|
||||||
actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
|
actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
|
||||||
autocomplete_fields = ['tags']
|
autocomplete_fields = ['tags']
|
||||||
inlines = [ArchiveResultInline]
|
inlines = [TagInline, ArchiveResultInline]
|
||||||
list_per_page = SNAPSHOTS_PER_PAGE
|
list_per_page = SNAPSHOTS_PER_PAGE
|
||||||
|
|
||||||
action_form = SnapshotActionForm
|
action_form = SnapshotActionForm
|
||||||
|
|
||||||
|
save_on_top = True
|
||||||
|
|
||||||
def changelist_view(self, request, extra_context=None):
|
def changelist_view(self, request, extra_context=None):
|
||||||
extra_context = extra_context or {}
|
extra_context = extra_context or {}
|
||||||
|
try:
|
||||||
return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
|
return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
|
||||||
|
except Exception as e:
|
||||||
|
self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
|
||||||
|
return super().changelist_view(request, GLOBAL_CONTEXT)
|
||||||
|
|
||||||
|
def change_view(self, request, object_id, form_url="", extra_context=None):
|
||||||
|
snapshot = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
snapshot = snapshot or Snapshot.objects.get(id=object_id)
|
||||||
|
except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
|
||||||
|
except (Snapshot.DoesNotExist, ValidationError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
|
||||||
|
except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
if snapshot:
|
||||||
|
object_id = str(snapshot.id)
|
||||||
|
|
||||||
|
return super().change_view(
|
||||||
|
request,
|
||||||
|
object_id,
|
||||||
|
form_url,
|
||||||
|
extra_context=extra_context,
|
||||||
|
)
|
||||||
|
|
||||||
def get_urls(self):
|
def get_urls(self):
|
||||||
urls = super().get_urls()
|
urls = super().get_urls()
|
||||||
|
|
@ -220,7 +290,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
||||||
|
|
||||||
def get_queryset(self, request):
|
def get_queryset(self, request):
|
||||||
self.request = request
|
self.request = request
|
||||||
return super().get_queryset(request).prefetch_related('tags').annotate(archiveresult_count=Count('archiveresult'))
|
return super().get_queryset(request).prefetch_related('tags', 'archiveresult_set').annotate(archiveresult_count=Count('archiveresult'))
|
||||||
|
|
||||||
def tag_list(self, obj):
|
def tag_list(self, obj):
|
||||||
return ', '.join(obj.tags.values_list('name', flat=True))
|
return ', '.join(obj.tags.values_list('name', flat=True))
|
||||||
|
|
@ -281,8 +351,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
||||||
obj.extension or '-',
|
obj.extension or '-',
|
||||||
)
|
)
|
||||||
|
|
||||||
def identifiers(self, obj):
|
def API(self, obj):
|
||||||
|
try:
|
||||||
return get_abid_info(self, obj)
|
return get_abid_info(self, obj)
|
||||||
|
except Exception as e:
|
||||||
|
return str(e)
|
||||||
|
|
||||||
@admin.display(
|
@admin.display(
|
||||||
description='Title',
|
description='Title',
|
||||||
|
|
@ -442,20 +515,34 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# @admin.register(SnapshotTag, site=archivebox_admin)
|
||||||
|
# class SnapshotTagAdmin(admin.ModelAdmin):
|
||||||
|
# list_display = ('id', 'snapshot', 'tag')
|
||||||
|
# sort_fields = ('id', 'snapshot', 'tag')
|
||||||
|
# search_fields = ('id', 'snapshot_id', 'tag_id')
|
||||||
|
# fields = ('snapshot', 'id')
|
||||||
|
# actions = ['delete_selected']
|
||||||
|
# ordering = ['-id']
|
||||||
|
|
||||||
|
# def API(self, obj):
|
||||||
|
# return get_abid_info(self, obj)
|
||||||
|
|
||||||
|
|
||||||
@admin.register(Tag, site=archivebox_admin)
|
@admin.register(Tag, site=archivebox_admin)
|
||||||
class TagAdmin(admin.ModelAdmin):
|
class TagAdmin(admin.ModelAdmin):
|
||||||
list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'abid')
|
list_display = ('abid', 'name', 'created', 'created_by', 'num_snapshots', 'snapshots')
|
||||||
sort_fields = ('id', 'name', 'slug', 'abid')
|
sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
|
||||||
readonly_fields = ('created', 'modified', 'identifiers', 'num_snapshots', 'snapshots')
|
readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
|
||||||
search_fields = ('id', 'abid', 'uuid', 'name', 'slug')
|
search_fields = ('abid', 'name', 'slug')
|
||||||
fields = ('name', 'slug', 'created_by', *readonly_fields, )
|
fields = ('name', 'created_by', *readonly_fields)
|
||||||
actions = ['delete_selected']
|
actions = ['delete_selected']
|
||||||
ordering = ['-id']
|
ordering = ['-created']
|
||||||
|
|
||||||
def identifiers(self, obj):
|
def API(self, obj):
|
||||||
|
try:
|
||||||
return get_abid_info(self, obj)
|
return get_abid_info(self, obj)
|
||||||
|
except Exception as e:
|
||||||
|
return str(e)
|
||||||
|
|
||||||
def num_snapshots(self, tag):
|
def num_snapshots(self, tag):
|
||||||
return format_html(
|
return format_html(
|
||||||
|
|
@ -468,11 +555,10 @@ class TagAdmin(admin.ModelAdmin):
|
||||||
total_count = tag.snapshot_set.count()
|
total_count = tag.snapshot_set.count()
|
||||||
return mark_safe('<br/>'.join(
|
return mark_safe('<br/>'.join(
|
||||||
format_html(
|
format_html(
|
||||||
'{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
|
'<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
|
||||||
snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
|
|
||||||
snap.pk,
|
snap.pk,
|
||||||
snap.abid,
|
snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
|
||||||
snap.url,
|
snap.url[:64],
|
||||||
)
|
)
|
||||||
for snap in tag.snapshot_set.order_by('-updated')[:10]
|
for snap in tag.snapshot_set.order_by('-updated')[:10]
|
||||||
) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
|
) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
|
||||||
|
|
@ -482,9 +568,9 @@ class TagAdmin(admin.ModelAdmin):
|
||||||
class ArchiveResultAdmin(admin.ModelAdmin):
|
class ArchiveResultAdmin(admin.ModelAdmin):
|
||||||
list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
|
list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
|
||||||
sort_fields = ('start_ts', 'extractor', 'status')
|
sort_fields = ('start_ts', 'extractor', 'status')
|
||||||
readonly_fields = ('snapshot_info', 'tags_str', 'created_by', 'created', 'modified', 'identifiers')
|
readonly_fields = ('snapshot_info', 'tags_str', 'created', 'modified', 'API')
|
||||||
search_fields = ('id', 'uuid', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
|
search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
|
||||||
fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd', 'start_ts', 'end_ts', 'cmd_version', *readonly_fields)
|
fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', *readonly_fields)
|
||||||
autocomplete_fields = ['snapshot']
|
autocomplete_fields = ['snapshot']
|
||||||
|
|
||||||
list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
|
list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
|
||||||
|
|
@ -503,8 +589,11 @@ class ArchiveResultAdmin(admin.ModelAdmin):
|
||||||
result.snapshot.url[:128],
|
result.snapshot.url[:128],
|
||||||
)
|
)
|
||||||
|
|
||||||
def identifiers(self, obj):
|
def API(self, obj):
|
||||||
|
try:
|
||||||
return get_abid_info(self, obj)
|
return get_abid_info(self, obj)
|
||||||
|
except Exception as e:
|
||||||
|
return str(e)
|
||||||
|
|
||||||
@admin.display(
|
@admin.display(
|
||||||
description='Snapshot Tags'
|
description='Snapshot Tags'
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
from django.db import migrations
|
from django.db import migrations
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from abid_utils.abid import abid_from_values
|
from abid_utils.abid import abid_from_values, DEFAULT_ABID_URI_SALT
|
||||||
|
|
||||||
|
|
||||||
def calculate_abid(self):
|
def calculate_abid(self):
|
||||||
|
|
@ -41,18 +41,21 @@ def calculate_abid(self):
|
||||||
uri=uri,
|
uri=uri,
|
||||||
subtype=subtype,
|
subtype=subtype,
|
||||||
rand=rand,
|
rand=rand,
|
||||||
|
salt=DEFAULT_ABID_URI_SALT,
|
||||||
)
|
)
|
||||||
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
||||||
return abid
|
return abid
|
||||||
|
|
||||||
|
|
||||||
def copy_snapshot_uuids(apps, schema_editor):
|
def copy_snapshot_uuids(apps, schema_editor):
|
||||||
|
print(' Copying snapshot.id -> snapshot.uuid...')
|
||||||
Snapshot = apps.get_model("core", "Snapshot")
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
for snapshot in Snapshot.objects.all():
|
for snapshot in Snapshot.objects.all():
|
||||||
snapshot.uuid = snapshot.id
|
snapshot.uuid = snapshot.id
|
||||||
snapshot.save(update_fields=["uuid"])
|
snapshot.save(update_fields=["uuid"])
|
||||||
|
|
||||||
def generate_snapshot_abids(apps, schema_editor):
|
def generate_snapshot_abids(apps, schema_editor):
|
||||||
|
print(' Generating snapshot.abid values...')
|
||||||
Snapshot = apps.get_model("core", "Snapshot")
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
for snapshot in Snapshot.objects.all():
|
for snapshot in Snapshot.objects.all():
|
||||||
snapshot.abid_prefix = 'snp_'
|
snapshot.abid_prefix = 'snp_'
|
||||||
|
|
@ -62,9 +65,11 @@ def generate_snapshot_abids(apps, schema_editor):
|
||||||
snapshot.abid_rand_src = 'self.uuid'
|
snapshot.abid_rand_src = 'self.uuid'
|
||||||
|
|
||||||
snapshot.abid = calculate_abid(snapshot)
|
snapshot.abid = calculate_abid(snapshot)
|
||||||
snapshot.save(update_fields=["abid"])
|
snapshot.uuid = snapshot.abid.uuid
|
||||||
|
snapshot.save(update_fields=["abid", "uuid"])
|
||||||
|
|
||||||
def generate_archiveresult_abids(apps, schema_editor):
|
def generate_archiveresult_abids(apps, schema_editor):
|
||||||
|
print(' Generating ArchiveResult.abid values... (may take an hour or longer for large collections...)')
|
||||||
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
||||||
Snapshot = apps.get_model("core", "Snapshot")
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
for result in ArchiveResult.objects.all():
|
for result in ArchiveResult.objects.all():
|
||||||
|
|
|
||||||
106
archivebox/core/migrations/0027_update_snapshot_ids.py
Normal file
106
archivebox/core/migrations/0027_update_snapshot_ids.py
Normal file
|
|
@ -0,0 +1,106 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 02:48
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
from datetime import datetime
|
||||||
|
from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_abid(self):
|
||||||
|
"""
|
||||||
|
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||||
|
"""
|
||||||
|
prefix = self.abid_prefix
|
||||||
|
ts = eval(self.abid_ts_src)
|
||||||
|
uri = eval(self.abid_uri_src)
|
||||||
|
subtype = eval(self.abid_subtype_src)
|
||||||
|
rand = eval(self.abid_rand_src)
|
||||||
|
|
||||||
|
if (not prefix) or prefix == 'obj_':
|
||||||
|
suggested_abid = self.__class__.__name__[:3].lower()
|
||||||
|
raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
|
||||||
|
|
||||||
|
if not ts:
|
||||||
|
ts = datetime.utcfromtimestamp(0)
|
||||||
|
print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
|
||||||
|
|
||||||
|
if not uri:
|
||||||
|
uri = str(self)
|
||||||
|
print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
|
||||||
|
|
||||||
|
if not subtype:
|
||||||
|
subtype = self.__class__.__name__
|
||||||
|
print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
|
||||||
|
|
||||||
|
if not rand:
|
||||||
|
rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
|
||||||
|
print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
|
||||||
|
|
||||||
|
abid = abid_from_values(
|
||||||
|
prefix=prefix,
|
||||||
|
ts=ts,
|
||||||
|
uri=uri,
|
||||||
|
subtype=subtype,
|
||||||
|
rand=rand,
|
||||||
|
salt=DEFAULT_ABID_URI_SALT,
|
||||||
|
)
|
||||||
|
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
||||||
|
return abid
|
||||||
|
|
||||||
|
def update_snapshot_ids(apps, schema_editor):
|
||||||
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
|
num_total = Snapshot.objects.all().count()
|
||||||
|
print(f' Updating {num_total} Snapshot.id, Snapshot.uuid values in place...')
|
||||||
|
for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()):
|
||||||
|
assert snapshot.abid
|
||||||
|
snapshot.abid_prefix = 'snp_'
|
||||||
|
snapshot.abid_ts_src = 'self.added'
|
||||||
|
snapshot.abid_uri_src = 'self.url'
|
||||||
|
snapshot.abid_subtype_src = '"01"'
|
||||||
|
snapshot.abid_rand_src = 'self.uuid'
|
||||||
|
|
||||||
|
snapshot.abid = calculate_abid(snapshot)
|
||||||
|
snapshot.uuid = snapshot.abid.uuid
|
||||||
|
snapshot.save(update_fields=["abid", "uuid"])
|
||||||
|
assert str(ABID.parse(snapshot.abid).uuid) == str(snapshot.uuid)
|
||||||
|
if idx % 1000 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} Snapshot objects...')
|
||||||
|
|
||||||
|
def update_archiveresult_ids(apps, schema_editor):
|
||||||
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
||||||
|
num_total = ArchiveResult.objects.all().count()
|
||||||
|
print(f' Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
|
||||||
|
for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator()):
|
||||||
|
assert result.abid
|
||||||
|
result.abid_prefix = 'res_'
|
||||||
|
result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)
|
||||||
|
result.snapshot_added = result.snapshot.added
|
||||||
|
result.snapshot_url = result.snapshot.url
|
||||||
|
result.abid_ts_src = 'self.snapshot_added'
|
||||||
|
result.abid_uri_src = 'self.snapshot_url'
|
||||||
|
result.abid_subtype_src = 'self.extractor'
|
||||||
|
result.abid_rand_src = 'self.id'
|
||||||
|
|
||||||
|
result.abid = calculate_abid(result)
|
||||||
|
result.uuid = result.abid.uuid
|
||||||
|
result.uuid = ABID.parse(result.abid).uuid
|
||||||
|
result.save(update_fields=["abid", "uuid"])
|
||||||
|
assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
|
||||||
|
if idx % 5000 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0026_archiveresult_created_archiveresult_created_by_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RunPython(update_snapshot_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
19
archivebox/core/migrations/0028_alter_archiveresult_uuid.py
Normal file
19
archivebox/core/migrations/0028_alter_archiveresult_uuid.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 04:28
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0027_update_snapshot_ids'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4),
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0029_alter_archiveresult_id.py
Normal file
18
archivebox/core/migrations/0029_alter_archiveresult_id.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 04:28
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0028_alter_archiveresult_uuid'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='id',
|
||||||
|
field=models.BigIntegerField(primary_key=True, serialize=False, verbose_name='ID'),
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0030_alter_archiveresult_uuid.py
Normal file
18
archivebox/core/migrations/0030_alter_archiveresult_uuid.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:00
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0029_alter_archiveresult_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:09
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0030_alter_archiveresult_uuid'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='id',
|
||||||
|
field=models.IntegerField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='ID'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, unique=True),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, unique=True),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, null=True, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0032_alter_archiveresult_id.py
Normal file
19
archivebox/core/migrations/0032_alter_archiveresult_id.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:20
|
||||||
|
|
||||||
|
import core.models
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='id',
|
||||||
|
field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:34
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0032_alter_archiveresult_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
old_name='id',
|
||||||
|
new_name='old_id',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:37
|
||||||
|
|
||||||
|
import core.models
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
from abid_utils.abid import ABID
|
||||||
|
|
||||||
|
|
||||||
|
def update_archiveresult_ids(apps, schema_editor):
|
||||||
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
||||||
|
num_total = ArchiveResult.objects.all().count()
|
||||||
|
print(f' Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
|
||||||
|
for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()):
|
||||||
|
assert result.abid
|
||||||
|
result.uuid = ABID.parse(result.abid).uuid
|
||||||
|
result.save(update_fields=["uuid"])
|
||||||
|
assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
|
||||||
|
if idx % 2500 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0033_rename_id_archiveresult_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='old_id',
|
||||||
|
field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
|
||||||
|
),
|
||||||
|
migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='uuid',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:49
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0034_alter_archiveresult_old_id_alter_archiveresult_uuid'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
old_name='uuid',
|
||||||
|
new_name='id',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 05:59
|
||||||
|
|
||||||
|
import core.models
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0035_remove_archiveresult_uuid_archiveresult_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='old_id',
|
||||||
|
field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0037_rename_id_snapshot_old_id.py
Normal file
18
archivebox/core/migrations/0037_rename_id_snapshot_old_id.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:08
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0036_alter_archiveresult_id_alter_archiveresult_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='snapshot',
|
||||||
|
old_name='id',
|
||||||
|
new_name='old_id',
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0038_rename_uuid_snapshot_id.py
Normal file
18
archivebox/core/migrations/0038_rename_uuid_snapshot_id.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:09
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0037_rename_id_snapshot_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='snapshot',
|
||||||
|
old_name='uuid',
|
||||||
|
new_name='id',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:25
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0038_rename_uuid_snapshot_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
old_name='snapshot',
|
||||||
|
new_name='snapshot_old',
|
||||||
|
),
|
||||||
|
]
|
||||||
34
archivebox/core/migrations/0040_archiveresult_snapshot.py
Normal file
34
archivebox/core/migrations/0040_archiveresult_snapshot.py
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:46
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
def update_archiveresult_snapshot_ids(apps, schema_editor):
|
||||||
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
||||||
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
|
num_total = ArchiveResult.objects.all().count()
|
||||||
|
print(f' Updating {num_total} ArchiveResult.snapshot_id values in place... (may take an hour or longer for large collections...)')
|
||||||
|
for idx, result in enumerate(ArchiveResult.objects.all().only('snapshot_old_id').iterator(chunk_size=5000)):
|
||||||
|
assert result.snapshot_old_id
|
||||||
|
snapshot = Snapshot.objects.only('id').get(old_id=result.snapshot_old_id)
|
||||||
|
result.snapshot_id = snapshot.id
|
||||||
|
result.save(update_fields=["snapshot_id"])
|
||||||
|
assert str(result.snapshot_id) == str(snapshot.id)
|
||||||
|
if idx % 5000 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0039_rename_snapshot_archiveresult_snapshot_old'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults', to='core.snapshot', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.RunPython(update_archiveresult_snapshot_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:50
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0040_archiveresult_snapshot'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot_old',
|
||||||
|
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults_old', to='core.snapshot'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:51
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0041_alter_archiveresult_snapshot_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot_old',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-18 06:52
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0042_remove_archiveresult_snapshot_old'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-19 23:01
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.SeparateDatabaseAndState(
|
||||||
|
database_operations=[
|
||||||
|
# No-op, SnapshotTag model already exists in DB
|
||||||
|
],
|
||||||
|
state_operations=[
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='SnapshotTag',
|
||||||
|
fields=[
|
||||||
|
('id', models.AutoField(primary_key=True, serialize=False)),
|
||||||
|
('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot')),
|
||||||
|
('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.tag')),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'db_table': 'core_snapshot_tags',
|
||||||
|
'unique_together': {('snapshot', 'tag')},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='tags',
|
||||||
|
field=models.ManyToManyField(blank=True, related_name='snapshot_set', through='core.SnapshotTag', to='core.tag'),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0045_alter_snapshot_old_id.py
Normal file
19
archivebox/core/migrations/0045_alter_snapshot_old_id.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 01:54
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='old_id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 01:55
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0045_alter_snapshot_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='old_id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:16
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='tag',
|
||||||
|
field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:17
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0047_alter_snapshottag_unique_together_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='archiveresult',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:26
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0048_alter_archiveresult_snapshot_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
old_name='snapshot',
|
||||||
|
new_name='snapshot_old',
|
||||||
|
),
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together={('snapshot_old', 'tag')},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:30
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0049_rename_snapshot_snapshottag_snapshot_old_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot_old',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:31
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
def update_snapshottag_ids(apps, schema_editor):
|
||||||
|
Snapshot = apps.get_model("core", "Snapshot")
|
||||||
|
SnapshotTag = apps.get_model("core", "SnapshotTag")
|
||||||
|
num_total = SnapshotTag.objects.all().count()
|
||||||
|
print(f' Updating {num_total} SnapshotTag.snapshot_id values in place... (may take an hour or longer for large collections...)')
|
||||||
|
for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator()):
|
||||||
|
assert snapshottag.snapshot_old_id
|
||||||
|
snapshot = Snapshot.objects.get(old_id=snapshottag.snapshot_old_id)
|
||||||
|
snapshottag.snapshot_id = snapshot.id
|
||||||
|
snapshottag.save(update_fields=["snapshot_id"])
|
||||||
|
assert str(snapshottag.snapshot_id) == str(snapshot.id)
|
||||||
|
if idx % 100 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} SnapshotTag objects...')
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0050_alter_snapshottag_snapshot_old'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(blank=True, db_column='snapshot_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot_old',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottag_old_set', to='core.snapshot', to_field='old_id'),
|
||||||
|
),
|
||||||
|
migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:37
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0051_snapshottag_snapshot_alter_snapshottag_snapshot_old'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together=set(),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
|
||||||
|
),
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together={('snapshot', 'tag')},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:38
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0052_alter_snapshottag_unique_together_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='snapshot_old',
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0054_alter_snapshot_timestamp.py
Normal file
18
archivebox/core/migrations/0054_alter_snapshot_timestamp.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 02:40
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0053_remove_snapshottag_snapshot_old'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshot',
|
||||||
|
name='timestamp',
|
||||||
|
field=models.CharField(db_index=True, editable=False, max_length=32, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0055_alter_tag_slug.py
Normal file
18
archivebox/core/migrations/0055_alter_tag_slug.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:24
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0054_alter_snapshot_timestamp'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='slug',
|
||||||
|
field=models.SlugField(editable=False, max_length=100, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
17
archivebox/core/migrations/0056_remove_tag_uuid.py
Normal file
17
archivebox/core/migrations/0056_remove_tag_uuid.py
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:25
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0055_alter_tag_slug'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name='tag',
|
||||||
|
name='uuid',
|
||||||
|
),
|
||||||
|
]
|
||||||
18
archivebox/core/migrations/0057_rename_id_tag_old_id.py
Normal file
18
archivebox/core/migrations/0057_rename_id_tag_old_id.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:29
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0056_remove_tag_uuid'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='tag',
|
||||||
|
old_name='id',
|
||||||
|
new_name='old_id',
|
||||||
|
),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0058_alter_tag_old_id.py
Normal file
19
archivebox/core/migrations/0058_alter_tag_old_id.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:30
|
||||||
|
|
||||||
|
import core.models
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0057_rename_id_tag_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='old_id',
|
||||||
|
field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
|
||||||
|
),
|
||||||
|
]
|
||||||
81
archivebox/core/migrations/0059_tag_id.py
Normal file
81
archivebox/core/migrations/0059_tag_id.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:33
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
from abid_utils.models import ABID, abid_from_values
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_abid(self):
|
||||||
|
"""
|
||||||
|
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||||
|
"""
|
||||||
|
prefix = self.abid_prefix
|
||||||
|
ts = eval(self.abid_ts_src)
|
||||||
|
uri = eval(self.abid_uri_src)
|
||||||
|
subtype = eval(self.abid_subtype_src)
|
||||||
|
rand = eval(self.abid_rand_src)
|
||||||
|
|
||||||
|
if (not prefix) or prefix == 'obj_':
|
||||||
|
suggested_abid = self.__class__.__name__[:3].lower()
|
||||||
|
raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
|
||||||
|
|
||||||
|
if not ts:
|
||||||
|
ts = datetime.utcfromtimestamp(0)
|
||||||
|
print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
|
||||||
|
|
||||||
|
if not uri:
|
||||||
|
uri = str(self)
|
||||||
|
print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
|
||||||
|
|
||||||
|
if not subtype:
|
||||||
|
subtype = self.__class__.__name__
|
||||||
|
print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
|
||||||
|
|
||||||
|
if not rand:
|
||||||
|
rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
|
||||||
|
print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
|
||||||
|
|
||||||
|
abid = abid_from_values(
|
||||||
|
prefix=prefix,
|
||||||
|
ts=ts,
|
||||||
|
uri=uri,
|
||||||
|
subtype=subtype,
|
||||||
|
rand=rand,
|
||||||
|
)
|
||||||
|
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
||||||
|
return abid
|
||||||
|
|
||||||
|
|
||||||
|
def update_archiveresult_ids(apps, schema_editor):
|
||||||
|
Tag = apps.get_model("core", "Tag")
|
||||||
|
num_total = Tag.objects.all().count()
|
||||||
|
print(f' Updating {num_total} Tag.id, ArchiveResult.uuid values in place...')
|
||||||
|
for idx, tag in enumerate(Tag.objects.all().iterator()):
|
||||||
|
assert tag.name
|
||||||
|
tag.abid_prefix = 'tag_'
|
||||||
|
tag.abid_ts_src = 'self.created'
|
||||||
|
tag.abid_uri_src = 'self.slug'
|
||||||
|
tag.abid_subtype_src = '"03"'
|
||||||
|
tag.abid_rand_src = 'self.old_id'
|
||||||
|
tag.abid = calculate_abid(tag)
|
||||||
|
tag.id = tag.abid.uuid
|
||||||
|
tag.save(update_fields=["abid", "id"])
|
||||||
|
assert str(ABID.parse(tag.abid).uuid) == str(tag.id)
|
||||||
|
if idx % 10 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} Tag objects...')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0058_alter_tag_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='tag',
|
||||||
|
name='id',
|
||||||
|
field=models.UUIDField(blank=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0060_alter_tag_id.py
Normal file
19
archivebox/core/migrations/0060_alter_tag_id.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:42
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0059_tag_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:43
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0060_alter_tag_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
old_name='tag',
|
||||||
|
new_name='old_tag',
|
||||||
|
),
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together={('snapshot', 'old_tag')},
|
||||||
|
),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0062_alter_snapshottag_old_tag.py
Normal file
19
archivebox/core/migrations/0062_alter_snapshottag_old_tag.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:44
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0061_rename_tag_snapshottag_old_tag_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='old_tag',
|
||||||
|
field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:45
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
def update_snapshottag_ids(apps, schema_editor):
|
||||||
|
Tag = apps.get_model("core", "Tag")
|
||||||
|
SnapshotTag = apps.get_model("core", "SnapshotTag")
|
||||||
|
num_total = SnapshotTag.objects.all().count()
|
||||||
|
print(f' Updating {num_total} SnapshotTag.tag_id values in place... (may take an hour or longer for large collections...)')
|
||||||
|
for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator()):
|
||||||
|
assert snapshottag.old_tag_id
|
||||||
|
tag = Tag.objects.get(old_id=snapshottag.old_tag_id)
|
||||||
|
snapshottag.tag_id = tag.id
|
||||||
|
snapshottag.save(update_fields=["tag_id"])
|
||||||
|
assert str(snapshottag.tag_id) == str(tag.id)
|
||||||
|
if idx % 100 == 0:
|
||||||
|
print(f'Migrated {idx}/{num_total} SnapshotTag objects...')
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0062_alter_snapshottag_old_tag'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='tag',
|
||||||
|
field=models.ForeignKey(blank=True, db_column='tag_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='old_tag',
|
||||||
|
field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottags_old', to='core.tag'),
|
||||||
|
),
|
||||||
|
migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:50
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0063_snapshottag_tag_alter_snapshottag_old_tag'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together=set(),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='tag',
|
||||||
|
field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterUniqueTogether(
|
||||||
|
name='snapshottag',
|
||||||
|
unique_together={('snapshot', 'tag')},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:51
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0064_alter_snapshottag_unique_together_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='old_tag',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:52
|
||||||
|
|
||||||
|
import core.models
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0065_remove_snapshottag_old_tag'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='tag',
|
||||||
|
field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='id',
|
||||||
|
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='tag',
|
||||||
|
name='old_id',
|
||||||
|
field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
|
||||||
|
),
|
||||||
|
]
|
||||||
19
archivebox/core/migrations/0067_alter_snapshottag_tag.py
Normal file
19
archivebox/core/migrations/0067_alter_snapshottag_tag.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 03:53
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='snapshottag',
|
||||||
|
name='tag',
|
||||||
|
field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by Django 5.0.6 on 2024-08-20 07:26
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0067_alter_snapshottag_tag'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterModelOptions(
|
||||||
|
name='archiveresult',
|
||||||
|
options={'verbose_name': 'Archive Result', 'verbose_name_plural': 'Archive Results Log'},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -5,6 +5,7 @@ from typing import Optional, List, Dict
|
||||||
from django_stubs_ext.db.models import TypedModelMeta
|
from django_stubs_ext.db.models import TypedModelMeta
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import random
|
||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
@ -14,9 +15,8 @@ from django.db import models
|
||||||
from django.utils.functional import cached_property
|
from django.utils.functional import cached_property
|
||||||
from django.utils.text import slugify
|
from django.utils.text import slugify
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
from django.urls import reverse
|
from django.urls import reverse, reverse_lazy
|
||||||
from django.db.models import Case, When, Value, IntegerField
|
from django.db.models import Case, When, Value, IntegerField
|
||||||
from django.contrib.auth.models import User # noqa
|
|
||||||
|
|
||||||
from abid_utils.models import ABIDModel, ABIDField
|
from abid_utils.models import ABIDModel, ABIDField
|
||||||
|
|
||||||
|
|
@ -35,6 +35,8 @@ STATUS_CHOICES = [
|
||||||
("skipped", "skipped")
|
("skipped", "skipped")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def rand_int_id():
|
||||||
|
return random.getrandbits(32)
|
||||||
|
|
||||||
|
|
||||||
# class BaseModel(models.Model):
|
# class BaseModel(models.Model):
|
||||||
|
|
@ -48,24 +50,26 @@ STATUS_CHOICES = [
|
||||||
# abstract = True
|
# abstract = True
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class Tag(ABIDModel):
|
class Tag(ABIDModel):
|
||||||
"""
|
"""
|
||||||
Based on django-taggit model + ABID base.
|
Based on django-taggit model + ABID base.
|
||||||
"""
|
"""
|
||||||
abid_prefix = 'tag_'
|
abid_prefix = 'tag_'
|
||||||
abid_ts_src = 'self.created' # TODO: add created/modified time
|
abid_ts_src = 'self.created' # TODO: add created/modified time
|
||||||
abid_uri_src = 'self.name'
|
abid_uri_src = 'self.slug'
|
||||||
abid_subtype_src = '"03"'
|
abid_subtype_src = '"03"'
|
||||||
abid_rand_src = 'self.id'
|
abid_rand_src = 'self.old_id'
|
||||||
|
|
||||||
# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
|
old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID') # legacy PK
|
||||||
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
|
|
||||||
uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
|
||||||
|
|
||||||
name = models.CharField(unique=True, blank=False, max_length=100)
|
name = models.CharField(unique=True, blank=False, max_length=100)
|
||||||
slug = models.SlugField(unique=True, blank=True, max_length=100)
|
slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
|
||||||
# slug is autoset on save from name, never set it manually
|
# slug is autoset on save from name, never set it manually
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -76,6 +80,10 @@ class Tag(ABIDModel):
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.name
|
return self.name
|
||||||
|
|
||||||
|
# @property
|
||||||
|
# def old_id(self):
|
||||||
|
# return self.id
|
||||||
|
|
||||||
def slugify(self, tag, i=None):
|
def slugify(self, tag, i=None):
|
||||||
slug = slugify(tag)
|
slug = slugify(tag)
|
||||||
if i is not None:
|
if i is not None:
|
||||||
|
|
@ -104,37 +112,66 @@ class Tag(ABIDModel):
|
||||||
else:
|
else:
|
||||||
return super().save(*args, **kwargs)
|
return super().save(*args, **kwargs)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_url(self) -> str:
|
||||||
|
# /api/v1/core/snapshot/{uulid}
|
||||||
|
return reverse_lazy('api-1:get_tag', args=[self.abid])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_docs_url(self) -> str:
|
||||||
|
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
|
||||||
|
|
||||||
|
class SnapshotTag(models.Model):
|
||||||
|
id = models.AutoField(primary_key=True)
|
||||||
|
|
||||||
|
snapshot = models.ForeignKey('Snapshot', db_column='snapshot_id', on_delete=models.CASCADE, to_field='id')
|
||||||
|
tag = models.ForeignKey(Tag, db_column='tag_id', on_delete=models.CASCADE, to_field='id')
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
db_table = 'core_snapshot_tags'
|
||||||
|
unique_together = [('snapshot', 'tag')]
|
||||||
|
|
||||||
class Snapshot(ABIDModel):
|
class Snapshot(ABIDModel):
|
||||||
abid_prefix = 'snp_'
|
abid_prefix = 'snp_'
|
||||||
abid_ts_src = 'self.added'
|
abid_ts_src = 'self.added'
|
||||||
abid_uri_src = 'self.url'
|
abid_uri_src = 'self.url'
|
||||||
abid_subtype_src = '"01"'
|
abid_subtype_src = '"01"'
|
||||||
abid_rand_src = 'self.id'
|
abid_rand_src = 'self.old_id'
|
||||||
|
|
||||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) # legacy pk
|
old_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) # legacy pk
|
||||||
uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True)
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
|
||||||
url = models.URLField(unique=True, db_index=True)
|
url = models.URLField(unique=True, db_index=True)
|
||||||
timestamp = models.CharField(max_length=32, unique=True, db_index=True)
|
timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
|
||||||
|
|
||||||
title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
|
title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
|
||||||
|
|
||||||
|
tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
|
||||||
|
|
||||||
added = models.DateTimeField(auto_now_add=True, db_index=True)
|
added = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||||
updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
|
updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
|
||||||
tags = models.ManyToManyField(Tag, blank=True)
|
|
||||||
|
|
||||||
keys = ('url', 'timestamp', 'title', 'tags', 'updated')
|
keys = ('url', 'timestamp', 'title', 'tags', 'updated')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def uuid(self):
|
||||||
|
return self.id
|
||||||
|
|
||||||
def __repr__(self) -> str:
|
def __repr__(self) -> str:
|
||||||
title = self.title or '-'
|
title = (self.title_stripped or '-')[:64]
|
||||||
return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
|
return f'[{self.timestamp}] {self.url[:64]} ({title})'
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
title = self.title or '-'
|
title = (self.title_stripped or '-')[:64]
|
||||||
return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
|
return f'[{self.timestamp}] {self.url[:64]} ({title})'
|
||||||
|
|
||||||
|
def save(self, *args, **kwargs):
|
||||||
|
super().save(*args, **kwargs)
|
||||||
|
try:
|
||||||
|
assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'Snapshot.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
|
||||||
|
except AssertionError as e:
|
||||||
|
print(e)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_json(cls, info: dict):
|
def from_json(cls, info: dict):
|
||||||
|
|
@ -168,6 +205,19 @@ class Snapshot(ABIDModel):
|
||||||
def icons(self) -> str:
|
def icons(self) -> str:
|
||||||
return snapshot_icons(self)
|
return snapshot_icons(self)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_url(self) -> str:
|
||||||
|
# /api/v1/core/snapshot/{uulid}
|
||||||
|
return reverse_lazy('api-1:get_snapshot', args=[self.abid])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_docs_url(self) -> str:
|
||||||
|
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def title_stripped(self) -> str:
|
||||||
|
return (self.title or '').replace("\n", " ").replace("\r", "")
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def extension(self) -> str:
|
def extension(self) -> str:
|
||||||
from ..util import extension
|
from ..util import extension
|
||||||
|
|
@ -317,21 +367,21 @@ class ArchiveResultManager(models.Manager):
|
||||||
qs = qs.annotate(indexing_precedence=Case(*precedence, default=Value(1000),output_field=IntegerField())).order_by('indexing_precedence')
|
qs = qs.annotate(indexing_precedence=Case(*precedence, default=Value(1000),output_field=IntegerField())).order_by('indexing_precedence')
|
||||||
return qs
|
return qs
|
||||||
|
|
||||||
|
|
||||||
class ArchiveResult(ABIDModel):
|
class ArchiveResult(ABIDModel):
|
||||||
abid_prefix = 'res_'
|
abid_prefix = 'res_'
|
||||||
abid_ts_src = 'self.snapshot.added'
|
abid_ts_src = 'self.snapshot.added'
|
||||||
abid_uri_src = 'self.snapshot.url'
|
abid_uri_src = 'self.snapshot.url'
|
||||||
abid_subtype_src = 'self.extractor'
|
abid_subtype_src = 'self.extractor'
|
||||||
abid_rand_src = 'self.uuid'
|
abid_rand_src = 'self.id'
|
||||||
EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
|
EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
|
||||||
|
|
||||||
# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
|
||||||
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID') # legacy pk
|
|
||||||
uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
|
||||||
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
|
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
|
||||||
|
|
||||||
extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
|
extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
|
||||||
cmd = models.JSONField()
|
cmd = models.JSONField()
|
||||||
pwd = models.CharField(max_length=256)
|
pwd = models.CharField(max_length=256)
|
||||||
|
|
@ -344,15 +394,36 @@ class ArchiveResult(ABIDModel):
|
||||||
objects = ArchiveResultManager()
|
objects = ArchiveResultManager()
|
||||||
|
|
||||||
class Meta(TypedModelMeta):
|
class Meta(TypedModelMeta):
|
||||||
verbose_name = 'Result'
|
verbose_name = 'Archive Result'
|
||||||
|
verbose_name_plural = 'Archive Results Log'
|
||||||
|
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.extractor
|
return self.extractor
|
||||||
|
|
||||||
|
def save(self, *args, **kwargs):
|
||||||
|
super().save(*args, **kwargs)
|
||||||
|
try:
|
||||||
|
assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'ArchiveResult.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
|
||||||
|
except AssertionError as e:
|
||||||
|
print(e)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def uuid(self):
|
||||||
|
return self.id
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def snapshot_dir(self):
|
def snapshot_dir(self):
|
||||||
return Path(self.snapshot.link_dir)
|
return Path(self.snapshot.link_dir)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_url(self) -> str:
|
||||||
|
# /api/v1/core/archiveresult/{uulid}
|
||||||
|
return reverse_lazy('api-1:get_archiveresult', args=[self.abid])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_docs_url(self) -> str:
|
||||||
|
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def extractor_module(self):
|
def extractor_module(self):
|
||||||
|
|
|
||||||
|
|
@ -120,6 +120,8 @@ MIDDLEWARE = [
|
||||||
### Authentication Settings
|
### Authentication Settings
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
|
# AUTH_USER_MODEL = 'auth.User' # cannot be easily changed unfortunately
|
||||||
|
|
||||||
AUTHENTICATION_BACKENDS = [
|
AUTHENTICATION_BACKENDS = [
|
||||||
'django.contrib.auth.backends.RemoteUserBackend',
|
'django.contrib.auth.backends.RemoteUserBackend',
|
||||||
'django.contrib.auth.backends.ModelBackend',
|
'django.contrib.auth.backends.ModelBackend',
|
||||||
|
|
@ -463,6 +465,7 @@ SIGNAL_WEBHOOKS = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
|
||||||
|
|
||||||
ADMIN_DATA_VIEWS = {
|
ADMIN_DATA_VIEWS = {
|
||||||
"NAME": "Environment",
|
"NAME": "Environment",
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ urlpatterns = [
|
||||||
path('accounts/', include('django.contrib.auth.urls')),
|
path('accounts/', include('django.contrib.auth.urls')),
|
||||||
path('admin/', archivebox_admin.urls),
|
path('admin/', archivebox_admin.urls),
|
||||||
|
|
||||||
path("api/", include('api.urls')),
|
path("api/", include('api.urls'), name='api'),
|
||||||
|
|
||||||
path('health/', HealthCheckView.as_view(), name='healthcheck'),
|
path('health/', HealthCheckView.as_view(), name='healthcheck'),
|
||||||
path('error/', lambda *_: 1/0),
|
path('error/', lambda *_: 1/0),
|
||||||
|
|
|
||||||
|
|
@ -90,7 +90,7 @@ class SnapshotView(View):
|
||||||
archiveresults[result.extractor] = result_info
|
archiveresults[result.extractor] = result_info
|
||||||
|
|
||||||
existing_files = {result['path'] for result in archiveresults.values()}
|
existing_files = {result['path'] for result in archiveresults.values()}
|
||||||
min_size_threshold = 128 # bytes
|
min_size_threshold = 10_000 # bytes
|
||||||
allowed_extensions = {
|
allowed_extensions = {
|
||||||
'txt',
|
'txt',
|
||||||
'html',
|
'html',
|
||||||
|
|
@ -104,16 +104,19 @@ class SnapshotView(View):
|
||||||
'webm',
|
'webm',
|
||||||
'mp4',
|
'mp4',
|
||||||
'mp3',
|
'mp3',
|
||||||
|
'opus',
|
||||||
'pdf',
|
'pdf',
|
||||||
'md',
|
'md',
|
||||||
}
|
}
|
||||||
|
|
||||||
# iterate through all the files in the snapshot dir and add the biggest ones to the result list
|
|
||||||
for result_file in Path(snapshot.link_dir).glob('*/*/*'):
|
# iterate through all the files in the snapshot dir and add the biggest ones to1 the result list
|
||||||
|
snap_dir = Path(snapshot.link_dir)
|
||||||
|
for result_file in (*snap_dir.glob('*'), *snap_dir.glob('*/*')):
|
||||||
extension = result_file.suffix.lstrip('.').lower()
|
extension = result_file.suffix.lstrip('.').lower()
|
||||||
if result_file.is_dir() or result_file.name.startswith('.') or extension not in allowed_extensions:
|
if result_file.is_dir() or result_file.name.startswith('.') or extension not in allowed_extensions:
|
||||||
continue
|
continue
|
||||||
if result_file.name in existing_files:
|
if result_file.name in existing_files or result_file.name == 'index.html':
|
||||||
continue
|
continue
|
||||||
|
|
||||||
file_size = result_file.stat().st_size or 0
|
file_size = result_file.stat().st_size or 0
|
||||||
|
|
@ -121,12 +124,12 @@ class SnapshotView(View):
|
||||||
if file_size > min_size_threshold:
|
if file_size > min_size_threshold:
|
||||||
archiveresults[result_file.name] = {
|
archiveresults[result_file.name] = {
|
||||||
'name': result_file.stem,
|
'name': result_file.stem,
|
||||||
'path': result_file.relative_to(snapshot.link_dir),
|
'path': result_file.relative_to(snap_dir),
|
||||||
'ts': ts_to_date_str(result_file.stat().st_mtime or 0),
|
'ts': ts_to_date_str(result_file.stat().st_mtime or 0),
|
||||||
'size': file_size,
|
'size': file_size,
|
||||||
}
|
}
|
||||||
|
|
||||||
preferred_types = ('singlefile', 'wget', 'screenshot', 'dom', 'media', 'pdf', 'readability', 'mercury')
|
preferred_types = ('singlefile', 'screenshot', 'wget', 'dom', 'media', 'pdf', 'readability', 'mercury')
|
||||||
all_types = preferred_types + tuple(result_type for result_type in archiveresults.keys() if result_type not in preferred_types)
|
all_types = preferred_types + tuple(result_type for result_type in archiveresults.keys() if result_type not in preferred_types)
|
||||||
|
|
||||||
best_result = {'path': 'None'}
|
best_result = {'path': 'None'}
|
||||||
|
|
@ -140,7 +143,7 @@ class SnapshotView(View):
|
||||||
link_info = link._asdict(extended=True)
|
link_info = link._asdict(extended=True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
warc_path = 'warc/' + list(Path(snapshot.link_dir).glob('warc/*.warc.*'))[0].name
|
warc_path = 'warc/' + list(Path(snap_dir).glob('warc/*.warc.*'))[0].name
|
||||||
except IndexError:
|
except IndexError:
|
||||||
warc_path = 'warc/'
|
warc_path = 'warc/'
|
||||||
|
|
||||||
|
|
@ -160,7 +163,7 @@ class SnapshotView(View):
|
||||||
'warc_path': warc_path,
|
'warc_path': warc_path,
|
||||||
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
|
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
|
||||||
'PREVIEW_ORIGINALS': PREVIEW_ORIGINALS,
|
'PREVIEW_ORIGINALS': PREVIEW_ORIGINALS,
|
||||||
'archiveresults': sorted(archiveresults.values(), key=lambda r: all_types.index(r['name'])),
|
'archiveresults': sorted(archiveresults.values(), key=lambda r: all_types.index(r['name']) if r['name'] in all_types else -r['size']),
|
||||||
'best_result': best_result,
|
'best_result': best_result,
|
||||||
# 'tags_str': 'somealskejrewlkrjwer,werlmwrwlekrjewlkrjwer324m532l,4m32,23m324234',
|
# 'tags_str': 'somealskejrewlkrjwer,werlmwrwlekrjewlkrjwer324m532l,4m32,23m324234',
|
||||||
}
|
}
|
||||||
|
|
@ -178,6 +181,7 @@ class SnapshotView(View):
|
||||||
except (IndexError, ValueError):
|
except (IndexError, ValueError):
|
||||||
slug, archivefile = path.split('/', 1)[0], 'index.html'
|
slug, archivefile = path.split('/', 1)[0], 'index.html'
|
||||||
|
|
||||||
|
|
||||||
# slug is a timestamp
|
# slug is a timestamp
|
||||||
if slug.replace('.','').isdigit():
|
if slug.replace('.','').isdigit():
|
||||||
|
|
||||||
|
|
@ -224,7 +228,7 @@ class SnapshotView(View):
|
||||||
snap.timestamp,
|
snap.timestamp,
|
||||||
snap.timestamp,
|
snap.timestamp,
|
||||||
snap.url,
|
snap.url,
|
||||||
snap.title or '',
|
snap.title_stripped[:64] or '',
|
||||||
)
|
)
|
||||||
for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
|
for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
|
||||||
)
|
)
|
||||||
|
|
@ -275,12 +279,35 @@ class SnapshotView(View):
|
||||||
content_type="text/html",
|
content_type="text/html",
|
||||||
status=404,
|
status=404,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# # slud is an ID
|
||||||
|
# ulid = slug.split('_', 1)[-1]
|
||||||
|
# try:
|
||||||
|
# try:
|
||||||
|
# snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid) | Q(old_id=ulid))
|
||||||
|
# except Snapshot.DoesNotExist:
|
||||||
|
# pass
|
||||||
|
|
||||||
|
# try:
|
||||||
|
# snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug) | Q(old_id__startswith=slug))
|
||||||
|
# except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned):
|
||||||
|
# pass
|
||||||
|
|
||||||
|
# try:
|
||||||
|
# snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
|
||||||
|
# except Snapshot.DoesNotExist:
|
||||||
|
# pass
|
||||||
|
# return redirect(f'/archive/{snapshot.timestamp}/index.html')
|
||||||
|
# except Snapshot.DoesNotExist:
|
||||||
|
# pass
|
||||||
|
|
||||||
# slug is a URL
|
# slug is a URL
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
# try exact match on full url first
|
# try exact match on full url / ABID first
|
||||||
snapshot = Snapshot.objects.get(
|
snapshot = Snapshot.objects.get(
|
||||||
Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
|
Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
|
||||||
|
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
|
||||||
)
|
)
|
||||||
except Snapshot.DoesNotExist:
|
except Snapshot.DoesNotExist:
|
||||||
# fall back to match on exact base_url
|
# fall back to match on exact base_url
|
||||||
|
|
@ -314,15 +341,17 @@ class SnapshotView(View):
|
||||||
except Snapshot.MultipleObjectsReturned:
|
except Snapshot.MultipleObjectsReturned:
|
||||||
snapshot_hrefs = mark_safe('<br/>').join(
|
snapshot_hrefs = mark_safe('<br/>').join(
|
||||||
format_html(
|
format_html(
|
||||||
'{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
|
'{} <code style="font-size: 0.8em">{}</code> <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
|
||||||
snap.added.strftime('%Y-%m-%d %H:%M:%S'),
|
snap.added.strftime('%Y-%m-%d %H:%M:%S'),
|
||||||
|
snap.abid,
|
||||||
snap.timestamp,
|
snap.timestamp,
|
||||||
snap.timestamp,
|
snap.timestamp,
|
||||||
snap.url,
|
snap.url,
|
||||||
snap.title or '',
|
snap.title_stripped[:64] or '',
|
||||||
)
|
)
|
||||||
for snap in Snapshot.objects.filter(
|
for snap in Snapshot.objects.filter(
|
||||||
Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
|
Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
|
||||||
|
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
|
||||||
).only('url', 'timestamp', 'title', 'added').order_by('-added')
|
).only('url', 'timestamp', 'title', 'added').order_by('-added')
|
||||||
)
|
)
|
||||||
return HttpResponse(
|
return HttpResponse(
|
||||||
|
|
|
||||||
|
|
@ -266,7 +266,7 @@ class Link:
|
||||||
@cached_property
|
@cached_property
|
||||||
def snapshot(self):
|
def snapshot(self):
|
||||||
from core.models import Snapshot
|
from core.models import Snapshot
|
||||||
return Snapshot.objects.only('uuid').get(url=self.url)
|
return Snapshot.objects.only('id').get(url=self.url)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def snapshot_id(self):
|
def snapshot_id(self):
|
||||||
|
|
@ -274,7 +274,7 @@ class Link:
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def snapshot_uuid(self):
|
def snapshot_uuid(self):
|
||||||
return str(self.snapshot.uuid)
|
return str(self.snapshot.id)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def snapshot_abid(self):
|
def snapshot_abid(self):
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ if __name__ == '__main__':
|
||||||
# versions of ./manage.py commands whenever possible. When that's not possible
|
# versions of ./manage.py commands whenever possible. When that's not possible
|
||||||
# (e.g. makemigrations), you can comment out this check temporarily
|
# (e.g. makemigrations), you can comment out this check temporarily
|
||||||
|
|
||||||
if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv):
|
if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv or 'squashmigrations' in sys.argv):
|
||||||
print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
|
print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
|
||||||
print()
|
print()
|
||||||
print(' Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')
|
print(' Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')
|
||||||
|
|
|
||||||
114
archivebox/package-lock.json
generated
114
archivebox/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@postlight/parser": "^2.2.3",
|
"@postlight/parser": "^2.2.3",
|
||||||
|
|
@ -26,9 +26,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@babel/runtime-corejs2": {
|
"node_modules/@babel/runtime-corejs2": {
|
||||||
"version": "7.24.6",
|
"version": "7.25.0",
|
||||||
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.24.6.tgz",
|
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.25.0.tgz",
|
||||||
"integrity": "sha512-5UK2PnfpmiCftYGBeJ+SpFIMNaoMPU/eQt1P5ISx0TB7nGGzEMLT4/3PapNZEfGZh+nGxGOGj2t59prGFBhunQ==",
|
"integrity": "sha512-aoYVE3tm+vgAoezmXFWmVcp+NlSdsUqQMPL7c6zRxq8KDHCf570pamC7005Q/UkSlTuoL6oeE16zIw/9J3YFyw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"core-js": "^2.6.12",
|
"core-js": "^2.6.12",
|
||||||
|
|
@ -180,9 +180,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@postman/tunnel-agent": {
|
"node_modules/@postman/tunnel-agent": {
|
||||||
"version": "0.6.3",
|
"version": "0.6.4",
|
||||||
"resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.3.tgz",
|
"resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.4.tgz",
|
||||||
"integrity": "sha512-k57fzmAZ2PJGxfOA4SGR05ejorHbVAa/84Hxh/2nAztjNXc4ZjOm9NUIk6/Z6LCrBvJZqjRZbN8e/nROVUPVdg==",
|
"integrity": "sha512-CJJlq8V7rNKhAw4sBfjixKpJW00SHqebqNUQKxMoepgeWZIbdPcD+rguRcivGhS4N12PymDcKgUgSD4rVC+RjQ==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"safe-buffer": "^5.0.1"
|
"safe-buffer": "^5.0.1"
|
||||||
|
|
@ -236,13 +236,13 @@
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "20.14.0",
|
"version": "22.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.4.1.tgz",
|
||||||
"integrity": "sha512-5cHBxFGJx6L4s56Bubp4fglrEpmyJypsqI6RgzMfBHWUJQGWAAi8cWcgetEbZXHYXo9C2Fa4EEds/uSyS4cxmA==",
|
"integrity": "sha512-1tbpb9325+gPnKK0dMm+/LMriX0vKxf6RnB0SZUqfyVkQ4fMgUSySqhxE/y8Jvs4NyF1yHzTfG9KlnkIODxPKg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~5.26.4"
|
"undici-types": "~6.19.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/yauzl": {
|
"node_modules/@types/yauzl": {
|
||||||
|
|
@ -353,9 +353,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/aws4": {
|
"node_modules/aws4": {
|
||||||
"version": "1.13.0",
|
"version": "1.13.1",
|
||||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.0.tgz",
|
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz",
|
||||||
"integrity": "sha512-3AungXC4I8kKsS9PuS4JH2nc+0bVY/mjgrephHTIi8fpEeGsTHBUJeosp0Wc1myYMElmD0B3Oc4XL/HVJ4PV2g==",
|
"integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/b4a": {
|
"node_modules/b4a": {
|
||||||
|
|
@ -365,9 +365,9 @@
|
||||||
"license": "Apache-2.0"
|
"license": "Apache-2.0"
|
||||||
},
|
},
|
||||||
"node_modules/bare-events": {
|
"node_modules/bare-events": {
|
||||||
"version": "2.3.1",
|
"version": "2.4.2",
|
||||||
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz",
|
||||||
"integrity": "sha512-sJnSOTVESURZ61XgEleqmP255T6zTYwHPwE4r6SssIh0U9/uDvfpdoJYpVUerJJZH2fueO+CdT8ZT+OC/7aZDA==",
|
"integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"optional": true
|
"optional": true
|
||||||
},
|
},
|
||||||
|
|
@ -700,9 +700,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/debug": {
|
"node_modules/debug": {
|
||||||
"version": "4.3.5",
|
"version": "4.3.6",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz",
|
||||||
"integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
|
"integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"ms": "2.1.2"
|
"ms": "2.1.2"
|
||||||
|
|
@ -793,9 +793,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/dompurify": {
|
"node_modules/dompurify": {
|
||||||
"version": "3.1.5",
|
"version": "3.1.6",
|
||||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.6.tgz",
|
||||||
"integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==",
|
"integrity": "sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ==",
|
||||||
"license": "(MPL-2.0 OR Apache-2.0)"
|
"license": "(MPL-2.0 OR Apache-2.0)"
|
||||||
},
|
},
|
||||||
"node_modules/domutils": {
|
"node_modules/domutils": {
|
||||||
|
|
@ -1174,9 +1174,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/https-proxy-agent": {
|
"node_modules/https-proxy-agent": {
|
||||||
"version": "7.0.4",
|
"version": "7.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz",
|
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz",
|
||||||
"integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==",
|
"integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"agent-base": "^7.0.2",
|
"agent-base": "^7.0.2",
|
||||||
|
|
@ -1629,9 +1629,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/nwsapi": {
|
"node_modules/nwsapi": {
|
||||||
"version": "2.2.10",
|
"version": "2.2.12",
|
||||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.10.tgz",
|
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz",
|
||||||
"integrity": "sha512-QK0sRs7MKv0tKe1+5uZIQk/C8XGza4DAnztJG8iD+TpJIORARrCxczA738awHrZoHeTjSSoHqao2teO0dC/gFQ==",
|
"integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/oauth-sign": {
|
"node_modules/oauth-sign": {
|
||||||
|
|
@ -1653,9 +1653,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/pac-proxy-agent": {
|
"node_modules/pac-proxy-agent": {
|
||||||
"version": "7.0.1",
|
"version": "7.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz",
|
||||||
"integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==",
|
"integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tootallnate/quickjs-emscripten": "^0.23.0",
|
"@tootallnate/quickjs-emscripten": "^0.23.0",
|
||||||
|
|
@ -1663,9 +1663,9 @@
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
"get-uri": "^6.0.1",
|
"get-uri": "^6.0.1",
|
||||||
"http-proxy-agent": "^7.0.0",
|
"http-proxy-agent": "^7.0.0",
|
||||||
"https-proxy-agent": "^7.0.2",
|
"https-proxy-agent": "^7.0.5",
|
||||||
"pac-resolver": "^7.0.0",
|
"pac-resolver": "^7.0.1",
|
||||||
"socks-proxy-agent": "^8.0.2"
|
"socks-proxy-agent": "^8.0.4"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 14"
|
"node": ">= 14"
|
||||||
|
|
@ -1727,14 +1727,14 @@
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/postman-request": {
|
"node_modules/postman-request": {
|
||||||
"version": "2.88.1-postman.33",
|
"version": "2.88.1-postman.39",
|
||||||
"resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.33.tgz",
|
"resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.39.tgz",
|
||||||
"integrity": "sha512-uL9sCML4gPH6Z4hreDWbeinKU0p0Ke261nU7OvII95NU22HN6Dk7T/SaVPaj6T4TsQqGKIFw6/woLZnH7ugFNA==",
|
"integrity": "sha512-rsncxxDlbn1YpygXSgJqbJzIjGlHFcZjbYDzeBPTQHMDfLuSTzZz735JHV8i1+lOROuJ7MjNap4eaSD3UijHzQ==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@postman/form-data": "~3.1.1",
|
"@postman/form-data": "~3.1.1",
|
||||||
"@postman/tough-cookie": "~4.1.3-postman.1",
|
"@postman/tough-cookie": "~4.1.3-postman.1",
|
||||||
"@postman/tunnel-agent": "^0.6.3",
|
"@postman/tunnel-agent": "^0.6.4",
|
||||||
"aws-sign2": "~0.7.0",
|
"aws-sign2": "~0.7.0",
|
||||||
"aws4": "^1.12.0",
|
"aws4": "^1.12.0",
|
||||||
"brotli": "^1.3.3",
|
"brotli": "^1.3.3",
|
||||||
|
|
@ -1756,7 +1756,7 @@
|
||||||
"uuid": "^8.3.2"
|
"uuid": "^8.3.2"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 6"
|
"node": ">= 16"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/process-nextick-args": {
|
"node_modules/process-nextick-args": {
|
||||||
|
|
@ -2148,14 +2148,14 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/socks-proxy-agent": {
|
"node_modules/socks-proxy-agent": {
|
||||||
"version": "8.0.3",
|
"version": "8.0.4",
|
||||||
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz",
|
||||||
"integrity": "sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A==",
|
"integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"agent-base": "^7.1.1",
|
"agent-base": "^7.1.1",
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
"socks": "^2.7.1"
|
"socks": "^2.8.3"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 14"
|
"node": ">= 14"
|
||||||
|
|
@ -2322,9 +2322,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/text-decoder": {
|
"node_modules/text-decoder": {
|
||||||
"version": "1.1.0",
|
"version": "1.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz",
|
||||||
"integrity": "sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==",
|
"integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"b4a": "^1.6.4"
|
"b4a": "^1.6.4"
|
||||||
|
|
@ -2376,9 +2376,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/tslib": {
|
"node_modules/tslib": {
|
||||||
"version": "2.6.2",
|
"version": "2.6.3",
|
||||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
|
||||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==",
|
"integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
|
||||||
"license": "0BSD"
|
"license": "0BSD"
|
||||||
},
|
},
|
||||||
"node_modules/turndown": {
|
"node_modules/turndown": {
|
||||||
|
|
@ -2407,9 +2407,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/undici-types": {
|
"node_modules/undici-types": {
|
||||||
"version": "5.26.5",
|
"version": "6.19.8",
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
|
||||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true
|
"optional": true
|
||||||
},
|
},
|
||||||
|
|
@ -2575,9 +2575,9 @@
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/ws": {
|
"node_modules/ws": {
|
||||||
"version": "8.17.0",
|
"version": "8.18.0",
|
||||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
|
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
|
||||||
"integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
|
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=10.0.0"
|
"node": ">=10.0.0"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"description": "ArchiveBox: The self-hosted internet archive",
|
"description": "ArchiveBox: The self-hosted internet archive",
|
||||||
"author": "Nick Sweeting <archivebox-npm@sweeting.me>",
|
"author": "Nick Sweeting <archivebox-npm@sweeting.me>",
|
||||||
"repository": "github:ArchiveBox/ArchiveBox",
|
"repository": "github:ArchiveBox/ArchiveBox",
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,13 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
|
<script
|
||||||
|
src="https://code.jquery.com/jquery-3.7.1.slim.min.js"
|
||||||
|
integrity="sha256-kmHvs0B+OpCW5GVHUNjv9rOmY0IvSIRcf7zGUDTDQM8="
|
||||||
|
crossorigin="anonymous"></script>
|
||||||
|
<link href="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/css/select2.min.css" rel="stylesheet" />
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/js/select2.min.js"></script>
|
||||||
|
|
||||||
<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
|
<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
|
|
@ -264,6 +271,11 @@
|
||||||
.appendTo(buttons)
|
.appendTo(buttons)
|
||||||
})
|
})
|
||||||
console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
|
console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
|
||||||
|
jQuery('select[multiple]').select2();
|
||||||
|
}
|
||||||
|
function fixInlineAddRow() {
|
||||||
|
$('#id_snapshottag-MAX_NUM_FORMS').val('1000')
|
||||||
|
$('.add-row').show()
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupSnapshotGridListToggle() {
|
function setupSnapshotGridListToggle() {
|
||||||
|
|
@ -281,7 +293,7 @@
|
||||||
// if we arrive at the index with a url like ??id__startswith=...
|
// if we arrive at the index with a url like ??id__startswith=...
|
||||||
// we were hotlinked here with the intention of making it easy for the user to perform some
|
// we were hotlinked here with the intention of making it easy for the user to perform some
|
||||||
// actions on the given snapshot. therefore we should preselect the snapshot to save them a click
|
// actions on the given snapshot. therefore we should preselect the snapshot to save them a click
|
||||||
if (window.location.search.startsWith('?id__startswith=') || window.location.search.startsWith('?id__exact=')) {
|
if (window.location.search.startsWith('?')) {
|
||||||
const result_checkboxes = [...document.querySelectorAll('#result_list .action-checkbox input[type=checkbox]')]
|
const result_checkboxes = [...document.querySelectorAll('#result_list .action-checkbox input[type=checkbox]')]
|
||||||
if (result_checkboxes.length === 1) {
|
if (result_checkboxes.length === 1) {
|
||||||
result_checkboxes[0].click()
|
result_checkboxes[0].click()
|
||||||
|
|
@ -290,6 +302,7 @@
|
||||||
}
|
}
|
||||||
$(document).ready(function() {
|
$(document).ready(function() {
|
||||||
fix_actions()
|
fix_actions()
|
||||||
|
fixInlineAddRow()
|
||||||
setupSnapshotGridListToggle()
|
setupSnapshotGridListToggle()
|
||||||
setTimeOffset()
|
setTimeOffset()
|
||||||
selectSnapshotIfHotlinked()
|
selectSnapshotIfHotlinked()
|
||||||
|
|
|
||||||
|
|
@ -351,7 +351,7 @@
|
||||||
<a href="warc/" title="Any WARC archives for the page">WARC</a> |
|
<a href="warc/" title="Any WARC archives for the page">WARC</a> |
|
||||||
<a href="media/" title="Audio, Video, and Subtitle files.">Media</a> |
|
<a href="media/" title="Audio, Video, and Subtitle files.">Media</a> |
|
||||||
<a href="git/" title="Any git repos at the url">Git</a> |
|
<a href="git/" title="Any git repos at the url">Git</a> |
|
||||||
<a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> |
|
<a href="/admin/core/snapshot/?q={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> |
|
||||||
<a href="/admin/core/snapshot/{{snapshot_id}}/change/" title="Edit this snapshot in the Admin UI">Admin</a> |
|
<a href="/admin/core/snapshot/{{snapshot_id}}/change/" title="Edit this snapshot in the Admin UI">Admin</a> |
|
||||||
<a href="." title="Webserver-provided index of files directory.">See all files...</a><br/>
|
<a href="." title="Webserver-provided index of files directory.">See all files...</a><br/>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -349,7 +349,7 @@
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="badge badge-{{status_color}}" style="float: left">
|
<div class="badge badge-{{status_color}}" style="float: left">
|
||||||
<a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
|
<a href="/admin/core/snapshot/?q={{snapshot_id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
|
||||||
{{status|upper}}
|
{{status|upper}}
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -385,9 +385,10 @@
|
||||||
<br/>
|
<br/>
|
||||||
<div class="external-links">
|
<div class="external-links">
|
||||||
↗️
|
↗️
|
||||||
<a href="https://web.archive.org/web/{{url}}" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer">Archive.org</a> |
|
<a href="./index.json" title="Get the Snapshot details as a JSON file" target="_blank">JSON</a> | 🗃️
|
||||||
<a href="https://archive.md/{{url}}" title="Search for a copy of the URL saved in Archive.today" target="_blank" rel="noreferrer">Archive.today</a> |
|
<a href="{{warc_path}}" title="Download the ArchiveBox-generated WARC file" target="_blank">WARC</a> |
|
||||||
<a href="{{warc_path}}" title="Download the ArchiveBox-generated WARC file" target="_blank">WARC</a>
|
<a href="https://web.archive.org/web/{{url}}" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer">🏛️ Archive.org</a>
|
||||||
|
<!--<a href="https://archive.md/{{url}}" title="Search for a copy of the URL saved in Archive.today" target="_blank" rel="noreferrer">Archive.today</a> | -->
|
||||||
<!--<a href="https://ghostarchive.org/search?term={{url|urlencode}}" title="Search for a copy of the URL saved in GhostArchive.org" target="_blank" rel="noreferrer">More...</a>-->
|
<!--<a href="https://ghostarchive.org/search?term={{url|urlencode}}" title="Search for a copy of the URL saved in GhostArchive.org" target="_blank" rel="noreferrer">More...</a>-->
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -401,13 +402,13 @@
|
||||||
<div class="col-lg-2">
|
<div class="col-lg-2">
|
||||||
<div class="card {% if forloop.first %}selected-card{% endif %}">
|
<div class="card {% if forloop.first %}selected-card{% endif %}">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<a href="{{result.path}}" target="preview" title="./{{result.path}} (downloaded {{result.ts}})">
|
<a href="{{result.path|urlencode}}" target="preview" title="./{{result.path}} (downloaded {{result.ts}})">
|
||||||
<h4>{{result.name}} <small>({{result.size|filesizeformat}})</small></h4>
|
<h4>{{result.name|truncatechars:24}} <small>({{result.size|filesizeformat}})</small></h4>
|
||||||
<!-- <p class="card-text" ><code>./{{result.path|truncatechars:30}}</code></p> -->
|
<!-- <p class="card-text" ><code>./{{result.path|truncatechars:30}}</code></p> -->
|
||||||
</a>
|
</a>
|
||||||
<!--<a href="{{result.path}}" target="preview"><h4 class="card-title">{{result.name}}</h4></a>-->
|
<!--<a href="{{result.path}}" target="preview"><h4 class="card-title">{{result.name}}</h4></a>-->
|
||||||
</div>
|
</div>
|
||||||
<iframe class="card-img-top" src="{{result.path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
|
<iframe class="card-img-top" src="{{result.path|urlencode}}?autoplay=0" allow="autoplay 'none'; fullscreen 'none'; navigation-override 'none'; " sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
@ -419,7 +420,7 @@
|
||||||
<a href="./" target="preview">
|
<a href="./" target="preview">
|
||||||
<h4>Headers, JSON, etc.</h4>
|
<h4>Headers, JSON, etc.</h4>
|
||||||
</a>
|
</a>
|
||||||
<!--<a href="{{result.path}}" target="preview"><h4 class="card-title">{{result.name}}</h4></a>-->
|
<!--<a href="{{result.path|urlencode}}" target="preview"><h4 class="card-title">{{result.name}}</h4></a>-->
|
||||||
</div>
|
</div>
|
||||||
<iframe class="card-img-top" src="./" sandbox="" scrolling="no" loading="lazy"></iframe>
|
<iframe class="card-img-top" src="./" sandbox="" scrolling="no" loading="lazy"></iframe>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -430,7 +431,7 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<iframe id="main-frame" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" class="full-page-iframe" src="{{best_result.path}}" name="preview"></iframe>
|
<iframe id="main-frame" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" class="full-page-iframe" src="{{best_result.path|urlencode}}" name="preview"></iframe>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -444,9 +445,9 @@
|
||||||
this.src = this.src + '#toolbar=0'
|
this.src = this.src + '#toolbar=0'
|
||||||
}
|
}
|
||||||
this.onload = function() {
|
this.onload = function() {
|
||||||
if (this.src.endsWith('.pdf')) {
|
if (this.src.includes('.pdf')) {
|
||||||
this.removeAttribute('sandbox')
|
this.removeAttribute('sandbox')
|
||||||
this.src = this.src + '#toolbar=0'
|
this.src = this.src.split('?autoplay=')[0] + '#toolbar=0'
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
// doesnt work if frame origin rules prevent accessing its DOM via JS
|
// doesnt work if frame origin rules prevent accessing its DOM via JS
|
||||||
|
|
|
||||||
|
|
@ -116,7 +116,6 @@ body.model-snapshot.change-list #content .object-tools {
|
||||||
margin-right: 0px;
|
margin-right: 0px;
|
||||||
width: auto;
|
width: auto;
|
||||||
max-height: 40px;
|
max-height: 40px;
|
||||||
overflow: hidden;
|
|
||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
@media (max-width: 1000px) {
|
@media (max-width: 1000px) {
|
||||||
|
|
@ -166,14 +165,14 @@ body.model-snapshot.change-list #content .object-tools {
|
||||||
margin-right: 25px;
|
margin-right: 25px;
|
||||||
}
|
}
|
||||||
|
|
||||||
#content #changelist .actions .select2-selection {
|
#content #changelist .actions > label {
|
||||||
max-height: 25px;
|
max-height: 25px;
|
||||||
}
|
}
|
||||||
#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
|
#content #changelist .actions > label {
|
||||||
width: auto !important;
|
width: auto !important;
|
||||||
min-width: 90px;
|
min-width: 90px;
|
||||||
}
|
}
|
||||||
#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
|
#content #changelist .actions > label > select {
|
||||||
margin-top: 3px;
|
margin-top: 3px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -84,8 +84,8 @@ docker buildx build --platform "$SELECTED_PLATFORMS" --load . \
|
||||||
-t archivebox/archivebox:$GIT_SHA \
|
-t archivebox/archivebox:$GIT_SHA \
|
||||||
-t nikisweeting/archivebox:$TAG_NAME \
|
-t nikisweeting/archivebox:$TAG_NAME \
|
||||||
-t nikisweeting/archivebox:$GIT_SHA \
|
-t nikisweeting/archivebox:$GIT_SHA \
|
||||||
-t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
|
-t ghcr.io/archivebox/archivebox:$TAG_NAME \
|
||||||
-t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA
|
-t ghcr.io/archivebox/archivebox:$GIT_SHA
|
||||||
# -t archivebox/archivebox \
|
# -t archivebox/archivebox \
|
||||||
# -t archivebox/archivebox:$VERSION \
|
# -t archivebox/archivebox:$VERSION \
|
||||||
# -t archivebox/archivebox:$SHORT_VERSION \
|
# -t archivebox/archivebox:$SHORT_VERSION \
|
||||||
|
|
@ -94,6 +94,6 @@ docker buildx build --platform "$SELECTED_PLATFORMS" --load . \
|
||||||
# -t nikisweeting/archivebox:$VERSION \
|
# -t nikisweeting/archivebox:$VERSION \
|
||||||
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
||||||
# -t nikisweeting/archivebox:latest \
|
# -t nikisweeting/archivebox:latest \
|
||||||
# -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
|
# -t ghcr.io/archivebox/archivebox:$VERSION \
|
||||||
# -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
|
# -t ghcr.io/archivebox/archivebox:$SHORT_VERSION \
|
||||||
# -t ghcr.io/archivebox/archivebox/archivebox:latest
|
# -t ghcr.io/archivebox/archivebox:latest
|
||||||
|
|
|
||||||
|
|
@ -35,8 +35,8 @@ docker buildx build --platform "$SELECTED_PLATFORMS" --push . \
|
||||||
-t archivebox/archivebox:$GIT_SHA \
|
-t archivebox/archivebox:$GIT_SHA \
|
||||||
-t nikisweeting/archivebox:$TAG_NAME \
|
-t nikisweeting/archivebox:$TAG_NAME \
|
||||||
-t nikisweeting/archivebox:$GIT_SHA \
|
-t nikisweeting/archivebox:$GIT_SHA \
|
||||||
-t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
|
-t ghcr.io/archivebox/archivebox:$TAG_NAME \
|
||||||
-t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA
|
-t ghcr.io/archivebox/archivebox:$GIT_SHA
|
||||||
# -t archivebox/archivebox \
|
# -t archivebox/archivebox \
|
||||||
# -t archivebox/archivebox:$VERSION \
|
# -t archivebox/archivebox:$VERSION \
|
||||||
# -t archivebox/archivebox:$SHORT_VERSION \
|
# -t archivebox/archivebox:$SHORT_VERSION \
|
||||||
|
|
@ -45,6 +45,6 @@ docker buildx build --platform "$SELECTED_PLATFORMS" --push . \
|
||||||
# -t nikisweeting/archivebox:$VERSION \
|
# -t nikisweeting/archivebox:$VERSION \
|
||||||
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
# -t nikisweeting/archivebox:$SHORT_VERSION \
|
||||||
# -t nikisweeting/archivebox:latest \
|
# -t nikisweeting/archivebox:latest \
|
||||||
# -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
|
# -t ghcr.io/archivebox/archivebox:$VERSION \
|
||||||
# -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
|
# -t ghcr.io/archivebox/archivebox:$SHORT_VERSION \
|
||||||
|
|
||||||
|
|
|
||||||
114
package-lock.json
generated
114
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@postlight/parser": "^2.2.3",
|
"@postlight/parser": "^2.2.3",
|
||||||
|
|
@ -26,9 +26,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@babel/runtime-corejs2": {
|
"node_modules/@babel/runtime-corejs2": {
|
||||||
"version": "7.24.6",
|
"version": "7.25.0",
|
||||||
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.24.6.tgz",
|
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.25.0.tgz",
|
||||||
"integrity": "sha512-5UK2PnfpmiCftYGBeJ+SpFIMNaoMPU/eQt1P5ISx0TB7nGGzEMLT4/3PapNZEfGZh+nGxGOGj2t59prGFBhunQ==",
|
"integrity": "sha512-aoYVE3tm+vgAoezmXFWmVcp+NlSdsUqQMPL7c6zRxq8KDHCf570pamC7005Q/UkSlTuoL6oeE16zIw/9J3YFyw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"core-js": "^2.6.12",
|
"core-js": "^2.6.12",
|
||||||
|
|
@ -180,9 +180,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@postman/tunnel-agent": {
|
"node_modules/@postman/tunnel-agent": {
|
||||||
"version": "0.6.3",
|
"version": "0.6.4",
|
||||||
"resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.3.tgz",
|
"resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.4.tgz",
|
||||||
"integrity": "sha512-k57fzmAZ2PJGxfOA4SGR05ejorHbVAa/84Hxh/2nAztjNXc4ZjOm9NUIk6/Z6LCrBvJZqjRZbN8e/nROVUPVdg==",
|
"integrity": "sha512-CJJlq8V7rNKhAw4sBfjixKpJW00SHqebqNUQKxMoepgeWZIbdPcD+rguRcivGhS4N12PymDcKgUgSD4rVC+RjQ==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"safe-buffer": "^5.0.1"
|
"safe-buffer": "^5.0.1"
|
||||||
|
|
@ -236,13 +236,13 @@
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "20.14.0",
|
"version": "22.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.4.1.tgz",
|
||||||
"integrity": "sha512-5cHBxFGJx6L4s56Bubp4fglrEpmyJypsqI6RgzMfBHWUJQGWAAi8cWcgetEbZXHYXo9C2Fa4EEds/uSyS4cxmA==",
|
"integrity": "sha512-1tbpb9325+gPnKK0dMm+/LMriX0vKxf6RnB0SZUqfyVkQ4fMgUSySqhxE/y8Jvs4NyF1yHzTfG9KlnkIODxPKg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~5.26.4"
|
"undici-types": "~6.19.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/yauzl": {
|
"node_modules/@types/yauzl": {
|
||||||
|
|
@ -353,9 +353,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/aws4": {
|
"node_modules/aws4": {
|
||||||
"version": "1.13.0",
|
"version": "1.13.1",
|
||||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.0.tgz",
|
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz",
|
||||||
"integrity": "sha512-3AungXC4I8kKsS9PuS4JH2nc+0bVY/mjgrephHTIi8fpEeGsTHBUJeosp0Wc1myYMElmD0B3Oc4XL/HVJ4PV2g==",
|
"integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/b4a": {
|
"node_modules/b4a": {
|
||||||
|
|
@ -365,9 +365,9 @@
|
||||||
"license": "Apache-2.0"
|
"license": "Apache-2.0"
|
||||||
},
|
},
|
||||||
"node_modules/bare-events": {
|
"node_modules/bare-events": {
|
||||||
"version": "2.3.1",
|
"version": "2.4.2",
|
||||||
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz",
|
||||||
"integrity": "sha512-sJnSOTVESURZ61XgEleqmP255T6zTYwHPwE4r6SssIh0U9/uDvfpdoJYpVUerJJZH2fueO+CdT8ZT+OC/7aZDA==",
|
"integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"optional": true
|
"optional": true
|
||||||
},
|
},
|
||||||
|
|
@ -700,9 +700,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/debug": {
|
"node_modules/debug": {
|
||||||
"version": "4.3.5",
|
"version": "4.3.6",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz",
|
||||||
"integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
|
"integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"ms": "2.1.2"
|
"ms": "2.1.2"
|
||||||
|
|
@ -793,9 +793,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/dompurify": {
|
"node_modules/dompurify": {
|
||||||
"version": "3.1.5",
|
"version": "3.1.6",
|
||||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.6.tgz",
|
||||||
"integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==",
|
"integrity": "sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ==",
|
||||||
"license": "(MPL-2.0 OR Apache-2.0)"
|
"license": "(MPL-2.0 OR Apache-2.0)"
|
||||||
},
|
},
|
||||||
"node_modules/domutils": {
|
"node_modules/domutils": {
|
||||||
|
|
@ -1174,9 +1174,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/https-proxy-agent": {
|
"node_modules/https-proxy-agent": {
|
||||||
"version": "7.0.4",
|
"version": "7.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz",
|
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz",
|
||||||
"integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==",
|
"integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"agent-base": "^7.0.2",
|
"agent-base": "^7.0.2",
|
||||||
|
|
@ -1629,9 +1629,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/nwsapi": {
|
"node_modules/nwsapi": {
|
||||||
"version": "2.2.10",
|
"version": "2.2.12",
|
||||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.10.tgz",
|
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz",
|
||||||
"integrity": "sha512-QK0sRs7MKv0tKe1+5uZIQk/C8XGza4DAnztJG8iD+TpJIORARrCxczA738awHrZoHeTjSSoHqao2teO0dC/gFQ==",
|
"integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/oauth-sign": {
|
"node_modules/oauth-sign": {
|
||||||
|
|
@ -1653,9 +1653,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/pac-proxy-agent": {
|
"node_modules/pac-proxy-agent": {
|
||||||
"version": "7.0.1",
|
"version": "7.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz",
|
||||||
"integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==",
|
"integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tootallnate/quickjs-emscripten": "^0.23.0",
|
"@tootallnate/quickjs-emscripten": "^0.23.0",
|
||||||
|
|
@ -1663,9 +1663,9 @@
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
"get-uri": "^6.0.1",
|
"get-uri": "^6.0.1",
|
||||||
"http-proxy-agent": "^7.0.0",
|
"http-proxy-agent": "^7.0.0",
|
||||||
"https-proxy-agent": "^7.0.2",
|
"https-proxy-agent": "^7.0.5",
|
||||||
"pac-resolver": "^7.0.0",
|
"pac-resolver": "^7.0.1",
|
||||||
"socks-proxy-agent": "^8.0.2"
|
"socks-proxy-agent": "^8.0.4"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 14"
|
"node": ">= 14"
|
||||||
|
|
@ -1727,14 +1727,14 @@
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/postman-request": {
|
"node_modules/postman-request": {
|
||||||
"version": "2.88.1-postman.33",
|
"version": "2.88.1-postman.39",
|
||||||
"resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.33.tgz",
|
"resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.39.tgz",
|
||||||
"integrity": "sha512-uL9sCML4gPH6Z4hreDWbeinKU0p0Ke261nU7OvII95NU22HN6Dk7T/SaVPaj6T4TsQqGKIFw6/woLZnH7ugFNA==",
|
"integrity": "sha512-rsncxxDlbn1YpygXSgJqbJzIjGlHFcZjbYDzeBPTQHMDfLuSTzZz735JHV8i1+lOROuJ7MjNap4eaSD3UijHzQ==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@postman/form-data": "~3.1.1",
|
"@postman/form-data": "~3.1.1",
|
||||||
"@postman/tough-cookie": "~4.1.3-postman.1",
|
"@postman/tough-cookie": "~4.1.3-postman.1",
|
||||||
"@postman/tunnel-agent": "^0.6.3",
|
"@postman/tunnel-agent": "^0.6.4",
|
||||||
"aws-sign2": "~0.7.0",
|
"aws-sign2": "~0.7.0",
|
||||||
"aws4": "^1.12.0",
|
"aws4": "^1.12.0",
|
||||||
"brotli": "^1.3.3",
|
"brotli": "^1.3.3",
|
||||||
|
|
@ -1756,7 +1756,7 @@
|
||||||
"uuid": "^8.3.2"
|
"uuid": "^8.3.2"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 6"
|
"node": ">= 16"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/process-nextick-args": {
|
"node_modules/process-nextick-args": {
|
||||||
|
|
@ -2148,14 +2148,14 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/socks-proxy-agent": {
|
"node_modules/socks-proxy-agent": {
|
||||||
"version": "8.0.3",
|
"version": "8.0.4",
|
||||||
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz",
|
||||||
"integrity": "sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A==",
|
"integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"agent-base": "^7.1.1",
|
"agent-base": "^7.1.1",
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
"socks": "^2.7.1"
|
"socks": "^2.8.3"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 14"
|
"node": ">= 14"
|
||||||
|
|
@ -2322,9 +2322,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/text-decoder": {
|
"node_modules/text-decoder": {
|
||||||
"version": "1.1.0",
|
"version": "1.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz",
|
||||||
"integrity": "sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==",
|
"integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"b4a": "^1.6.4"
|
"b4a": "^1.6.4"
|
||||||
|
|
@ -2376,9 +2376,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/tslib": {
|
"node_modules/tslib": {
|
||||||
"version": "2.6.2",
|
"version": "2.6.3",
|
||||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
|
||||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==",
|
"integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
|
||||||
"license": "0BSD"
|
"license": "0BSD"
|
||||||
},
|
},
|
||||||
"node_modules/turndown": {
|
"node_modules/turndown": {
|
||||||
|
|
@ -2407,9 +2407,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/undici-types": {
|
"node_modules/undici-types": {
|
||||||
"version": "5.26.5",
|
"version": "6.19.8",
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
|
||||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true
|
"optional": true
|
||||||
},
|
},
|
||||||
|
|
@ -2575,9 +2575,9 @@
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/ws": {
|
"node_modules/ws": {
|
||||||
"version": "8.17.0",
|
"version": "8.18.0",
|
||||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
|
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
|
||||||
"integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
|
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=10.0.0"
|
"node": ">=10.0.0"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "archivebox",
|
"name": "archivebox",
|
||||||
"version": "0.8.1",
|
"version": "0.8.2",
|
||||||
"description": "ArchiveBox: The self-hosted internet archive",
|
"description": "ArchiveBox: The self-hosted internet archive",
|
||||||
"author": "Nick Sweeting <archivebox-npm@sweeting.me>",
|
"author": "Nick Sweeting <archivebox-npm@sweeting.me>",
|
||||||
"repository": "github:ArchiveBox/ArchiveBox",
|
"repository": "github:ArchiveBox/ArchiveBox",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[project]
|
[project]
|
||||||
name = "archivebox"
|
name = "archivebox"
|
||||||
version = "0.8.1"
|
version = "0.8.2"
|
||||||
package-dir = "archivebox"
|
package-dir = "archivebox"
|
||||||
requires-python = ">=3.10,<3.13"
|
requires-python = ">=3.10,<3.13"
|
||||||
platform = "py3-none-any"
|
platform = "py3-none-any"
|
||||||
|
|
@ -12,7 +12,7 @@ readme = "README.md"
|
||||||
# pdm install
|
# pdm install
|
||||||
# pdm update --unconstrained
|
# pdm update --unconstrained
|
||||||
dependencies = [
|
dependencies = [
|
||||||
# Last Bumped: 2024-04-25
|
# Last Bumped: 2024-08-20
|
||||||
# Base Framework and Language Dependencies
|
# Base Framework and Language Dependencies
|
||||||
"setuptools>=69.5.1",
|
"setuptools>=69.5.1",
|
||||||
"django>=5.0.4,<6.0",
|
"django>=5.0.4,<6.0",
|
||||||
|
|
|
||||||
|
|
@ -7,74 +7,74 @@ asgiref==3.8.1
|
||||||
asttokens==2.4.1
|
asttokens==2.4.1
|
||||||
brotli==1.1.0; implementation_name == "cpython"
|
brotli==1.1.0; implementation_name == "cpython"
|
||||||
brotlicffi==1.1.0.0; implementation_name != "cpython"
|
brotlicffi==1.1.0.0; implementation_name != "cpython"
|
||||||
certifi==2024.6.2
|
certifi==2024.7.4
|
||||||
cffi==1.16.0; platform_python_implementation != "PyPy" or implementation_name != "cpython"
|
cffi==1.17.0; platform_python_implementation != "PyPy" or implementation_name != "cpython"
|
||||||
charset-normalizer==3.3.2
|
charset-normalizer==3.3.2
|
||||||
colorama==0.4.6; sys_platform == "win32"
|
colorama==0.4.6; sys_platform == "win32"
|
||||||
croniter==2.0.5
|
croniter==3.0.3
|
||||||
cryptography==42.0.7
|
cryptography==43.0.0
|
||||||
dateparser==1.2.0
|
dateparser==1.2.0
|
||||||
decorator==5.1.1
|
decorator==5.1.1
|
||||||
django==5.0.6
|
django==5.1
|
||||||
django-admin-data-views==0.3.1
|
django-admin-data-views==0.3.1
|
||||||
django-auth-ldap==4.8.0
|
django-auth-ldap==4.8.0
|
||||||
django-charid-field==0.4
|
django-charid-field==0.4
|
||||||
django-extensions==3.2.3
|
django-extensions==3.2.3
|
||||||
django-jsonform==2.22.0
|
django-jsonform==2.22.0
|
||||||
django-ninja==1.1.0
|
django-ninja==1.3.0
|
||||||
django-pydantic-field==0.3.9
|
django-pydantic-field==0.3.10
|
||||||
django-settings-holder==0.1.2
|
django-settings-holder==0.1.2
|
||||||
django-signal-webhooks==0.3.0
|
django-signal-webhooks==0.3.0
|
||||||
django-stubs==5.0.2
|
django-stubs==5.0.4
|
||||||
django-stubs-ext==5.0.2
|
django-stubs-ext==5.0.4
|
||||||
exceptiongroup==1.2.1; python_version < "3.11"
|
exceptiongroup==1.2.2; python_version < "3.11"
|
||||||
executing==2.0.1
|
executing==2.0.1
|
||||||
feedparser==6.0.11
|
feedparser==6.0.11
|
||||||
h11==0.14.0
|
h11==0.14.0
|
||||||
httpcore==1.0.5
|
httpcore==1.0.5
|
||||||
httpx==0.27.0
|
httpx==0.27.0
|
||||||
idna==3.7
|
idna==3.7
|
||||||
ipython==8.25.0
|
ipython==8.26.0
|
||||||
jedi==0.19.1
|
jedi==0.19.1
|
||||||
matplotlib-inline==0.1.7
|
matplotlib-inline==0.1.7
|
||||||
mutagen==1.47.0
|
mutagen==1.47.0
|
||||||
mypy-extensions==1.0.0
|
mypy-extensions==1.0.0
|
||||||
parso==0.8.4
|
parso==0.8.4
|
||||||
pexpect==4.9.0; sys_platform != "win32" and sys_platform != "emscripten"
|
pexpect==4.9.0; sys_platform != "win32" and sys_platform != "emscripten"
|
||||||
prompt-toolkit==3.0.45
|
prompt-toolkit==3.0.47
|
||||||
ptyprocess==0.7.0; sys_platform != "win32" and sys_platform != "emscripten"
|
ptyprocess==0.7.0; sys_platform != "win32" and sys_platform != "emscripten"
|
||||||
pure-eval==0.2.2
|
pure-eval==0.2.3
|
||||||
pyasn1==0.6.0
|
pyasn1==0.6.0
|
||||||
pyasn1-modules==0.4.0
|
pyasn1-modules==0.4.0
|
||||||
pycparser==2.22; platform_python_implementation != "PyPy" or implementation_name != "cpython"
|
pycparser==2.22; platform_python_implementation != "PyPy" or implementation_name != "cpython"
|
||||||
pycryptodomex==3.20.0
|
pycryptodomex==3.20.0
|
||||||
pydantic==2.7.3
|
pydantic==2.8.2
|
||||||
pydantic-core==2.18.4
|
pydantic-core==2.20.1
|
||||||
pygments==2.18.0
|
pygments==2.18.0
|
||||||
python-crontab==3.1.0
|
python-crontab==3.2.0
|
||||||
python-dateutil==2.9.0.post0
|
python-dateutil==2.9.0.post0
|
||||||
python-ldap==3.4.4
|
python-ldap==3.4.4
|
||||||
pytz==2024.1
|
pytz==2024.1
|
||||||
regex==2024.5.15
|
regex==2024.7.24
|
||||||
requests==2.32.3
|
requests==2.32.3
|
||||||
setuptools==70.0.0
|
setuptools==73.0.0
|
||||||
sgmllib3k==1.0.0
|
sgmllib3k==1.0.0
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
sniffio==1.3.1
|
sniffio==1.3.1
|
||||||
sonic-client==1.0.0
|
sonic-client==1.0.0
|
||||||
sqlparse==0.5.0
|
sqlparse==0.5.1
|
||||||
stack-data==0.6.3
|
stack-data==0.6.3
|
||||||
tomli==2.0.1; python_version < "3.11"
|
tomli==2.0.1; python_version < "3.11"
|
||||||
traitlets==5.14.3
|
traitlets==5.14.3
|
||||||
typeid-python==0.3.0
|
typeid-python==0.3.1
|
||||||
types-pyyaml==6.0.12.20240311
|
types-pyyaml==6.0.12.20240808
|
||||||
typing-extensions==4.12.1
|
typing-extensions==4.12.2
|
||||||
tzdata==2024.1; sys_platform == "win32" or platform_system == "Windows"
|
tzdata==2024.1; sys_platform == "win32" or platform_system == "Windows"
|
||||||
tzlocal==5.2
|
tzlocal==5.2
|
||||||
ulid-py==1.1.0
|
ulid-py==1.1.0
|
||||||
urllib3==2.2.1
|
urllib3==2.2.2
|
||||||
uuid6==2023.5.2
|
uuid6==2024.7.10
|
||||||
w3lib==2.1.2
|
w3lib==2.2.1
|
||||||
wcwidth==0.2.13
|
wcwidth==0.2.13
|
||||||
websockets==12.0
|
websockets==12.0
|
||||||
yt-dlp==2024.5.27
|
yt-dlp==2024.8.6
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue