From bb2a2e758aff1953b0275a0fe18fb14ed1dd3daf Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Tue, 7 Sep 2021 21:53:36 -0300 Subject: [PATCH 001/124] Avoid KeyError on Pocket API parser When trying to import my pocket library I got a lot of ` KeyError` on Python. Pocket API has a few idiosyncrasies, such as sometimes returning the keys on json, sometimes not. ` ` ` sh archivebox add --parser pocket_api pocket://my_username ` ` ` Gave me this errors ` ` ` File "/app/archivebox/parsers/pocket_api.py", line 54, in link_from_article title = article['resolved_title'] or article['given_title'] or url KeyError: 'resolved_title' ` ` ` This commit are the patches I've changed to successfully import my library --- archivebox/parsers/pocket_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivebox/parsers/pocket_api.py b/archivebox/parsers/pocket_api.py index afad70ed..c51d1103 100644 --- a/archivebox/parsers/pocket_api.py +++ b/archivebox/parsers/pocket_api.py @@ -47,11 +47,11 @@ def get_pocket_articles(api: Pocket, since=None, page=0): def link_from_article(article: dict, sources: list): - url: str = article['resolved_url'] or article['given_url'] + url: str = articl.get('resolved_url') or article['given_url'] broken_protocol = _BROKEN_PROTOCOL_RE.match(url) if broken_protocol: url = url.replace(f'{broken_protocol.group(1)}:/', f'{broken_protocol.group(1)}://') - title = article['resolved_title'] or article['given_title'] or url + title = article.get('resolved_title') or article.get('given_title') or url return Link( url=url, From 05de1c9fe603b113813361c154b076cea22f208a Mon Sep 17 00:00:00 2001 From: Igor Rzegocki Date: Sun, 3 Oct 2021 19:12:03 +0200 Subject: [PATCH 002/124] healthcheck endpoint --- archivebox/core/urls.py | 8 +++++--- archivebox/core/views.py | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 87a302b8..8a3f0e22 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -6,7 +6,7 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.conf import settings from django.views.generic.base import RedirectView -from core.views import HomepageView, SnapshotView, PublicIndexView, AddView +from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView # print('DEBUG', settings.DEBUG) @@ -24,14 +24,16 @@ urlpatterns = [ path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')), path('add/', AddView.as_view(), name='add'), - + path('accounts/login/', RedirectView.as_view(url='/admin/login/')), path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')), path('accounts/', include('django.contrib.auth.urls')), path('admin/', admin.site.urls), - + + path('health/', HealthCheckView.as_view(), name='healthcheck'), + path('index.html', RedirectView.as_view(url='/')), path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}), path('', HomepageView.as_view(), name='Home'), diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 5385add9..3f3fec12 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -38,7 +38,7 @@ class HomepageView(View): if PUBLIC_INDEX: return redirect('/public') - + return redirect(f'/admin/login/?next={request.path}') @@ -205,7 +205,7 @@ class SnapshotView(View): content_type="text/html", status=404, ) - + class PublicIndexView(ListView): template_name = 'public_index.html' @@ -220,7 +220,7 @@ class PublicIndexView(ListView): 'FOOTER_INFO': FOOTER_INFO, } - def get_queryset(self, **kwargs): + def get_queryset(self, **kwargs): qs = super().get_queryset(**kwargs) query = self.request.GET.get('q') if query and query.strip(): @@ -249,7 +249,7 @@ class AddView(UserPassesTestMixin, FormView): url = self.request.GET.get('url', None) if url: return {'url': url if '://' in url else f'https://{url}'} - + return super().get_initial() def test_func(self): @@ -295,3 +295,18 @@ class AddView(UserPassesTestMixin, FormView): "form": AddLinkForm() }) return render(template_name=self.template_name, request=self.request, context=context) + + +class HealthCheckView(View): + """ + A Django view that renders plain text "OK" for service discovery tools + """ + def get(self, request): + """ + Handle a GET request + """ + return HttpResponse( + 'OK', + content_type='text/plain', + status=200 + ) From aaa459082db87a5ed70a10b3f02167b7ecd23c2f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Oct 2021 01:58:16 -0400 Subject: [PATCH 003/124] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5febf75d..3bcfcbdc 100644 --- a/README.md +++ b/README.md @@ -367,8 +367,8 @@ archivebox help #### 🖥  Web UI Usage ```bash -archivebox manage createsuperuser -archivebox server 0.0.0.0:8000 # open http://127.0.0.1:8000 to view it +archivebox manage createsuperuser # set an admin password +archivebox server 0.0.0.0:8000 # open http://127.0.0.1:8000 to view it # you can also configure whether or not login is required for most features archivebox config --set PUBLIC_INDEX=False From b54076054c6063cb79f2d6a231ac51019ee888aa Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Oct 2021 02:02:23 -0400 Subject: [PATCH 004/124] Update README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bcfcbdc..d0dcb1c5 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,13 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
-**📦  Get ArchiveBox with `docker-compose` / `docker` / `apt` / `brew` / `pip3` ([see Quickstart below](#quickstart)).** +**📦  Get ArchiveBox with Docker / `apt` / `brew` / `pip3` / etc. ([see Quickstart below](#quickstart)).** ```bash -# Or use this auto setup script to install it for you (optional) +# Follow the instructions for your package manager in the quickstart, e.g.: +pip3 install archivebox + +# Or use the optional auto setup script to install it for you: curl -sSL 'https://get.archivebox.io' | sh ``` From 3b6ef764c5562f46ab2ddb3662a37bac3de2174a Mon Sep 17 00:00:00 2001 From: Adam Wolf Date: Thu, 28 Oct 2021 10:01:34 -0500 Subject: [PATCH 005/124] Update README.md Bind the data directory when initializing a dev environment using Docker. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0dcb1c5..b25686ab 100644 --- a/README.md +++ b/README.md @@ -898,7 +898,9 @@ archivebox --version # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver # inside the container will reload and pick up your changes docker build . -t archivebox -docker run -it archivebox init --setup +docker run -it \ + -v $PWD/data:/data \ + archivebox init --setup docker run -it -p 8000:8000 \ -v $PWD/data:/data \ -v $PWD/archivebox:/app/archivebox \ From 83731f5a68544f612e0fca787ddc7b43d7d39ae6 Mon Sep 17 00:00:00 2001 From: Adam Wolf Date: Thu, 28 Oct 2021 22:22:46 -0500 Subject: [PATCH 006/124] Tweak JS so Safari can choose admin actions I noticed that Safari was submitting both the empty option and the selected options back to the server. Digging into it, I was able to get Safari to deselect the --------- option by using '[selected]' as the selector. For #658 --- archivebox/templates/admin/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 9dc62516..0592fa0a 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -197,7 +197,7 @@ // select the action button from the dropdown container.find('select[name=action]') - .find('op:selected').removeAttr('selected').end() + .find('[selected]').removeAttr('selected').end() .find('[value=' + action_type + ']').attr('selected', 'selected').click() // click submit & replace the archivebox logo with a spinner From 18e1fb0d9630ffcf7800c8255b1df98a2ef476ac Mon Sep 17 00:00:00 2001 From: Adam Wolf Date: Thu, 28 Oct 2021 22:31:54 -0500 Subject: [PATCH 007/124] Fixes Add button behavior on Safari Previously, when you clicked the Add button, the page wouldn't change. It looked like it wasn't doing anything, as noted by @rcarmo (https://github.com/ArchiveBox/ArchiveBox/issues/658#issuecomment-948300055) I didn't track it down the exact reason why. It may be that Safari didn't like the two opening

s, but I was able to find a bunch of people complaining about Safari being very finicky with innerHTML and actually repainting the page, enough that I decided to try just extending the block hide/show behavior already done with the delay-warning, and it works for me now in both Chrome and Safari. For #658. --- archivebox/templates/core/add.html | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 4a60b02e..7d6efc6c 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -28,6 +28,14 @@   Add more URLs âž• {% else %} +
{% csrf_token %}

Add new URLs to your archive


@@ -48,10 +56,9 @@ {% endif %} From 44f53384709855c895513ddcb39eb4c734e52191 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 12 Nov 2021 19:23:47 -0500 Subject: [PATCH 008/124] fix typo in pocket_api articl variable name --- archivebox/parsers/pocket_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/parsers/pocket_api.py b/archivebox/parsers/pocket_api.py index c51d1103..eec4d73b 100644 --- a/archivebox/parsers/pocket_api.py +++ b/archivebox/parsers/pocket_api.py @@ -47,7 +47,7 @@ def get_pocket_articles(api: Pocket, since=None, page=0): def link_from_article(article: dict, sources: list): - url: str = articl.get('resolved_url') or article['given_url'] + url: str = article.get('resolved_url') or article['given_url'] broken_protocol = _BROKEN_PROTOCOL_RE.match(url) if broken_protocol: url = url.replace(f'{broken_protocol.group(1)}:/', f'{broken_protocol.group(1)}://') From 84b927e3e5fb8da93fb86a9070291495a7563a35 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 16 Nov 2021 21:13:19 -0500 Subject: [PATCH 009/124] change header title --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b25686ab..364c9529 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,9 @@ See -#### 🛠  Manual Setup +#### 🛠  Package Manager Setup +
aptitude apt (Ubuntu/Debian)
From b40a1e19d1a5c34f3c88e5b1e377fd93e6ef774a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Dec 2021 21:03:19 -0500 Subject: [PATCH 010/124] comment out healthcheck --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c9daca81..06111f1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,8 +119,9 @@ RUN /app/bin/docker_entrypoint.sh archivebox version VOLUME "$DATA_DIR" EXPOSE 8000 -HEALTHCHECK --interval=30s --timeout=20s --retries=15 \ - CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1 +# Optional: +# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \ +# CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"] CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"] From 63fc835a705f0ac9f1e85d460ff05cb8264c55d9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 6 Dec 2021 16:23:31 -0500 Subject: [PATCH 011/124] link to archivebox-exporter browser extension --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 364c9529..2899d755 100644 --- a/README.md +++ b/README.md @@ -423,6 +423,7 @@ ArchiveBox supports many input formats for URLs, including Pocket & Pinboard exp - TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file) - [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)) +- Browser extension [`archivebox-exporter`](https://github.com/tjhorner/archivebox-exporter) (realtime archiving from Chrome/Chromium/Firefox) - [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) From b279c30b8d0be141ce9d807206c8d95a7d530ec2 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 16 Dec 2021 05:18:33 -0500 Subject: [PATCH 012/124] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b2959d5..e3bd8c05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ # Documentation: # https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker-compose -version: '2.4' +version: '2.4' # '3.9' or greater also works services: archivebox: From decab91ea239f4783cb91b42557d3c98f7d0078c Mon Sep 17 00:00:00 2001 From: TheCakeIsNaOH Date: Thu, 16 Dec 2021 16:46:12 -0600 Subject: [PATCH 013/124] (#847) Decode error output hints to string if needed --- archivebox/logging_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index a8c4e590..e2403850 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -432,7 +432,10 @@ def log_archive_method_finished(result: "ArchiveResult"): # Prettify error output hints string and limit to five lines hints = getattr(result.output, 'hints', None) or () if hints: - hints = hints if isinstance(hints, (list, tuple)) else hints.split('\n') + if not isinstance(hints, (list, tuple)): + if isinstance(hints, bytes): + hints = hints.decode() + hints = hints.split('\n') hints = ( ' {}{}{}'.format(ANSI['lightyellow'], line.strip(), ANSI['reset']) for line in hints[:5] if line.strip() From ea0d9c1d998d1fd789ca8dce2a00cfc8c98a0307 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 17 Dec 2021 17:55:40 -0500 Subject: [PATCH 014/124] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2899d755..68b46d81 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ ls ./archive/*/index.json # or browse directly via the filesyste ## Key Features -- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally +- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up online, stores all data locally - [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl), articles (readability), code (git), etc.](#output-formats) @@ -92,7 +92,7 @@ ls ./archive/*/index.json # or browse directly via the filesyste - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA) - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode) -- Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released) +- Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!) - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)...

From f14a861605915c4e17021e8846c107a282b58fba Mon Sep 17 00:00:00 2001 From: Mika Tuupola Date: Sun, 19 Dec 2021 23:17:33 +0200 Subject: [PATCH 015/124] Change logfile open to write mode only --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index ba68e2a3..0551de2e 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1157,7 +1157,7 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, from django.conf import settings # log startup message to the error log - with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f: + with open(settings.ERROR_LOG, "a", encoding='utf-8') as f: command = ' '.join(sys.argv) ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S') f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") From 4b8962b60b1540729a048483e76af1f5563707a9 Mon Sep 17 00:00:00 2001 From: hannah98 Date: Mon, 20 Dec 2021 08:58:58 -0600 Subject: [PATCH 016/124] Fix #725 - correctly parse tags on json import --- archivebox/index/sql.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 2fcabd61..6858ce5d 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -34,8 +34,11 @@ def write_link_to_sql_index(link: Link): from core.models import Snapshot, ArchiveResult info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} tags = info.pop("tags") - if tags is None: - tags = [] + + tag_set = ( + set(tag.strip() for tag in (link.tags or '').split(',')) + ) + tag_list = list(tag_set) or [] try: info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp @@ -44,7 +47,7 @@ def write_link_to_sql_index(link: Link): info["timestamp"] = str(float(info["timestamp"]) + 1.0) snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info) - snapshot.save_tags(tags) + snapshot.save_tags(tag_list) for extractor, entries in link.history.items(): for entry in entries: From b1b7ee2b856474d3ecfda3abf1f6b8afaac591de Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 12:17:55 -0500 Subject: [PATCH 017/124] Update sql.py --- archivebox/index/sql.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 6858ce5d..8a27b645 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -33,12 +33,10 @@ def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir: def write_link_to_sql_index(link: Link): from core.models import Snapshot, ArchiveResult info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} - tags = info.pop("tags") - tag_set = ( - set(tag.strip() for tag in (link.tags or '').split(',')) - ) - tag_list = list(tag_set) or [] + tag_list = list(dict.fromkeys( + tag.strip() for tag in (link.tags or '').split(',') + )) try: info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp @@ -107,10 +105,9 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: snap = write_link_to_sql_index(link) snap.title = link.title - tag_set = ( - set(tag.strip() for tag in (link.tags or '').split(',')) - ) - tag_list = list(tag_set) or [] + tag_list = list(dict.fromkeys( + tag.strip() for tag in (link.tags or '').split(',') + )) snap.save() snap.save_tags(tag_list) From d7f01922f32891164538c1dac8fcc31679cac4b3 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 12:29:17 -0500 Subject: [PATCH 018/124] fix direct assignment of tags to many-to-many set --- archivebox/index/sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 8a27b645..661436cf 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -37,6 +37,7 @@ def write_link_to_sql_index(link: Link): tag_list = list(dict.fromkeys( tag.strip() for tag in (link.tags or '').split(',') )) + info.pop('tags') try: info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp From 559c395c1e234b39cb7ac11bb986a7c822c4c171 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 12:49:03 -0500 Subject: [PATCH 019/124] run archivebox version after homebrew bottle build --- .github/workflows/homebrew.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index d9bb05f1..ce8ded30 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -23,6 +23,7 @@ jobs: cd brew_dist/ brew install --build-bottle ./archivebox.rb # brew bottle archivebox + archivebox version - name: Add some links to test run: | From 1f7e15e3c0eea3cc58bd1375753560ee6da016c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Dec 2021 17:49:42 +0000 Subject: [PATCH 020/124] Bump jszip from 3.6.0 to 3.7.1 Bumps [jszip](https://github.com/Stuk/jszip) from 3.6.0 to 3.7.1. - [Release notes](https://github.com/Stuk/jszip/releases) - [Changelog](https://github.com/Stuk/jszip/blob/master/CHANGES.md) - [Commits](https://github.com/Stuk/jszip/compare/v3.6.0...v3.7.1) --- updated-dependencies: - dependency-name: jszip dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 110 +++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d61f755..017cf80c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -105,9 +105,9 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, "@types/node": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.0.0.tgz", - "integrity": "sha512-TmCW5HoZ2o2/z2EYi109jLqIaPIi9y/lc2LmDCWzuCi35bcaQ+OtUh6nwBiFK7SOu25FAU5+YKdqFZUwtqGSdg==", + "version": "17.0.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.4.tgz", + "integrity": "sha512-6xwbrW4JJiJLgF+zNypN5wr2ykM9/jHcL7rQ8fZe2vuftggjzZeRSM4OwRc6Xk8qWjwJ99qVHo/JgOGmomWRog==", "optional": true }, "@types/yauzl": { @@ -170,9 +170,9 @@ } }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", @@ -445,9 +445,9 @@ } }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "requires": { "ms": "2.1.2" } @@ -515,9 +515,9 @@ } }, "dompurify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.0.tgz", - "integrity": "sha512-VV5C6Kr53YVHGOBKO/F86OYX6/iLTw2yVSI721gKetxpHCK/V5TaLEf9ODjRgl1KLSWRMY6cUhAbv/c+IUnwQw==" + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.4.tgz", + "integrity": "sha512-6BVcgOAVFXjI0JTjEvZy901Rghm+7fDQOrNIcxB4+gdhj6Kwp6T9VBhBY/AbagKHJocRkDYGd6wvI+p4/10xtQ==" }, "domutils": { "version": "1.5.1", @@ -702,9 +702,9 @@ } }, "glob": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -913,9 +913,9 @@ } }, "jszip": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", - "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.1.tgz", + "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", "requires": { "lie": "~3.3.0", "pako": "~1.0.2", @@ -1308,9 +1308,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1334,9 +1334,9 @@ }, "dependencies": { "acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==" }, "acorn-globals": { "version": "6.0.0", @@ -1417,9 +1417,9 @@ } }, "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" }, "form-data": { "version": "3.0.1", @@ -1440,9 +1440,9 @@ } }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "requires": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -1469,7 +1469,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" } }, @@ -1512,9 +1512,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1683,9 +1683,9 @@ }, "dependencies": { "acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==" }, "acorn-globals": { "version": "6.0.0", @@ -1766,9 +1766,9 @@ } }, "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" }, "form-data": { "version": "3.0.1", @@ -1797,9 +1797,9 @@ } }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "requires": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -1826,7 +1826,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" } }, @@ -1869,9 +1869,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1916,13 +1916,13 @@ "integrity": "sha1-PYRT5ydKLkShQrPchEnftk2a3jo=" }, "string-width": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", - "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" + "strip-ansi": "^6.0.1" } }, "string_decoder": { @@ -1934,11 +1934,11 @@ } }, "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "requires": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" } }, "strong-data-uri": { From 2c3a25765f5dab9c6f261c42ebcb828ff07fad43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Dec 2021 17:49:43 +0000 Subject: [PATCH 021/124] Bump ansi-regex from 5.0.0 to 5.0.1 Bumps [ansi-regex](https://github.com/chalk/ansi-regex) from 5.0.0 to 5.0.1. - [Release notes](https://github.com/chalk/ansi-regex/releases) - [Commits](https://github.com/chalk/ansi-regex/compare/v5.0.0...v5.0.1) --- updated-dependencies: - dependency-name: ansi-regex dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 110 +++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d61f755..017cf80c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -105,9 +105,9 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, "@types/node": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.0.0.tgz", - "integrity": "sha512-TmCW5HoZ2o2/z2EYi109jLqIaPIi9y/lc2LmDCWzuCi35bcaQ+OtUh6nwBiFK7SOu25FAU5+YKdqFZUwtqGSdg==", + "version": "17.0.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.4.tgz", + "integrity": "sha512-6xwbrW4JJiJLgF+zNypN5wr2ykM9/jHcL7rQ8fZe2vuftggjzZeRSM4OwRc6Xk8qWjwJ99qVHo/JgOGmomWRog==", "optional": true }, "@types/yauzl": { @@ -170,9 +170,9 @@ } }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", @@ -445,9 +445,9 @@ } }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "requires": { "ms": "2.1.2" } @@ -515,9 +515,9 @@ } }, "dompurify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.0.tgz", - "integrity": "sha512-VV5C6Kr53YVHGOBKO/F86OYX6/iLTw2yVSI721gKetxpHCK/V5TaLEf9ODjRgl1KLSWRMY6cUhAbv/c+IUnwQw==" + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.4.tgz", + "integrity": "sha512-6BVcgOAVFXjI0JTjEvZy901Rghm+7fDQOrNIcxB4+gdhj6Kwp6T9VBhBY/AbagKHJocRkDYGd6wvI+p4/10xtQ==" }, "domutils": { "version": "1.5.1", @@ -702,9 +702,9 @@ } }, "glob": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -913,9 +913,9 @@ } }, "jszip": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", - "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.1.tgz", + "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", "requires": { "lie": "~3.3.0", "pako": "~1.0.2", @@ -1308,9 +1308,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1334,9 +1334,9 @@ }, "dependencies": { "acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==" }, "acorn-globals": { "version": "6.0.0", @@ -1417,9 +1417,9 @@ } }, "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" }, "form-data": { "version": "3.0.1", @@ -1440,9 +1440,9 @@ } }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "requires": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -1469,7 +1469,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" } }, @@ -1512,9 +1512,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1683,9 +1683,9 @@ }, "dependencies": { "acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==" }, "acorn-globals": { "version": "6.0.0", @@ -1766,9 +1766,9 @@ } }, "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" }, "form-data": { "version": "3.0.1", @@ -1797,9 +1797,9 @@ } }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "requires": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -1826,7 +1826,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" } }, @@ -1869,9 +1869,9 @@ } }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==" + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -1916,13 +1916,13 @@ "integrity": "sha1-PYRT5ydKLkShQrPchEnftk2a3jo=" }, "string-width": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", - "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" + "strip-ansi": "^6.0.1" } }, "string_decoder": { @@ -1934,11 +1934,11 @@ } }, "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "requires": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" } }, "strong-data-uri": { From f144f7b5c653416ded06ec1db561c0d3708c2214 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 15:39:49 -0500 Subject: [PATCH 022/124] bump brew disk version --- brew_dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brew_dist b/brew_dist index 95a1c1a0..a4314719 160000 --- a/brew_dist +++ b/brew_dist @@ -1 +1 @@ -Subproject commit 95a1c1a0875841d076f06106bd4c2307504928c2 +Subproject commit a4314719746de549f359c2fa975762fc73b62f94 From 9850333b8816fb09b92dab9e82e832aabc94a598 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 15:43:13 -0500 Subject: [PATCH 023/124] Update _config.yml --- _config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index c50ff38d..9f63db0d 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1,3 @@ -theme: jekyll-theme-merlot \ No newline at end of file +production_url: https://archivebox.io +theme: jekyll-theme-merlot +# Github Pages static site settings for https://archivebox.io From 0aa75c861bdddaf2de2a7619ff661500fff6a5f6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 15:56:28 -0500 Subject: [PATCH 024/124] also install npm packages when testing brew package --- .github/workflows/homebrew.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index ce8ded30..af7a0795 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -28,7 +28,7 @@ jobs: - name: Add some links to test run: | mkdir data && cd data - archivebox init + archivebox init --setup archivebox add 'https://example.com' archivebox version archivebox status From 7a7217abb46890bc35b4ad1d939edd7673c330b6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 15:59:50 -0500 Subject: [PATCH 025/124] Bump mercury-parser version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 782a7be7..194713e9 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { - "@postlight/mercury-parser": "^2.2.0", + "@postlight/mercury-parser": "^2.2.1", "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git", "single-file": "git+https://github.com/gildas-lormeau/SingleFile.git" } From 745f6573c13a729b83618b83f44a34bf37514544 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Dec 2021 16:03:51 -0500 Subject: [PATCH 026/124] bump node dependencies --- package-lock.json | 83 +++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 017cf80c..2235c33e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@babel/runtime-corejs2": { - "version": "7.13.10", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.13.10.tgz", - "integrity": "sha512-rZw5P1ZewO6XZTDxtXuAuAFUqfNXyM8HO/9WiaDd34Anka0uFTpo0RvBLeV775AEE/zKw3LQB+poZw/O9lrZBg==", + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.16.5.tgz", + "integrity": "sha512-GHejyoK+JQqna9rUTkybaGahZOOM+EDcUsbWaLye1g4ZOQMotrMiY8VTBbX1gycZTMNG/YiWTo4WTABvyPUgOg==", "requires": { "core-js": "^2.6.5", "regenerator-runtime": "^0.13.4" @@ -28,9 +28,9 @@ } }, "@postlight/mercury-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@postlight/mercury-parser/-/mercury-parser-2.2.0.tgz", - "integrity": "sha512-nz6dIvCAaiv74o1vhhp0BRsUe+ysPbZG5mVNpJmgLoI/goOBqRMM3Yg8uXtnv++e7tzKFSXdls8b2/zKk1qL0Q==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@postlight/mercury-parser/-/mercury-parser-2.2.1.tgz", + "integrity": "sha512-WsXHcVfbwlAb0HO1nZ3tS2ipNvLeLk9frJj3y5hn/NAeQfEJeRm9AtMrxpOJ11X8L4he+bRyjr1bpIPvQPWvCA==", "requires": { "@babel/runtime-corejs2": "^7.2.0", "@postlight/ci-failed-test-reporter": "^1.0", @@ -50,7 +50,7 @@ "url": "^0.11.0", "valid-url": "^1.0.9", "wuzzy": "^0.1.4", - "yargs-parser": "^13.0.0" + "yargs-parser": "^14.0.0" }, "dependencies": { "http-headers": { @@ -188,9 +188,9 @@ "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" }, "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", "requires": { "safer-buffer": "~2.1.0" } @@ -729,9 +729,9 @@ } }, "heap": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.6.tgz", - "integrity": "sha1-CH4fELBGky/IWU3Z5tN4r8nR5aw=" + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" }, "html-encoding-sniffer": { "version": "1.0.2", @@ -773,12 +773,12 @@ } }, "http-signature": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.5.tgz", - "integrity": "sha512-NwoTQYSJoFt34jSBbwzDHDofoA61NGXzu6wXh95o1Ry62EnmKjXb/nR/RknLeZ3G/uGwrlKNY2z7uPt+Cdl7Tw==", + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz", + "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==", "requires": { "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", + "jsprim": "^2.0.2", "sshpk": "^1.14.1" } }, @@ -887,9 +887,9 @@ } }, "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" }, "json-schema-traverse": { "version": "0.4.1", @@ -902,13 +902,13 @@ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", + "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==", "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" } }, @@ -1207,9 +1207,9 @@ "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" }, "postman-request": { - "version": "2.88.1-postman.29", - "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.29.tgz", - "integrity": "sha512-QuL3+AvGlmPLb1Qf0t/rM8M4U8LCYbADZBijUNToLl6l37i65KH8wY1gTLWLxlw2I6ugxUfX2Zyyk5/J5HFZIg==", + "version": "2.88.1-postman.30", + "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.30.tgz", + "integrity": "sha512-zsGvs8OgNeno1Q44zTgGP2IL7kCqUy4DAtl8/ms0AQpqkIoysrxzR/Zg4kM1Kz8/duBvwxt8NN717wB7SMNm6w==", "requires": { "@postman/form-data": "~3.1.1", "@postman/tunnel-agent": "^0.6.3", @@ -1529,9 +1529,9 @@ } }, "regenerator-runtime": { - "version": "0.13.7", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" }, "request": { "version": "2.88.2", @@ -1569,6 +1569,17 @@ "jsprim": "^1.2.2", "sshpk": "^1.7.0" } + }, + "jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + } } } }, @@ -2187,9 +2198,9 @@ } }, "wuzzy": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/wuzzy/-/wuzzy-0.1.6.tgz", - "integrity": "sha512-x1lDcj0VvzJ1ygDpd9LWMnQVei6gEkUbCcZUG8TPnXhlPbaQWQa32ab/6xbm/samxJ2T3Y2+P3xHeeQIAcEvqQ==", + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/wuzzy/-/wuzzy-0.1.8.tgz", + "integrity": "sha512-FUzKQepFSTnANsDYwxpIzGJ/dIJaqxuMre6tzzbvWwFAiUHPsI1nVQVCLK4Xqr67KO7oYAK0kaCcI/+WYj/7JA==", "requires": { "lodash": "^4.17.15" } @@ -2231,9 +2242,9 @@ } }, "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-14.0.0.tgz", + "integrity": "sha512-zn/Mnx+tbFjkCFUodEpjXckNS65NfpB5oyqOkDDEG/8uxlfLZJu2IoBLQFjukUkn9rBbGkVYNzrDh6qy4NUd3g==", "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" From 049f88def984188b9ff49b169d3a27595f099936 Mon Sep 17 00:00:00 2001 From: hannah98 Date: Thu, 30 Dec 2021 20:19:48 +0000 Subject: [PATCH 027/124] Added TAG_SEPARATORS option to supply a regex of characters to use when splitting tags --- archivebox/config.py | 1 + archivebox/config_stubs.py | 1 + archivebox/index/sql.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 0551de2e..6eba098f 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -79,6 +79,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'URL_BLACKLIST': {'type': str, 'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$'}, # to avoid downloading code assets as their own pages 'URL_WHITELIST': {'type': str, 'default': None}, 'ENFORCE_ATOMIC_WRITES': {'type': bool, 'default': True}, + 'TAG_SEPARATORS': {'type': str, 'default': '[,]'}, }, 'SERVER_CONFIG': { diff --git a/archivebox/config_stubs.py b/archivebox/config_stubs.py index f9c22a0c..432b86d8 100644 --- a/archivebox/config_stubs.py +++ b/archivebox/config_stubs.py @@ -98,6 +98,7 @@ class ConfigDict(BaseConfig, total=False): WGET_ARGS: List[str] CURL_ARGS: List[str] GIT_ARGS: List[str] + TAG_SEPARATORS: str ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue] diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 661436cf..66402af7 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -1,5 +1,7 @@ __package__ = 'archivebox.index' +import re + from io import StringIO from pathlib import Path from typing import List, Tuple, Iterator @@ -8,7 +10,10 @@ from django.db import transaction from .schema import Link from ..util import enforce_types, parse_date -from ..config import OUTPUT_DIR +from ..config import ( + OUTPUT_DIR, + TAG_SEPARATORS, +) ### Main Links Index @@ -35,7 +40,7 @@ def write_link_to_sql_index(link: Link): info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} tag_list = list(dict.fromkeys( - tag.strip() for tag in (link.tags or '').split(',') + tag.strip() for tag in re.split(TAG_SEPARATORS, link.tags or '') )) info.pop('tags') @@ -107,7 +112,7 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: snap.title = link.title tag_list = list(dict.fromkeys( - tag.strip() for tag in (link.tags or '').split(',') + tag.strip() for tag in re.split(TAG_SEPARATORS, link.tags or '') )) snap.save() From fc3d2bb4dc01da73bb9fe068f3bc5e741083430d Mon Sep 17 00:00:00 2001 From: hannah98 Date: Thu, 6 Jan 2022 14:14:41 +0000 Subject: [PATCH 028/124] rename TAG_SEPARATORS to TAG_SEPARATOR_PATTERN --- archivebox/config.py | 2 +- archivebox/config_stubs.py | 2 +- archivebox/index/sql.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 6eba098f..89d009ce 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -79,7 +79,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'URL_BLACKLIST': {'type': str, 'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$'}, # to avoid downloading code assets as their own pages 'URL_WHITELIST': {'type': str, 'default': None}, 'ENFORCE_ATOMIC_WRITES': {'type': bool, 'default': True}, - 'TAG_SEPARATORS': {'type': str, 'default': '[,]'}, + 'TAG_SEPARATOR_PATTERN': {'type': str, 'default': '[,]'}, }, 'SERVER_CONFIG': { diff --git a/archivebox/config_stubs.py b/archivebox/config_stubs.py index 432b86d8..ead541a5 100644 --- a/archivebox/config_stubs.py +++ b/archivebox/config_stubs.py @@ -98,7 +98,7 @@ class ConfigDict(BaseConfig, total=False): WGET_ARGS: List[str] CURL_ARGS: List[str] GIT_ARGS: List[str] - TAG_SEPARATORS: str + TAG_SEPARATOR_PATTERN: str ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue] diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 66402af7..420b9de6 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -12,7 +12,7 @@ from .schema import Link from ..util import enforce_types, parse_date from ..config import ( OUTPUT_DIR, - TAG_SEPARATORS, + TAG_SEPARATOR_PATTERN, ) @@ -40,7 +40,7 @@ def write_link_to_sql_index(link: Link): info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} tag_list = list(dict.fromkeys( - tag.strip() for tag in re.split(TAG_SEPARATORS, link.tags or '') + tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '') )) info.pop('tags') @@ -112,7 +112,7 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: snap.title = link.title tag_list = list(dict.fromkeys( - tag.strip() for tag in re.split(TAG_SEPARATORS, link.tags or '') + tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '') )) snap.save() From 666ab20df576cc841c57b00e0821e219e4b84e37 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 10 Jan 2022 20:42:09 -0500 Subject: [PATCH 029/124] Update archivebox/config.py --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index 89d009ce..c8882b9e 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -79,7 +79,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'URL_BLACKLIST': {'type': str, 'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$'}, # to avoid downloading code assets as their own pages 'URL_WHITELIST': {'type': str, 'default': None}, 'ENFORCE_ATOMIC_WRITES': {'type': bool, 'default': True}, - 'TAG_SEPARATOR_PATTERN': {'type': str, 'default': '[,]'}, + 'TAG_SEPARATOR_PATTERN': {'type': str, 'default': r'[,]'}, }, 'SERVER_CONFIG': { From 9231e7f091bfaf979afe32f1ba59b5826e99527e Mon Sep 17 00:00:00 2001 From: akhilleusuggo <52116577+akhilleusuggo@users.noreply.github.com> Date: Thu, 27 Jan 2022 19:24:59 +0100 Subject: [PATCH 030/124] typo ERROR: The Compose file './docker-compose.yml' is invalid because: services.pywb.volumes contains an invalid type, it should be an array --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e3bd8c05..324540aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,5 +87,5 @@ services: # ports: # - 8080:8080 # volumes: - # ./data:/archivebox - # ./data/wayback:/webarchive + # - ./data:/archivebox + # - ./data/wayback:/webarchive From 6dae0cc252095604b4c15b035845e83b000809c1 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 6 Feb 2022 15:39:44 -0500 Subject: [PATCH 031/124] Fix arch installation command The linked package is on AUR, typically installed with yay. "pacman install" is not a valid command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68b46d81..2f06c87d 100644 --- a/README.md +++ b/README.md @@ -276,7 +276,7 @@ See the pip-archive Arch pacman / FreeBSD pkg / Nix nix (Arch/FreeBSD/NixOS/more)
    -
  • Arch: pacman install archivebox (contributed by @imlonghao)
    • +
  • Arch: yay -S archivebox (contributed by @imlonghao)
  • FreeBSD: curl -sSL 'https://get.archivebox.io' | sh (uses pkg + pip3 under-the-hood)
  • Nix: nix-env --install archivebox (contributed by @siraben)
  • More: contribute another distribution...!
    • From de8e22efb78cf9d148f0497d903948fb29111eef Mon Sep 17 00:00:00 2001 From: papersnake Date: Tue, 8 Feb 2022 23:17:52 +0800 Subject: [PATCH 032/124] improve title extractor --- archivebox/extractors/__init__.py | 6 +++--- archivebox/extractors/readability.py | 22 +--------------------- archivebox/extractors/title.py | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index be5832e7..dc1b9692 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -42,7 +42,6 @@ from .headers import should_save_headers, save_headers def get_default_archive_methods(): return [ - ('title', should_save_title, save_title), ('favicon', should_save_favicon, save_favicon), ('headers', should_save_headers, save_headers), ('singlefile', should_save_singlefile, save_singlefile), @@ -50,7 +49,8 @@ def get_default_archive_methods(): ('screenshot', should_save_screenshot, save_screenshot), ('dom', should_save_dom, save_dom), ('wget', should_save_wget, save_wget), - ('readability', should_save_readability, save_readability), # keep readability below wget and singlefile, as it depends on them + ('title', should_save_title, save_title), # keep title and readability below wget and singlefile, as it depends on them + ('readability', should_save_readability, save_readability), ('mercury', should_save_mercury, save_mercury), ('git', should_save_git, save_git), ('media', should_save_media, save_media), @@ -182,7 +182,7 @@ def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=Fa except KeyboardInterrupt: log_archiving_paused(num_links, idx, link.timestamp) raise SystemExit(0) - except BaseException: # lgtm [py/catch-base-exception] + except BaseException: print() raise diff --git a/archivebox/extractors/readability.py b/archivebox/extractors/readability.py index 7e5ed592..43fccbb6 100644 --- a/archivebox/extractors/readability.py +++ b/archivebox/extractors/readability.py @@ -22,28 +22,8 @@ from ..config import ( READABILITY_VERSION, ) from ..logging_util import TimedProgress +from .title import get_html -@enforce_types -def get_html(link: Link, path: Path) -> str: - """ - Try to find wget, singlefile and then dom files. - If none is found, download the url again. - """ - canonical = link.canonical_outputs() - abs_path = path.absolute() - sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]] - document = None - for source in sources: - try: - with open(abs_path / source, "r", encoding="utf-8") as f: - document = f.read() - break - except (FileNotFoundError, TypeError): - continue - if document is None: - return download_url(link.url) - else: - return document @enforce_types def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool: diff --git a/archivebox/extractors/title.py b/archivebox/extractors/title.py index 272eebc8..19a78591 100644 --- a/archivebox/extractors/title.py +++ b/archivebox/extractors/title.py @@ -58,6 +58,27 @@ class TitleParser(HTMLParser): if tag.lower() == "title": self.inside_title_tag = False +@enforce_types +def get_html(link: Link, path: Path, timeout: int=TIMEOUT) -> str: + """ + Try to find wget, singlefile and then dom files. + If none is found, download the url again. + """ + canonical = link.canonical_outputs() + abs_path = path.absolute() + sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]] + document = None + for source in sources: + try: + with open(abs_path / source, "r", encoding="utf-8") as f: + document = f.read() + break + except (FileNotFoundError, TypeError): + continue + if document is None: + return download_url(link.url, timeout=timeout) + else: + return document @enforce_types def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool: @@ -90,7 +111,7 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) - status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') try: - html = download_url(link.url, timeout=timeout) + html = get_html(link, out_dir, timeout=timeout) try: # try using relatively strict html parser first parser = TitleParser() From 011bd104cb0b411b156b62ff686ca2e14c48afb6 Mon Sep 17 00:00:00 2001 From: prnake Date: Wed, 9 Feb 2022 10:48:51 +0800 Subject: [PATCH 033/124] remove unused import --- archivebox/extractors/readability.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/archivebox/extractors/readability.py b/archivebox/extractors/readability.py index 43fccbb6..a1689f95 100644 --- a/archivebox/extractors/readability.py +++ b/archivebox/extractors/readability.py @@ -10,9 +10,7 @@ from ..index.schema import Link, ArchiveResult, ArchiveError from ..system import run, atomic_write from ..util import ( enforce_types, - download_url, is_static_file, - ) from ..config import ( TIMEOUT, From 532aeca8a08ea6662df20a433c4ae55bf13bc335 Mon Sep 17 00:00:00 2001 From: Andrew Louis Date: Fri, 18 Feb 2022 13:03:11 -0500 Subject: [PATCH 034/124] Fix typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68b46d81..e454f4ec 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ archivebox help - `archivebox setup/init/config/status/manage` to administer your collection - `archivebox add/schedule/remove/update/list/shell/oneshot` to manage Snapshots in the archive -- `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats) +- `archivebox schedule` to pull in fresh URLs in regularly from [bookmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats) #### 🖥  Web UI Usage From 5e9d05483ef65cdc92424c6990c797481f0a02a3 Mon Sep 17 00:00:00 2001 From: Pellaeon Lin Date: Thu, 24 Feb 2022 17:01:00 +0800 Subject: [PATCH 035/124] Fix bin_version: set LANG=C when calling executables to avoid parsing localized output. --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index c8882b9e..743fc9a0 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -650,7 +650,7 @@ def bin_version(binary: Optional[str]) -> Optional[str]: return None try: - version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode() + version_str = run([abspath, "--version"], stdout=PIPE, env={'LANG': 'C'}).stdout.strip().decode() # take first 3 columns of first line of version info return ' '.join(version_str.split('\n')[0].strip().split()[:3]) except OSError: From 29ee73660bf6152bd9c68b7cd411d20589b27550 Mon Sep 17 00:00:00 2001 From: Fuqiao Xue Date: Thu, 3 Mar 2022 21:04:19 +0800 Subject: [PATCH 036/124] Fix broken URI fragment --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68b46d81..189bf260 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ ls ./archive/*/index.json # or browse directly via the filesyste - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats) - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA) -- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode) +- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode) - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!) - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)... From 5cd2b328c006db9e39b919a8d7ab01aaecee2fe9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 16 Mar 2022 14:12:18 -0700 Subject: [PATCH 037/124] Add example pihole docker-compose config for ad/tracker blocking during archiving --- docker-compose.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 324540aa..0654c9a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: # - SEARCH_BACKEND_ENGINE=sonic # uncomment these if you enable sonic below # - SEARCH_BACKEND_HOST_NAME=sonic # - SEARCH_BACKEND_PASSWORD=SecretPassword + # dns: # uncomment to use pihole below for ad/tracker blocking during archiving + # - pihole volumes: - ./data:/data # - ./archivebox:/app/archivebox # for developers working on archivebox @@ -39,7 +41,17 @@ services: # volumes: # - ./sonic.cfg:/etc/sonic.cfg:ro # - ./data/sonic:/var/lib/sonic/store - + + # To run pihole in order to block ad/tracker requests during archiving, uncomment this block and set up pihole using its admin interface + # pihole: + # image: pihole/pihole:latest + # ports: + # - 80/80 # uncomment to access the admin HTTP interface on http://localhost:80 + # environment: + # WEBPASSWORD: 'set a secure password here or it will be random' + # volumes: + # - ./data/pihole:/etc/pihole + # - ./data/dnsmasq:/etc/dnsmasq.d ### Optional Addons: tweak these examples as needed for your specific use case From a15350cc770a5b7e0c118b55fc5992255007fe9e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 16 Mar 2022 14:14:35 -0700 Subject: [PATCH 038/124] Update docker-compose.yml --- docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0654c9a4..4b4dad8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,9 @@ services: - ./data:/data # - ./archivebox:/app/archivebox # for developers working on archivebox + + ### Optional Addons: tweak these examples as needed for your specific use case + # To run the Sonic full-text search backend, first download the config file to sonic.cfg # curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/etc/sonic.cfg # after starting, backfill any existing Snapshots into the index: docker-compose run archivebox update --index-only @@ -53,10 +56,8 @@ services: # - ./data/pihole:/etc/pihole # - ./data/dnsmasq:/etc/dnsmasq.d - ### Optional Addons: tweak these examples as needed for your specific use case - # Example: Run scheduled imports in a docker instead of using cron on the - # host machine, add tasks and see more info with archivebox schedule --help + # host machine, add tasks and see more info with archivebox schedule --help # scheduler: # image: archivebox/archivebox:latest # command: schedule --foreground --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all' From 0ec5ba0595772c00bf8befdb3cefc82f075c811a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 16 Mar 2022 14:15:37 -0700 Subject: [PATCH 039/124] fix pihole admin port format in docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4b4dad8a..dcaed1a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,7 @@ services: # pihole: # image: pihole/pihole:latest # ports: - # - 80/80 # uncomment to access the admin HTTP interface on http://localhost:80 + # - 80:80 # uncomment to access the admin HTTP interface on http://localhost:80 # environment: # WEBPASSWORD: 'set a secure password here or it will be random' # volumes: From d26851387729863110591a04b4902ca313efbe1e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 22 Mar 2022 20:55:17 -0400 Subject: [PATCH 040/124] cleanup example configs --- docker-compose.yml | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dcaed1a1..6e05f302 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,9 +32,10 @@ services: ### Optional Addons: tweak these examples as needed for your specific use case - # To run the Sonic full-text search backend, first download the config file to sonic.cfg + ### Example: To run the Sonic full-text search backend, first download the config file to sonic.cfg # curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/etc/sonic.cfg # after starting, backfill any existing Snapshots into the index: docker-compose run archivebox update --index-only + # sonic: # image: valeriansaliou/sonic:v1.3.0 # expose: @@ -45,19 +46,24 @@ services: # - ./sonic.cfg:/etc/sonic.cfg:ro # - ./data/sonic:/var/lib/sonic/store - # To run pihole in order to block ad/tracker requests during archiving, uncomment this block and set up pihole using its admin interface - # pihole: - # image: pihole/pihole:latest - # ports: - # - 80:80 # uncomment to access the admin HTTP interface on http://localhost:80 - # environment: - # WEBPASSWORD: 'set a secure password here or it will be random' - # volumes: - # - ./data/pihole:/etc/pihole - # - ./data/dnsmasq:/etc/dnsmasq.d + + ### Example: To run pihole in order to block ad/tracker requests during archiving, + # uncomment this block and set up pihole using its admin interface + + # pihole: + # image: pihole/pihole:latest + # ports: + # - 80:80 # uncomment to access the admin HTTP interface on http://localhost:80 + # environment: + # WEBPASSWORD: 'set a secure password here or it will be random' + # volumes: + # - ./data/pihole:/etc/pihole + # - ./data/dnsmasq:/etc/dnsmasq.d + + + ### Example: Run scheduled imports in a docker instead of using cron on the + # host machine, add tasks and see more info with archivebox schedule --help - # Example: Run scheduled imports in a docker instead of using cron on the - # host machine, add tasks and see more info with archivebox schedule --help # scheduler: # image: archivebox/archivebox:latest # command: schedule --foreground --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all' @@ -67,7 +73,9 @@ services: # volumes: # - ./data:/data - # Example: Put Nginx in front of the ArchiveBox server for SSL termination + + ### Example: Put Nginx in front of the ArchiveBox server for SSL termination + # nginx: # image: nginx:alpine # ports: @@ -77,7 +85,9 @@ services: # - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf # - ./data:/var/www - # Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel + + ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel + # wireguard: # image: linuxserver/wireguard # network_mode: 'service:archivebox' @@ -91,7 +101,9 @@ services: # - /lib/modules:/lib/modules # - ./wireguard.conf:/config/wg0.conf:ro - # Example: Run PYWB in parallel and auto-import WARCs from ArchiveBox + + ### Example: Run PYWB in parallel and auto-import WARCs from ArchiveBox + # pywb: # image: webrecorder/pywb:latest # entrypoint: /bin/sh 'wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback --proxy;' From dfef21a814f3271e38a00f2ed9711d94cb9ce02f Mon Sep 17 00:00:00 2001 From: Mike Kusold Date: Tue, 5 Apr 2022 20:42:29 -0600 Subject: [PATCH 041/124] Update pywb entrypoint wb-manager first needs you to init the collection. If you attempt to init a collection that already exists, it exits with a `2`. Also, `wayback --proxy` requires an argument, so I remove the proxy argument. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6e05f302..644bef9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,7 +106,7 @@ services: # pywb: # image: webrecorder/pywb:latest - # entrypoint: /bin/sh 'wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback --proxy;' + # entrypoint: /bin/sh '(wb-manager init default || test $? -eq 2) && wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback;' # environment: # - INIT_COLLECTION=archivebox # ports: From 4febbacef6bc9b8903a6afa3b6b531e44124fb90 Mon Sep 17 00:00:00 2001 From: Mike Kusold Date: Tue, 5 Apr 2022 20:58:40 -0600 Subject: [PATCH 042/124] Fix entrypoint docker-compose requires `$$` for env vars that are supposed to be executed inside the container. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 644bef9a..66462310 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,7 +106,7 @@ services: # pywb: # image: webrecorder/pywb:latest - # entrypoint: /bin/sh '(wb-manager init default || test $? -eq 2) && wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback;' + # entrypoint: /bin/sh -c '(wb-manager init default || test $$? -eq 2) && wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback;' # environment: # - INIT_COLLECTION=archivebox # ports: From 183253e9839bfb95baa75bc86e80aeb4fae05130 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 12 Apr 2022 18:24:24 -0400 Subject: [PATCH 043/124] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5628abe2..038cf7d1 100644 --- a/README.md +++ b/README.md @@ -345,7 +345,7 @@ For more discussion on managed and paid hosting options see here: below for more usage examples using the C See below for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
    -See setup.sh for the source code of the auto-install script. +See setup.sh for the source code of the auto-install script.
    +See "Against curl | sh as an install method" blog post for my thoughts on the shortcomings of this install method.

From 9e74676af340eeeec6286aceb584aec6aaa1bb98 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 12 Apr 2022 20:24:44 -0400 Subject: [PATCH 045/124] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a36d8d47..faf046d0 100644 --- a/README.md +++ b/README.md @@ -931,6 +931,8 @@ archivebox config --set DEBUG=True archivebox server --debug ... ``` +https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running + #### Install and run a specific GitHub branch @@ -985,7 +987,8 @@ cd path/to/test/data/ archivebox shell archivebox manage dbshell ``` -(uses `pytest -s`) +(uses `pytest -s`) +https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running From c63822a5e54a75e48411c21bf75e9472d1a384d3 Mon Sep 17 00:00:00 2001 From: Ross Date: Tue, 19 Apr 2022 22:25:49 +0100 Subject: [PATCH 046/124] Fix missing input redirection in a hint text --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index 65588ef7..ed1df69b 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -427,7 +427,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path= print(' archivebox server # then visit http://127.0.0.1:8000') print() print(' To add new links, you can run:') - print(" archivebox add ~/some/path/or/url/to/list_of_links.txt") + print(" archivebox add < ~/some/path/to/list_of_links.txt") print() print(' For more usage and examples, run:') print(' archivebox help') From eb81d41f842c4f345d9899b1ad4c65d8e1e1e10a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 21 Apr 2022 07:08:40 -0700 Subject: [PATCH 047/124] bump Dockerfile base image version and install yt-dlp --- Dockerfile | 8 +++++--- archivebox/config.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06111f1c..2ee6cbbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ # docker run -v "$PWD/data":/data -it archivebox manage createsuperuser # docker run -v "$PWD/data":/data -p 8000:8000 archivebox server -FROM python:3.9-slim-buster +FROM python:3.10-slim-bullseye LABEL name="archivebox" \ maintainer="Nick Sweeting " \ @@ -48,11 +48,12 @@ RUN apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ wget curl chromium git ffmpeg youtube-dl ripgrep \ fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \ + && ln -s /usr/bin/chromium /usr/bin/chromium-browser \ && rm -rf /var/lib/apt/lists/* # Install Node environment RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ - && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \ + && echo 'deb https://deb.nodesource.com/node_17.x buster main' >> /etc/apt/sources.list \ && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ nodejs \ @@ -109,7 +110,8 @@ ENV IN_DOCKER=True \ USE_READABILITY=True \ READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \ USE_MERCURY=True \ - MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser" + MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser" \ + YOUTUBEDL_BINARY="yt-dlp" # Print version for nice docker finish summary # RUN archivebox version diff --git a/archivebox/config.py b/archivebox/config.py index 743fc9a0..b4bcb7fc 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -120,9 +120,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'CHECK_SSL_VALIDITY': {'type': bool, 'default': True}, 'MEDIA_MAX_SIZE': {'type': str, 'default': '750m'}, - 'CURL_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'}, - 'WGET_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'}, - 'CHROME_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'}, + 'CURL_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'}, + 'WGET_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'}, + 'CHROME_USER_AGENT': {'type': str, 'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'}, 'COOKIES_FILE': {'type': str, 'default': None}, 'CHROME_USER_DATA_DIR': {'type': str, 'default': None}, From 57df65f28fe72677d21e29ce6c9e9d7bcd0ba918 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 21 Apr 2022 07:09:17 -0700 Subject: [PATCH 048/124] use yt-dlp for media archiving instead of youtube-dl --- Dockerfile | 5 +++-- archivebox/config.py | 3 ++- archivebox/extractors/media.py | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ee6cbbb..4a63fd34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,8 @@ RUN apt-get update -qq \ build-essential python-dev python3-dev \ && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \ && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \ - && pip install --quiet -r /tmp/requirements.txt \ + && pip install -r /tmp/requirements.txt \ + && pip install --upgrade youtube-dl yt-dlp \ && apt-get purge -y build-essential python-dev python3-dev \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* @@ -104,7 +105,7 @@ RUN pip install -e . WORKDIR "$DATA_DIR" ENV IN_DOCKER=True \ CHROME_SANDBOX=False \ - CHROME_BINARY="chromium" \ + CHROME_BINARY="/usr/bin/chromium-browser" \ USE_SINGLEFILE=True \ SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \ USE_READABILITY=True \ diff --git a/archivebox/config.py b/archivebox/config.py index b4bcb7fc..8abe7f00 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -142,6 +142,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { '--yes-playlist', '--continue', '--ignore-errors', + '--no-abort-on-error', '--geo-bypass', '--add-metadata', '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']), @@ -929,7 +930,7 @@ def get_chrome_info(config: ConfigDict) -> ConfigValue: 'TIMEOUT': config['TIMEOUT'], 'RESOLUTION': config['RESOLUTION'], 'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'], - 'CHROME_BINARY': config['CHROME_BINARY'], + 'CHROME_BINARY': bin_path(config['CHROME_BINARY']), 'CHROME_HEADLESS': config['CHROME_HEADLESS'], 'CHROME_SANDBOX': config['CHROME_SANDBOX'], 'CHROME_USER_AGENT': config['CHROME_USER_AGENT'], diff --git a/archivebox/extractors/media.py b/archivebox/extractors/media.py index e41a4002..1b093e8a 100644 --- a/archivebox/extractors/media.py +++ b/archivebox/extractors/media.py @@ -43,6 +43,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME YOUTUBEDL_BINARY, *YOUTUBEDL_ARGS, *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']), + # TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR} link.url, ] status = 'succeeded' From 2b35d14c0fae777767c187a152f3ac22a6f8775f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 21 Apr 2022 07:11:20 -0700 Subject: [PATCH 049/124] update npm package versions --- package-lock.json | 2956 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 2936 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2235c33e..e7f3feab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,8 +1,2914 @@ { "name": "archivebox", "version": "0.6.3", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "archivebox", + "version": "0.6.3", + "license": "MIT", + "dependencies": { + "@postlight/mercury-parser": "^2.2.1", + "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git", + "single-file": "git+https://github.com/gildas-lormeau/SingleFile.git" + } + }, + "node_modules/@babel/runtime-corejs2": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.16.5.tgz", + "integrity": "sha512-GHejyoK+JQqna9rUTkybaGahZOOM+EDcUsbWaLye1g4ZOQMotrMiY8VTBbX1gycZTMNG/YiWTo4WTABvyPUgOg==", + "dependencies": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@mozilla/readability": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.4.1.tgz", + "integrity": "sha512-yar/f0w0fRUVM895s6yd5Z2oIxjG/6c3ROB/uQboSOBaDlri/nqI4aKtdqrldWciTLcdpjB2Z6MiVF2Bl9b8LA==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@postlight/ci-failed-test-reporter": { + "version": "1.0.26", + "resolved": "https://registry.npmjs.org/@postlight/ci-failed-test-reporter/-/ci-failed-test-reporter-1.0.26.tgz", + "integrity": "sha512-xfXzxyOiKhco7Gx2OLTe9b66b0dFJw0elg94KGHoQXf5F8JqqFvdo35J8wayGOor64CSMvn+4Bjlu2NKV+yTGA==", + "dependencies": { + "dotenv": "^6.2.0", + "node-fetch": "^2.3.0" + }, + "bin": { + "ciftr": "cli.js" + } + }, + "node_modules/@postlight/mercury-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@postlight/mercury-parser/-/mercury-parser-2.2.1.tgz", + "integrity": "sha512-WsXHcVfbwlAb0HO1nZ3tS2ipNvLeLk9frJj3y5hn/NAeQfEJeRm9AtMrxpOJ11X8L4he+bRyjr1bpIPvQPWvCA==", + "bundleDependencies": [ + "jquery", + "moment-timezone", + "browser-request" + ], + "dependencies": { + "@babel/runtime-corejs2": "^7.2.0", + "@postlight/ci-failed-test-reporter": "^1.0", + "browser-request": "github:postlight/browser-request#feat-add-headers-to-response", + "cheerio": "^0.22.0", + "difflib": "github:postlight/difflib.js", + "ellipsize": "0.1.0", + "iconv-lite": "0.5.0", + "jquery": "^3.4.1", + "moment": "^2.23.0", + "moment-parseformat": "3.0.0", + "moment-timezone": "0.5.26", + "postman-request": "^2.88.1-postman.7.1", + "request-promise": "^4.2.2", + "string-direction": "^0.1.2", + "turndown": "^5.0.3", + "url": "^0.11.0", + "valid-url": "^1.0.9", + "wuzzy": "^0.1.4", + "yargs-parser": "^14.0.0" + }, + "bin": { + "mercury-parser": "cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@postlight/mercury-parser/node_modules/browser-request": { + "version": "0.3.2", + "engines": [ + "node" + ], + "inBundle": true, + "dependencies": { + "http-headers": "^3.0.1" + } + }, + "node_modules/@postlight/mercury-parser/node_modules/http-headers": { + "version": "3.0.2", + "inBundle": true, + "license": "MIT", + "dependencies": { + "next-line": "^1.1.0" + } + }, + "node_modules/@postlight/mercury-parser/node_modules/jquery": { + "version": "3.4.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/@postlight/mercury-parser/node_modules/moment": { + "version": "2.23.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@postlight/mercury-parser/node_modules/moment-timezone": { + "version": "0.5.26", + "inBundle": true, + "license": "MIT", + "dependencies": { + "moment": ">= 2.9.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@postlight/mercury-parser/node_modules/next-line": { + "version": "1.1.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/@postman/form-data": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@postman/form-data/-/form-data-3.1.1.tgz", + "integrity": "sha512-vjh8Q2a8S6UCm/KKs31XFJqEEgmbjBmpPNVV2eVav6905wyFAwaUOBGA1NPBI4ERH9MMZc6w0umFgM6WbEPMdg==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@postman/tunnel-agent": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.3.tgz", + "integrity": "sha512-k57fzmAZ2PJGxfOA4SGR05ejorHbVAa/84Hxh/2nAztjNXc4ZjOm9NUIk6/Z6LCrBvJZqjRZbN8e/nROVUPVdg==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/node": { + "version": "17.0.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.4.tgz", + "integrity": "sha512-6xwbrW4JJiJLgF+zNypN5wr2ykM9/jHcL7rQ8fZe2vuftggjzZeRSM4OwRc6Xk8qWjwJ99qVHo/JgOGmomWRog==", + "optional": true + }, + "node_modules/@types/yauzl": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz", + "integrity": "sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/abab": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", + "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==" + }, + "node_modules/acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-globals": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz", + "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==", + "dependencies": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, + "node_modules/acorn-globals/node_modules/acorn": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", + "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "engines": { + "node": "*" + } + }, + "node_modules/aws4": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bluebird": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", + "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=" + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/brotli": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.2.tgz", + "integrity": "sha1-UlqcrU/LqWR119OI9q7LE+7VL0Y=", + "dependencies": { + "base64-js": "^1.1.2" + } + }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" + }, + "node_modules/browser-request": { + "version": "0.3.2", + "resolved": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", + "integrity": "sha512-TOvTWJ0BrWcB8Ach1AvdSBuczm2fsJdBlmo8D4N8fei7xfboW9VEk67zfriCiBo3/19Xe1waSstCEcLFUeBCjA==", + "engines": [ + "node" + ], + "dependencies": { + "http-headers": "^3.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", + "engines": { + "node": "*" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "node_modules/cheerio": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", + "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=", + "dependencies": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash.assignin": "^4.0.9", + "lodash.bind": "^4.1.4", + "lodash.defaults": "^4.0.1", + "lodash.filter": "^4.4.0", + "lodash.flatten": "^4.2.0", + "lodash.foreach": "^4.3.0", + "lodash.map": "^4.4.0", + "lodash.merge": "^4.4.0", + "lodash.pick": "^4.2.1", + "lodash.reduce": "^4.4.0", + "lodash.reject": "^4.4.0", + "lodash.some": "^4.4.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3.4 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Please, upgrade your dependencies to the actual version of core-js.", + "hasInstallScript": true + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "node_modules/css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "dependencies": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "node_modules/css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "engines": { + "node": "*" + } + }, + "node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + }, + "node_modules/cssstyle": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.4.0.tgz", + "integrity": "sha512-GBrLZYZ4X4x6/QEoBnIrqb8B/f5l4+8me2dkom/j1Gtbxy0kBv6OGzKuAsGM75bkGwGAFkt56Iwg28S3XTZgSA==", + "dependencies": { + "cssom": "0.3.x" + } + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dependencies": { + "assert-plus": "^1.0.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "dependencies": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decimal.js": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" + }, + "node_modules/deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.818844", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.818844.tgz", + "integrity": "sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg==" + }, + "node_modules/difflib": { + "version": "0.2.6", + "resolved": "git+ssh://git@github.com/postlight/difflib.js.git#32e8e38c7fcd935241b9baab71bb432fd9b166ed", + "integrity": "sha512-uFNs7czGYLWdMP22WQhD/vlFen/CuKzC+KiajNCj+ik2Ah/I9i2AFyMWkBjFgbVFGhv95kBHOtx7tgF6IVngqA==", + "dependencies": { + "heap": ">= 0.2.0" + } + }, + "node_modules/dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dependencies": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "node_modules/domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "node_modules/domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "dependencies": { + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dependencies": { + "domelementtype": "1" + } + }, + "node_modules/dompurify": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.4.tgz", + "integrity": "sha512-6BVcgOAVFXjI0JTjEvZy901Rghm+7fDQOrNIcxB4+gdhj6Kwp6T9VBhBY/AbagKHJocRkDYGd6wvI+p4/10xtQ==" + }, + "node_modules/domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/dotenv": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", + "integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/ellipsize": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/ellipsize/-/ellipsize-0.1.0.tgz", + "integrity": "sha1-nUNoLUS5GtFuvYQmisEDFwplU/g=" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "engines": [ + "node >=0.6.0" + ] + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/file-url": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/file-url/-/file-url-3.0.0.tgz", + "integrity": "sha512-g872QGsHexznxkIAdK8UiZRe7SkE6kvylShU4Nsj8NvfvZag7S0QuQ4IgvPDkk75HxgjIVDwycFTDAgIiO4nDA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "engines": { + "node": "*" + } + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "engines": { + "node": ">=4" + } + }, + "node_modules/har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", + "dependencies": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, + "node_modules/html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "dependencies": { + "whatwg-encoding": "^1.0.1" + } + }, + "node_modules/htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dependencies": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, + "node_modules/http-headers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-headers/-/http-headers-3.0.2.tgz", + "integrity": "sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==", + "dependencies": { + "next-line": "^1.1.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-signature": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz", + "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==", + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^2.0.2", + "sshpk": "^1.14.1" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/iconv-lite": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz", + "integrity": "sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "node_modules/jsdom": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", + "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", + "dependencies": { + "abab": "^2.0.0", + "acorn": "^5.5.3", + "acorn-globals": "^4.1.0", + "array-equal": "^1.0.0", + "cssom": ">= 0.3.2 < 0.4.0", + "cssstyle": "^1.0.0", + "data-urls": "^1.0.0", + "domexception": "^1.0.1", + "escodegen": "^1.9.1", + "html-encoding-sniffer": "^1.0.2", + "left-pad": "^1.3.0", + "nwsapi": "^2.0.7", + "parse5": "4.0.0", + "pn": "^1.1.0", + "request": "^2.87.0", + "request-promise-native": "^1.0.5", + "sax": "^1.2.4", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.3.4", + "w3c-hr-time": "^1.0.1", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.3", + "whatwg-mimetype": "^2.1.0", + "whatwg-url": "^6.4.1", + "ws": "^5.2.0", + "xml-name-validator": "^3.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "node_modules/jsprim": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", + "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==", + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + } + }, + "node_modules/jszip": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.1.tgz", + "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==", + "deprecated": "use String.prototype.padStart()" + }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.assignin": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignin/-/lodash.assignin-4.2.0.tgz", + "integrity": "sha1-uo31+4QesKPoBEIysOJjqNxqKKI=" + }, + "node_modules/lodash.bind": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz", + "integrity": "sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU=" + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "node_modules/lodash.filter": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.filter/-/lodash.filter-4.6.0.tgz", + "integrity": "sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, + "node_modules/lodash.foreach": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz", + "integrity": "sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=" + }, + "node_modules/lodash.map": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", + "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "node_modules/lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=" + }, + "node_modules/lodash.reduce": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz", + "integrity": "sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=" + }, + "node_modules/lodash.reject": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reject/-/lodash.reject-4.6.0.tgz", + "integrity": "sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=" + }, + "node_modules/lodash.some": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz", + "integrity": "sha1-G7nzFO9ri63tE7VJFpsqlF62jk0=" + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, + "node_modules/mime-db": { + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", + "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.30", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", + "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "dependencies": { + "mime-db": "1.47.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/moment-parseformat": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/moment-parseformat/-/moment-parseformat-3.0.0.tgz", + "integrity": "sha512-dVgXe6b6DLnv4CHG7a1zUe5mSXaIZ3c6lSHm/EKeVeQI2/4pwe0VRde8OyoCE1Ro2lKT5P6uT9JElF7KDLV+jw==" + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/next-line": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", + "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dependencies": { + "boolbase": "~1.0.0" + } + }, + "node_modules/nwsapi": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", + "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==" + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "engines": { + "node": "*" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/parse5": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", + "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, + "node_modules/postman-request": { + "version": "2.88.1-postman.30", + "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.30.tgz", + "integrity": "sha512-zsGvs8OgNeno1Q44zTgGP2IL7kCqUy4DAtl8/ms0AQpqkIoysrxzR/Zg4kM1Kz8/duBvwxt8NN717wB7SMNm6w==", + "dependencies": { + "@postman/form-data": "~3.1.1", + "@postman/tunnel-agent": "^0.6.3", + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "brotli": "~1.3.2", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "har-validator": "~5.1.3", + "http-signature": "~1.3.1", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "stream-length": "^1.0.2", + "tough-cookie": "~2.5.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/puppeteer-core": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-5.5.0.tgz", + "integrity": "sha512-tlA+1n+ziW/Db03hVV+bAecDKse8ihFRXYiEypBe9IlLRvOCzYFG6qrCMBYK34HO/Q/Ecjc+tvkHRAfLVH+NgQ==", + "dependencies": { + "debug": "^4.1.0", + "devtools-protocol": "0.0.818844", + "extract-zip": "^2.0.0", + "https-proxy-agent": "^4.0.0", + "node-fetch": "^2.6.1", + "pkg-dir": "^4.2.0", + "progress": "^2.0.1", + "proxy-from-env": "^1.0.0", + "rimraf": "^3.0.2", + "tar-fs": "^2.0.0", + "unbzip2-stream": "^1.3.3", + "ws": "^7.2.3" + }, + "engines": { + "node": ">=10.18.1" + } + }, + "node_modules/puppeteer-core/node_modules/agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "dependencies": { + "agent-base": "5", + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/readability-extractor": { + "version": "0.0.2", + "resolved": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", + "integrity": "sha512-B+oZuG4FwPYg5hxEafuhrwNOS8uiv/gYKlLKbIaeXXHlyznARYOqHpkHumiLMU6vkbZ3VAC7WucnWwh5jVOaBQ==", + "license": "MIT", + "dependencies": { + "@mozilla/readability": "^0.4.1", + "dompurify": "^2.2.7", + "jsdom": "^16.5.2" + }, + "bin": { + "readability-extractor": "readability-extractor" + } + }, + "node_modules/readability-extractor/node_modules/acorn": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/readability-extractor/node_modules/acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "dependencies": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + } + }, + "node_modules/readability-extractor/node_modules/acorn-globals/node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/readability-extractor/node_modules/acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/readability-extractor/node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + }, + "node_modules/readability-extractor/node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dependencies": { + "cssom": "~0.3.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/readability-extractor/node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + }, + "node_modules/readability-extractor/node_modules/data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dependencies": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/readability-extractor/node_modules/domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "dependencies": { + "webidl-conversions": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/readability-extractor/node_modules/domexception/node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/readability-extractor/node_modules/escodegen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", + "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/readability-extractor/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/readability-extractor/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readability-extractor/node_modules/html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "dependencies": { + "whatwg-encoding": "^1.0.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/readability-extractor/node_modules/jsdom": { + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "dependencies": { + "abab": "^2.0.5", + "acorn": "^8.2.4", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "form-data": "^3.0.0", + "html-encoding-sniffer": "^2.0.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.5.0", + "ws": "^7.4.6", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/readability-extractor/node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" + }, + "node_modules/readability-extractor/node_modules/tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/readability-extractor/node_modules/tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/readability-extractor/node_modules/webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "engines": { + "node": ">=10.4" + } + }, + "node_modules/readability-extractor/node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/readability-extractor/node_modules/ws": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/request-promise": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.6.tgz", + "integrity": "sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==", + "deprecated": "request-promise has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", + "dependencies": { + "bluebird": "^3.5.0", + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "dependencies": { + "lodash": "^4.17.19" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-native": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", + "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", + "deprecated": "request-promise-native has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", + "dependencies": { + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.12.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise/node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "node_modules/request/node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + }, + "engines": { + "node": ">=0.8", + "npm": ">=1.3.7" + } + }, + "node_modules/request/node_modules/jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/selenium-webdriver": { + "version": "4.0.0-alpha.7", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.0.0-alpha.7.tgz", + "integrity": "sha512-D4qnTsyTr91jT8f7MfN+OwY0IlU5+5FmlO5xlgRUV6hDEV8JyYx2NerdTEqDDkNq7RZDYc4VoPALk8l578RBHw==", + "dependencies": { + "jszip": "^3.2.2", + "rimraf": "^2.7.1", + "tmp": "0.0.30" + }, + "engines": { + "node": ">= 10.15.0" + } + }, + "node_modules/selenium-webdriver/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/single-file": { + "version": "0.3.16", + "resolved": "git+ssh://git@github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", + "integrity": "sha512-NwwtloJdZlPG2zKrTAlcPRowIcwQx+1U39wcyPnfsbpVZKi6FAyLpo+CV0/xVJmvdOO0DSTvimuZtnc8/gJfTw==", + "license": "AGPL-3.0-or-later", + "dependencies": { + "file-url": "^3.0.0", + "iconv-lite": "^0.6.2", + "jsdom": "^16.4.0", + "puppeteer-core": "^5.3.0", + "selenium-webdriver": "4.0.0-alpha.7", + "strong-data-uri": "^1.0.6", + "yargs": "^16.2.0" + }, + "bin": { + "single-file": "cli/single-file" + } + }, + "node_modules/single-file/node_modules/acorn": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", + "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/single-file/node_modules/acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "dependencies": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + } + }, + "node_modules/single-file/node_modules/acorn-globals/node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/single-file/node_modules/acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/single-file/node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + }, + "node_modules/single-file/node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dependencies": { + "cssom": "~0.3.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/single-file/node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + }, + "node_modules/single-file/node_modules/data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dependencies": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/single-file/node_modules/domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "dependencies": { + "webidl-conversions": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/single-file/node_modules/domexception/node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/single-file/node_modules/escodegen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", + "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/single-file/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/single-file/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/single-file/node_modules/html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "dependencies": { + "whatwg-encoding": "^1.0.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/single-file/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/single-file/node_modules/jsdom": { + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "dependencies": { + "abab": "^2.0.5", + "acorn": "^8.2.4", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "form-data": "^3.0.0", + "html-encoding-sniffer": "^2.0.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.5.0", + "ws": "^7.4.6", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/single-file/node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" + }, + "node_modules/single-file/node_modules/tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/single-file/node_modules/tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/single-file/node_modules/webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "engines": { + "node": ">=10.4" + } + }, + "node_modules/single-file/node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/single-file/node_modules/ws": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "bin": { + "sshpk-conv": "bin/sshpk-conv", + "sshpk-sign": "bin/sshpk-sign", + "sshpk-verify": "bin/sshpk-verify" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stream-length": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stream-length/-/stream-length-1.0.2.tgz", + "integrity": "sha1-gnfzy+5JpNqrz9tOL0qbXp8snwA=", + "dependencies": { + "bluebird": "^2.6.2" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-direction": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/string-direction/-/string-direction-0.1.2.tgz", + "integrity": "sha1-PYRT5ydKLkShQrPchEnftk2a3jo=" + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strong-data-uri": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/strong-data-uri/-/strong-data-uri-1.0.6.tgz", + "integrity": "sha512-zhzBZev0uhT2IrFUerenXhfaE0vFUYwAZsnG0gIKGpfM/Gi6jOUQ3cmcvyTsXeDLIPiTubHESeO7EbD6FoPmzw==", + "dependencies": { + "truncate": "^2.0.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "node_modules/tmp": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.30.tgz", + "integrity": "sha1-ckGdSovn1s51FI/YsyTlk6cRwu0=", + "dependencies": { + "os-tmpdir": "~1.0.1" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/truncate": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/truncate/-/truncate-2.1.0.tgz", + "integrity": "sha512-em3E3SUDONOjTBcZ36DTm3RvDded3IRU9rX32oHwwXNt3rJD5MVaFlJTQvs8tJoHRoeYP36OuQ1eL/Q7bNEWIQ==", + "engines": { + "node": "*" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/turndown": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-5.0.3.tgz", + "integrity": "sha512-popfGXEiedpq6F5saRIAThKxq/bbEPVFnsDnUdjaDGIre9f3/OL9Yi/yPbPcZ7RYUDpekghr666bBfZPrwNnhQ==", + "dependencies": { + "jsdom": "^11.9.0" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/valid-url": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", + "integrity": "sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA=" + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "dependencies": { + "browser-process-hrtime": "^1.0.0" + } + }, + "node_modules/w3c-xmlserializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", + "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "dependencies": { + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "node_modules/whatwg-url": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", + "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "node_modules/ws": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.3.tgz", + "integrity": "sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA==", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/wuzzy": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/wuzzy/-/wuzzy-0.1.8.tgz", + "integrity": "sha512-FUzKQepFSTnANsDYwxpIzGJ/dIJaqxuMre6tzzbvWwFAiUHPsI1nVQVCLK4Xqr67KO7oYAK0kaCcI/+WYj/7JA==", + "dependencies": { + "lodash": "^4.17.15" + } + }, + "node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-14.0.0.tgz", + "integrity": "sha512-zn/Mnx+tbFjkCFUodEpjXckNS65NfpB5oyqOkDDEG/8uxlfLZJu2IoBLQFjukUkn9rBbGkVYNzrDh6qy4NUd3g==", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + } + }, "dependencies": { "@babel/runtime-corejs2": { "version": "7.16.5", @@ -53,6 +2959,13 @@ "yargs-parser": "^14.0.0" }, "dependencies": { + "browser-request": { + "version": "0.3.2", + "bundled": true, + "requires": { + "http-headers": "^3.0.1" + } + }, "http-headers": { "version": "3.0.2", "bundled": true, @@ -281,8 +3194,8 @@ "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, "browser-request": { - "version": "github:postlight/browser-request#38faa5b85741aabfca61aa37d1ef044d68969ddf", - "from": "github:postlight/browser-request#feat-add-headers-to-response", + "version": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", + "from": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", "requires": { "http-headers": "^3.0.1" } @@ -478,8 +3391,8 @@ "integrity": "sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg==" }, "difflib": { - "version": "github:postlight/difflib.js#32e8e38c7fcd935241b9baab71bb432fd9b166ed", - "from": "github:postlight/difflib.js", + "version": "git+ssh://git@github.com/postlight/difflib.js.git#32e8e38c7fcd935241b9baab71bb432fd9b166ed", + "from": "difflib@github:postlight/difflib.js", "requires": { "heap": ">= 0.2.0" } @@ -1310,7 +4223,8 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "requires": {} } } }, @@ -1325,8 +4239,8 @@ "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, "readability-extractor": { - "version": "git+https://github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", - "from": "git+https://github.com/ArchiveBox/readability-extractor.git", + "version": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", + "from": "readability-extractor@git+https://github.com/ArchiveBox/readability-extractor.git", "requires": { "@mozilla/readability": "^0.4.1", "dompurify": "^2.2.7", @@ -1514,7 +4428,8 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "requires": {} } } }, @@ -1681,8 +4596,8 @@ "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" }, "single-file": { - "version": "git+https://github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", - "from": "git+https://github.com/gildas-lormeau/SingleFile.git", + "version": "git+ssh://git@github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", + "from": "single-file@git+https://github.com/gildas-lormeau/SingleFile.git", "requires": { "file-url": "^3.0.0", "iconv-lite": "^0.6.2", @@ -1882,7 +4797,8 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", + "requires": {} } } }, @@ -1921,6 +4837,14 @@ "bluebird": "^2.6.2" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, "string-direction": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/string-direction/-/string-direction-0.1.2.tgz", @@ -1936,14 +4860,6 @@ "strip-ansi": "^6.0.1" } }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", From 66f79fc041077f97056868b6cf2c8c89684c659d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:12:19 +0000 Subject: [PATCH 050/124] Bump node-fetch from 2.6.1 to 2.6.7 Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.1 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.1...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 113 ++++++++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/package-lock.json b/package-lock.json index e7f3feab..175b0c95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -387,17 +387,6 @@ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, - "node_modules/browser-request": { - "version": "0.3.2", - "resolved": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", - "integrity": "sha512-TOvTWJ0BrWcB8Ach1AvdSBuczm2fsJdBlmo8D4N8fei7xfboW9VEk67zfriCiBo3/19Xe1waSstCEcLFUeBCjA==", - "engines": [ - "node" - ], - "dependencies": { - "http-headers": "^3.0.1" - } - }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -988,14 +977,6 @@ "readable-stream": "^3.1.1" } }, - "node_modules/http-headers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/http-headers/-/http-headers-3.0.2.tgz", - "integrity": "sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==", - "dependencies": { - "next-line": "^1.1.0" - } - }, "node_modules/http-proxy-agent": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", @@ -1368,17 +1349,42 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/next-line": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", - "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" - }, "node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" } }, "node_modules/nth-check": { @@ -3193,13 +3199,6 @@ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, - "browser-request": { - "version": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", - "from": "git+ssh://git@github.com/postlight/browser-request.git#38faa5b85741aabfca61aa37d1ef044d68969ddf", - "requires": { - "http-headers": "^3.0.1" - } - }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -3392,6 +3391,7 @@ }, "difflib": { "version": "git+ssh://git@github.com/postlight/difflib.js.git#32e8e38c7fcd935241b9baab71bb432fd9b166ed", + "integrity": "sha512-uFNs7czGYLWdMP22WQhD/vlFen/CuKzC+KiajNCj+ik2Ah/I9i2AFyMWkBjFgbVFGhv95kBHOtx7tgF6IVngqA==", "from": "difflib@github:postlight/difflib.js", "requires": { "heap": ">= 0.2.0" @@ -3667,14 +3667,6 @@ "readable-stream": "^3.1.1" } }, - "http-headers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/http-headers/-/http-headers-3.0.2.tgz", - "integrity": "sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==", - "requires": { - "next-line": "^1.1.0" - } - }, "http-proxy-agent": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", @@ -4001,15 +3993,34 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "next-line": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", - "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" - }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } }, "nth-check": { "version": "1.0.2", @@ -4240,6 +4251,7 @@ }, "readability-extractor": { "version": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", + "integrity": "sha512-B+oZuG4FwPYg5hxEafuhrwNOS8uiv/gYKlLKbIaeXXHlyznARYOqHpkHumiLMU6vkbZ3VAC7WucnWwh5jVOaBQ==", "from": "readability-extractor@git+https://github.com/ArchiveBox/readability-extractor.git", "requires": { "@mozilla/readability": "^0.4.1", @@ -4597,6 +4609,7 @@ }, "single-file": { "version": "git+ssh://git@github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", + "integrity": "sha512-NwwtloJdZlPG2zKrTAlcPRowIcwQx+1U39wcyPnfsbpVZKi6FAyLpo+CV0/xVJmvdOO0DSTvimuZtnc8/gJfTw==", "from": "single-file@git+https://github.com/gildas-lormeau/SingleFile.git", "requires": { "file-url": "^3.0.0", From ed9ef588d970f604a2722fa1af1186b2db0eaf76 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 21 Apr 2022 07:29:27 -0700 Subject: [PATCH 051/124] add instructions for multi-arch build at top of Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4a63fd34..fe2070c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,10 @@ # docker run -v "$PWD/data":/data archivebox add 'https://example.com' # docker run -v "$PWD/data":/data -it archivebox manage createsuperuser # docker run -v "$PWD/data":/data -p 8000:8000 archivebox server +# Multi-arch build: +# docker buildx create --use +# docker buildx build . --platform=linux/amd64,linux/arm64,linux/386,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev + FROM python:3.10-slim-bullseye From c5ccb05d4a39700a16abfc8612c83f92767219ab Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 21 Apr 2022 07:35:34 -0700 Subject: [PATCH 052/124] update Dockerfile buildx instructions to exclude intel/386 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fe2070c2..7d422628 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ # docker run -v "$PWD/data":/data -p 8000:8000 archivebox server # Multi-arch build: # docker buildx create --use -# docker buildx build . --platform=linux/amd64,linux/arm64,linux/386,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev +# docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev FROM python:3.10-slim-bullseye From 6e6686387168eb6358f5927e8a338876e03a8c4f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 18:36:27 -0700 Subject: [PATCH 053/124] add max 5s writing delay for concurrent writers and flush WAL slower --- archivebox/config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index 8abe7f00..21e48ff2 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1169,10 +1169,17 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, # Enable WAL mode in sqlite3 from django.db import connection with connection.cursor() as cursor: + + # Set Journal mode to WAL to allow for multiple writers current_mode = cursor.execute("PRAGMA journal_mode") if current_mode != 'wal': cursor.execute("PRAGMA journal_mode=wal;") + # Set max blocking delay for concurrent writes and write sync mode + # https://litestream.io/tips/#busy-timeout + cursor.execute("PRAGMA busy_timeout = 5000;") + cursor.execute("PRAGMA synchronous = NORMAL;") + # Create cache table in DB if needed try: from django.core.cache import cache From acd53c854d68509ebea679ffab4ad67f8a9c9dc5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:07:42 -0700 Subject: [PATCH 054/124] handle new wallabag export format with newlines mid-tag attributes --- archivebox/parsers/wallabag_atom.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/archivebox/parsers/wallabag_atom.py b/archivebox/parsers/wallabag_atom.py index 32740097..3a39c54a 100644 --- a/archivebox/parsers/wallabag_atom.py +++ b/archivebox/parsers/wallabag_atom.py @@ -34,13 +34,19 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]: trailing_removed = entry.split('', 1)[0] leading_removed = trailing_removed.strip() - rows = leading_removed.split('\n') + splits_fixed = leading_removed.replace('"\n href="', '" href="') + rows = splits_fixed.split('\n') - def get_row(key): - return [r.strip() for r in rows if r.strip().startswith('<{}'.format(key))][0] + def get_row(prefix): + return [ + row.strip() + for row in rows + if row.strip().startswith('<{}'.format(prefix)) + ][0] title = str_between(get_row('title'), '<![CDATA[', ']]>').strip() - url = str_between(get_row('link rel="via"'), '', '') + url_inside_link = str_between(get_row('link rel="via"'), '', '') + url_inside_attr = str_between(get_row('link rel="via"'), 'href="', '"/>') ts_str = str_between(get_row('published'), '', '') time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") try: @@ -49,7 +55,7 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]: tags = None yield Link( - url=htmldecode(url), + url=htmldecode(url_inside_attr or url_inside_link), timestamp=str(time.timestamp()), title=htmldecode(title) or None, tags=tags or '', From 5ddffae83ea84a13168a6abe2e1cecd9a981b020 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:14:26 -0700 Subject: [PATCH 055/124] bump sonic version to support ARM --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 66462310..7e494e65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: # after starting, backfill any existing Snapshots into the index: docker-compose run archivebox update --index-only # sonic: - # image: valeriansaliou/sonic:v1.3.0 + # image: valeriansaliou/sonic:v1.3.1 # expose: # - 1491 # environment: From e83132c24e1245bb27840373b203d6870d57d378 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:24:05 -0700 Subject: [PATCH 056/124] bump js lockfile --- package-lock.json | 3018 +-------------------------------------------- 1 file changed, 58 insertions(+), 2960 deletions(-) diff --git a/package-lock.json b/package-lock.json index 175b0c95..fed13aaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,2920 +1,8 @@ { "name": "archivebox", "version": "0.6.3", - "lockfileVersion": 2, + "lockfileVersion": 1, "requires": true, - "packages": { - "": { - "name": "archivebox", - "version": "0.6.3", - "license": "MIT", - "dependencies": { - "@postlight/mercury-parser": "^2.2.1", - "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git", - "single-file": "git+https://github.com/gildas-lormeau/SingleFile.git" - } - }, - "node_modules/@babel/runtime-corejs2": { - "version": "7.16.5", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.16.5.tgz", - "integrity": "sha512-GHejyoK+JQqna9rUTkybaGahZOOM+EDcUsbWaLye1g4ZOQMotrMiY8VTBbX1gycZTMNG/YiWTo4WTABvyPUgOg==", - "dependencies": { - "core-js": "^2.6.5", - "regenerator-runtime": "^0.13.4" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@mozilla/readability": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.4.1.tgz", - "integrity": "sha512-yar/f0w0fRUVM895s6yd5Z2oIxjG/6c3ROB/uQboSOBaDlri/nqI4aKtdqrldWciTLcdpjB2Z6MiVF2Bl9b8LA==", - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@postlight/ci-failed-test-reporter": { - "version": "1.0.26", - "resolved": "https://registry.npmjs.org/@postlight/ci-failed-test-reporter/-/ci-failed-test-reporter-1.0.26.tgz", - "integrity": "sha512-xfXzxyOiKhco7Gx2OLTe9b66b0dFJw0elg94KGHoQXf5F8JqqFvdo35J8wayGOor64CSMvn+4Bjlu2NKV+yTGA==", - "dependencies": { - "dotenv": "^6.2.0", - "node-fetch": "^2.3.0" - }, - "bin": { - "ciftr": "cli.js" - } - }, - "node_modules/@postlight/mercury-parser": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@postlight/mercury-parser/-/mercury-parser-2.2.1.tgz", - "integrity": "sha512-WsXHcVfbwlAb0HO1nZ3tS2ipNvLeLk9frJj3y5hn/NAeQfEJeRm9AtMrxpOJ11X8L4he+bRyjr1bpIPvQPWvCA==", - "bundleDependencies": [ - "jquery", - "moment-timezone", - "browser-request" - ], - "dependencies": { - "@babel/runtime-corejs2": "^7.2.0", - "@postlight/ci-failed-test-reporter": "^1.0", - "browser-request": "github:postlight/browser-request#feat-add-headers-to-response", - "cheerio": "^0.22.0", - "difflib": "github:postlight/difflib.js", - "ellipsize": "0.1.0", - "iconv-lite": "0.5.0", - "jquery": "^3.4.1", - "moment": "^2.23.0", - "moment-parseformat": "3.0.0", - "moment-timezone": "0.5.26", - "postman-request": "^2.88.1-postman.7.1", - "request-promise": "^4.2.2", - "string-direction": "^0.1.2", - "turndown": "^5.0.3", - "url": "^0.11.0", - "valid-url": "^1.0.9", - "wuzzy": "^0.1.4", - "yargs-parser": "^14.0.0" - }, - "bin": { - "mercury-parser": "cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@postlight/mercury-parser/node_modules/browser-request": { - "version": "0.3.2", - "engines": [ - "node" - ], - "inBundle": true, - "dependencies": { - "http-headers": "^3.0.1" - } - }, - "node_modules/@postlight/mercury-parser/node_modules/http-headers": { - "version": "3.0.2", - "inBundle": true, - "license": "MIT", - "dependencies": { - "next-line": "^1.1.0" - } - }, - "node_modules/@postlight/mercury-parser/node_modules/jquery": { - "version": "3.4.1", - "inBundle": true, - "license": "MIT" - }, - "node_modules/@postlight/mercury-parser/node_modules/moment": { - "version": "2.23.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/@postlight/mercury-parser/node_modules/moment-timezone": { - "version": "0.5.26", - "inBundle": true, - "license": "MIT", - "dependencies": { - "moment": ">= 2.9.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@postlight/mercury-parser/node_modules/next-line": { - "version": "1.1.0", - "inBundle": true, - "license": "MIT" - }, - "node_modules/@postman/form-data": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@postman/form-data/-/form-data-3.1.1.tgz", - "integrity": "sha512-vjh8Q2a8S6UCm/KKs31XFJqEEgmbjBmpPNVV2eVav6905wyFAwaUOBGA1NPBI4ERH9MMZc6w0umFgM6WbEPMdg==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@postman/tunnel-agent": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/@postman/tunnel-agent/-/tunnel-agent-0.6.3.tgz", - "integrity": "sha512-k57fzmAZ2PJGxfOA4SGR05ejorHbVAa/84Hxh/2nAztjNXc4ZjOm9NUIk6/Z6LCrBvJZqjRZbN8e/nROVUPVdg==", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", - "engines": { - "node": ">= 6" - } - }, - "node_modules/@types/node": { - "version": "17.0.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.4.tgz", - "integrity": "sha512-6xwbrW4JJiJLgF+zNypN5wr2ykM9/jHcL7rQ8fZe2vuftggjzZeRSM4OwRc6Xk8qWjwJ99qVHo/JgOGmomWRog==", - "optional": true - }, - "node_modules/@types/yauzl": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz", - "integrity": "sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA==", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/abab": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", - "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==" - }, - "node_modules/acorn": { - "version": "5.7.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", - "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-globals": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz", - "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==", - "dependencies": { - "acorn": "^6.0.1", - "acorn-walk": "^6.0.1" - } - }, - "node_modules/acorn-globals/node_modules/acorn": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", - "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", - "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/array-equal": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", - "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" - }, - "node_modules/asn1": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", - "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/async-limiter": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", - "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "node_modules/aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "engines": { - "node": "*" - } - }, - "node_modules/aws4": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/bluebird": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", - "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=" - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/brotli": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.2.tgz", - "integrity": "sha1-UlqcrU/LqWR119OI9q7LE+7VL0Y=", - "dependencies": { - "base64-js": "^1.1.2" - } - }, - "node_modules/browser-process-hrtime": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", - "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", - "engines": { - "node": "*" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "engines": { - "node": ">=6" - } - }, - "node_modules/caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "node_modules/cheerio": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", - "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=", - "dependencies": { - "css-select": "~1.2.0", - "dom-serializer": "~0.1.0", - "entities": "~1.1.1", - "htmlparser2": "^3.9.1", - "lodash.assignin": "^4.0.9", - "lodash.bind": "^4.1.4", - "lodash.defaults": "^4.0.1", - "lodash.filter": "^4.4.0", - "lodash.flatten": "^4.2.0", - "lodash.foreach": "^4.3.0", - "lodash.map": "^4.4.0", - "lodash.merge": "^4.4.0", - "lodash.pick": "^4.2.1", - "lodash.reduce": "^4.4.0", - "lodash.reject": "^4.4.0", - "lodash.some": "^4.4.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "node_modules/core-js": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", - "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", - "deprecated": "core-js@<3.4 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Please, upgrade your dependencies to the actual version of core-js.", - "hasInstallScript": true - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "node_modules/css-select": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", - "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", - "dependencies": { - "boolbase": "~1.0.0", - "css-what": "2.1", - "domutils": "1.5.1", - "nth-check": "~1.0.1" - } - }, - "node_modules/css-what": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", - "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", - "engines": { - "node": "*" - } - }, - "node_modules/cssom": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" - }, - "node_modules/cssstyle": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.4.0.tgz", - "integrity": "sha512-GBrLZYZ4X4x6/QEoBnIrqb8B/f5l4+8me2dkom/j1Gtbxy0kBv6OGzKuAsGM75bkGwGAFkt56Iwg28S3XTZgSA==", - "dependencies": { - "cssom": "0.3.x" - } - }, - "node_modules/dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dependencies": { - "assert-plus": "^1.0.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/data-urls": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", - "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", - "dependencies": { - "abab": "^2.0.0", - "whatwg-mimetype": "^2.2.0", - "whatwg-url": "^7.0.0" - } - }, - "node_modules/data-urls/node_modules/whatwg-url": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", - "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", - "dependencies": { - "lodash.sortby": "^4.7.0", - "tr46": "^1.0.1", - "webidl-conversions": "^4.0.2" - } - }, - "node_modules/debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/decimal.js": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", - "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" - }, - "node_modules/deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/devtools-protocol": { - "version": "0.0.818844", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.818844.tgz", - "integrity": "sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg==" - }, - "node_modules/difflib": { - "version": "0.2.6", - "resolved": "git+ssh://git@github.com/postlight/difflib.js.git#32e8e38c7fcd935241b9baab71bb432fd9b166ed", - "integrity": "sha512-uFNs7czGYLWdMP22WQhD/vlFen/CuKzC+KiajNCj+ik2Ah/I9i2AFyMWkBjFgbVFGhv95kBHOtx7tgF6IVngqA==", - "dependencies": { - "heap": ">= 0.2.0" - } - }, - "node_modules/dom-serializer": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", - "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", - "dependencies": { - "domelementtype": "^1.3.0", - "entities": "^1.1.1" - } - }, - "node_modules/domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" - }, - "node_modules/domexception": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", - "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", - "dependencies": { - "webidl-conversions": "^4.0.2" - } - }, - "node_modules/domhandler": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", - "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", - "dependencies": { - "domelementtype": "1" - } - }, - "node_modules/dompurify": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.4.tgz", - "integrity": "sha512-6BVcgOAVFXjI0JTjEvZy901Rghm+7fDQOrNIcxB4+gdhj6Kwp6T9VBhBY/AbagKHJocRkDYGd6wvI+p4/10xtQ==" - }, - "node_modules/domutils": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", - "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", - "dependencies": { - "dom-serializer": "0", - "domelementtype": "1" - } - }, - "node_modules/dotenv": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", - "integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==", - "engines": { - "node": ">=6" - } - }, - "node_modules/ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "dependencies": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "node_modules/ellipsize": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/ellipsize/-/ellipsize-0.1.0.tgz", - "integrity": "sha1-nUNoLUS5GtFuvYQmisEDFwplU/g=" - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" - }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "engines": { - "node": ">=6" - } - }, - "node_modules/escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=4.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "node_modules/extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "dependencies": { - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - }, - "bin": { - "extract-zip": "cli.js" - }, - "engines": { - "node": ">= 10.17.0" - }, - "optionalDependencies": { - "@types/yauzl": "^2.9.1" - } - }, - "node_modules/extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "engines": [ - "node >=0.6.0" - ] - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/file-url": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/file-url/-/file-url-3.0.0.tgz", - "integrity": "sha512-g872QGsHexznxkIAdK8UiZRe7SkE6kvylShU4Nsj8NvfvZag7S0QuQ4IgvPDkk75HxgjIVDwycFTDAgIiO4nDA==", - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "engines": { - "node": "*" - } - }, - "node_modules/form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "dependencies": { - "pump": "^3.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dependencies": { - "assert-plus": "^1.0.0" - } - }, - "node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "engines": { - "node": ">=4" - } - }, - "node_modules/har-validator": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", - "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", - "deprecated": "this library is no longer supported", - "dependencies": { - "ajv": "^6.12.3", - "har-schema": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/heap": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", - "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" - }, - "node_modules/html-encoding-sniffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", - "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", - "dependencies": { - "whatwg-encoding": "^1.0.1" - } - }, - "node_modules/htmlparser2": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", - "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", - "dependencies": { - "domelementtype": "^1.3.1", - "domhandler": "^2.3.0", - "domutils": "^1.5.1", - "entities": "^1.1.1", - "inherits": "^2.0.1", - "readable-stream": "^3.1.1" - } - }, - "node_modules/http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", - "dependencies": { - "@tootallnate/once": "1", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/http-signature": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz", - "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==", - "dependencies": { - "assert-plus": "^1.0.0", - "jsprim": "^2.0.2", - "sshpk": "^1.14.1" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/iconv-lite": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz", - "integrity": "sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-potential-custom-element-name": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", - "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" - }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "node_modules/jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "node_modules/jsdom": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", - "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", - "dependencies": { - "abab": "^2.0.0", - "acorn": "^5.5.3", - "acorn-globals": "^4.1.0", - "array-equal": "^1.0.0", - "cssom": ">= 0.3.2 < 0.4.0", - "cssstyle": "^1.0.0", - "data-urls": "^1.0.0", - "domexception": "^1.0.1", - "escodegen": "^1.9.1", - "html-encoding-sniffer": "^1.0.2", - "left-pad": "^1.3.0", - "nwsapi": "^2.0.7", - "parse5": "4.0.0", - "pn": "^1.1.0", - "request": "^2.87.0", - "request-promise-native": "^1.0.5", - "sax": "^1.2.4", - "symbol-tree": "^3.2.2", - "tough-cookie": "^2.3.4", - "w3c-hr-time": "^1.0.1", - "webidl-conversions": "^4.0.2", - "whatwg-encoding": "^1.0.3", - "whatwg-mimetype": "^2.1.0", - "whatwg-url": "^6.4.1", - "ws": "^5.2.0", - "xml-name-validator": "^3.0.0" - } - }, - "node_modules/json-schema": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", - "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "node_modules/jsprim": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", - "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==", - "engines": [ - "node >=0.6.0" - ], - "dependencies": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.4.0", - "verror": "1.10.0" - } - }, - "node_modules/jszip": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.1.tgz", - "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "set-immediate-shim": "~1.0.1" - } - }, - "node_modules/jszip/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/jszip/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/jszip/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/left-pad": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", - "integrity": "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==", - "deprecated": "use String.prototype.padStart()" - }, - "node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "dependencies": { - "immediate": "~3.0.5" - } - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, - "node_modules/lodash.assignin": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assignin/-/lodash.assignin-4.2.0.tgz", - "integrity": "sha1-uo31+4QesKPoBEIysOJjqNxqKKI=" - }, - "node_modules/lodash.bind": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz", - "integrity": "sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU=" - }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" - }, - "node_modules/lodash.filter": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.filter/-/lodash.filter-4.6.0.tgz", - "integrity": "sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=" - }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" - }, - "node_modules/lodash.foreach": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz", - "integrity": "sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=" - }, - "node_modules/lodash.map": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", - "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" - }, - "node_modules/lodash.pick": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", - "integrity": "sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=" - }, - "node_modules/lodash.reduce": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz", - "integrity": "sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=" - }, - "node_modules/lodash.reject": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.reject/-/lodash.reject-4.6.0.tgz", - "integrity": "sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=" - }, - "node_modules/lodash.some": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz", - "integrity": "sha1-G7nzFO9ri63tE7VJFpsqlF62jk0=" - }, - "node_modules/lodash.sortby": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", - "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" - }, - "node_modules/mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", - "dependencies": { - "mime-db": "1.47.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, - "node_modules/moment-parseformat": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/moment-parseformat/-/moment-parseformat-3.0.0.tgz", - "integrity": "sha512-dVgXe6b6DLnv4CHG7a1zUe5mSXaIZ3c6lSHm/EKeVeQI2/4pwe0VRde8OyoCE1Ro2lKT5P6uT9JElF7KDLV+jw==" - }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/nth-check": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", - "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", - "dependencies": { - "boolbase": "~1.0.0" - } - }, - "node_modules/nwsapi": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", - "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==" - }, - "node_modules/oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "engines": { - "node": "*" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "engines": { - "node": ">=6" - } - }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" - }, - "node_modules/parse5": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", - "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, - "node_modules/performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", - "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" - }, - "node_modules/postman-request": { - "version": "2.88.1-postman.30", - "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.30.tgz", - "integrity": "sha512-zsGvs8OgNeno1Q44zTgGP2IL7kCqUy4DAtl8/ms0AQpqkIoysrxzR/Zg4kM1Kz8/duBvwxt8NN717wB7SMNm6w==", - "dependencies": { - "@postman/form-data": "~3.1.1", - "@postman/tunnel-agent": "^0.6.3", - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "brotli": "~1.3.2", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "har-validator": "~5.1.3", - "http-signature": "~1.3.1", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "stream-length": "^1.0.2", - "tough-cookie": "~2.5.0", - "uuid": "^3.3.2" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, - "node_modules/psl": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", - "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" - }, - "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "engines": { - "node": ">=6" - } - }, - "node_modules/puppeteer-core": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-5.5.0.tgz", - "integrity": "sha512-tlA+1n+ziW/Db03hVV+bAecDKse8ihFRXYiEypBe9IlLRvOCzYFG6qrCMBYK34HO/Q/Ecjc+tvkHRAfLVH+NgQ==", - "dependencies": { - "debug": "^4.1.0", - "devtools-protocol": "0.0.818844", - "extract-zip": "^2.0.0", - "https-proxy-agent": "^4.0.0", - "node-fetch": "^2.6.1", - "pkg-dir": "^4.2.0", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^3.0.2", - "tar-fs": "^2.0.0", - "unbzip2-stream": "^1.3.3", - "ws": "^7.2.3" - }, - "engines": { - "node": ">=10.18.1" - } - }, - "node_modules/puppeteer-core/node_modules/agent-base": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/puppeteer-core/node_modules/https-proxy-agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", - "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", - "dependencies": { - "agent-base": "5", - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/puppeteer-core/node_modules/ws": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/readability-extractor": { - "version": "0.0.2", - "resolved": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", - "integrity": "sha512-B+oZuG4FwPYg5hxEafuhrwNOS8uiv/gYKlLKbIaeXXHlyznARYOqHpkHumiLMU6vkbZ3VAC7WucnWwh5jVOaBQ==", - "license": "MIT", - "dependencies": { - "@mozilla/readability": "^0.4.1", - "dompurify": "^2.2.7", - "jsdom": "^16.5.2" - }, - "bin": { - "readability-extractor": "readability-extractor" - } - }, - "node_modules/readability-extractor/node_modules/acorn": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", - "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/readability-extractor/node_modules/acorn-globals": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", - "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", - "dependencies": { - "acorn": "^7.1.1", - "acorn-walk": "^7.1.1" - } - }, - "node_modules/readability-extractor/node_modules/acorn-globals/node_modules/acorn": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", - "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/readability-extractor/node_modules/acorn-walk": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", - "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/readability-extractor/node_modules/cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" - }, - "node_modules/readability-extractor/node_modules/cssstyle": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", - "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", - "dependencies": { - "cssom": "~0.3.6" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/readability-extractor/node_modules/cssstyle/node_modules/cssom": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" - }, - "node_modules/readability-extractor/node_modules/data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", - "dependencies": { - "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/readability-extractor/node_modules/domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", - "dependencies": { - "webidl-conversions": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/readability-extractor/node_modules/domexception/node_modules/webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", - "engines": { - "node": ">=8" - } - }, - "node_modules/readability-extractor/node_modules/escodegen": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", - "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/readability-extractor/node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/readability-extractor/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readability-extractor/node_modules/html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", - "dependencies": { - "whatwg-encoding": "^1.0.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/readability-extractor/node_modules/jsdom": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", - "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", - "dependencies": { - "abab": "^2.0.5", - "acorn": "^8.2.4", - "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", - "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", - "domexception": "^2.0.1", - "escodegen": "^2.0.0", - "form-data": "^3.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", - "https-proxy-agent": "^5.0.0", - "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.0", - "parse5": "6.0.1", - "saxes": "^5.0.1", - "symbol-tree": "^3.2.4", - "tough-cookie": "^4.0.0", - "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.6", - "xml-name-validator": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "canvas": "^2.5.0" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } - } - }, - "node_modules/readability-extractor/node_modules/parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" - }, - "node_modules/readability-extractor/node_modules/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.1.2" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/readability-extractor/node_modules/tr46": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", - "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", - "dependencies": { - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/readability-extractor/node_modules/webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", - "engines": { - "node": ">=10.4" - } - }, - "node_modules/readability-extractor/node_modules/whatwg-url": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", - "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", - "dependencies": { - "lodash": "^4.7.0", - "tr46": "^2.1.0", - "webidl-conversions": "^6.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/readability-extractor/node_modules/ws": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/regenerator-runtime": { - "version": "0.13.9", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" - }, - "node_modules/request": { - "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", - "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", - "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", - "dependencies": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.3", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.5.0", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/request-promise": { - "version": "4.2.6", - "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.6.tgz", - "integrity": "sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==", - "deprecated": "request-promise has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", - "dependencies": { - "bluebird": "^3.5.0", - "request-promise-core": "1.1.4", - "stealthy-require": "^1.1.1", - "tough-cookie": "^2.3.3" - }, - "engines": { - "node": ">=0.10.0" - }, - "peerDependencies": { - "request": "^2.34" - } - }, - "node_modules/request-promise-core": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", - "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", - "dependencies": { - "lodash": "^4.17.19" - }, - "engines": { - "node": ">=0.10.0" - }, - "peerDependencies": { - "request": "^2.34" - } - }, - "node_modules/request-promise-native": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", - "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", - "deprecated": "request-promise-native has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", - "dependencies": { - "request-promise-core": "1.1.4", - "stealthy-require": "^1.1.1", - "tough-cookie": "^2.3.3" - }, - "engines": { - "node": ">=0.12.0" - }, - "peerDependencies": { - "request": "^2.34" - } - }, - "node_modules/request-promise/node_modules/bluebird": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" - }, - "node_modules/request/node_modules/http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dependencies": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - }, - "engines": { - "node": ">=0.8", - "npm": ">=1.3.7" - } - }, - "node_modules/request/node_modules/jsprim": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", - "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", - "dependencies": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.4.0", - "verror": "1.10.0" - }, - "engines": { - "node": ">=0.6.0" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/sax": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" - }, - "node_modules/saxes": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", - "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", - "dependencies": { - "xmlchars": "^2.2.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/selenium-webdriver": { - "version": "4.0.0-alpha.7", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.0.0-alpha.7.tgz", - "integrity": "sha512-D4qnTsyTr91jT8f7MfN+OwY0IlU5+5FmlO5xlgRUV6hDEV8JyYx2NerdTEqDDkNq7RZDYc4VoPALk8l578RBHw==", - "dependencies": { - "jszip": "^3.2.2", - "rimraf": "^2.7.1", - "tmp": "0.0.30" - }, - "engines": { - "node": ">= 10.15.0" - } - }, - "node_modules/selenium-webdriver/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, - "node_modules/set-immediate-shim": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", - "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/single-file": { - "version": "0.3.16", - "resolved": "git+ssh://git@github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", - "integrity": "sha512-NwwtloJdZlPG2zKrTAlcPRowIcwQx+1U39wcyPnfsbpVZKi6FAyLpo+CV0/xVJmvdOO0DSTvimuZtnc8/gJfTw==", - "license": "AGPL-3.0-or-later", - "dependencies": { - "file-url": "^3.0.0", - "iconv-lite": "^0.6.2", - "jsdom": "^16.4.0", - "puppeteer-core": "^5.3.0", - "selenium-webdriver": "4.0.0-alpha.7", - "strong-data-uri": "^1.0.6", - "yargs": "^16.2.0" - }, - "bin": { - "single-file": "cli/single-file" - } - }, - "node_modules/single-file/node_modules/acorn": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz", - "integrity": "sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/single-file/node_modules/acorn-globals": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", - "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", - "dependencies": { - "acorn": "^7.1.1", - "acorn-walk": "^7.1.1" - } - }, - "node_modules/single-file/node_modules/acorn-globals/node_modules/acorn": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", - "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/single-file/node_modules/acorn-walk": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", - "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/single-file/node_modules/cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" - }, - "node_modules/single-file/node_modules/cssstyle": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", - "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", - "dependencies": { - "cssom": "~0.3.6" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/single-file/node_modules/cssstyle/node_modules/cssom": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" - }, - "node_modules/single-file/node_modules/data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", - "dependencies": { - "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/single-file/node_modules/domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", - "dependencies": { - "webidl-conversions": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/single-file/node_modules/domexception/node_modules/webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", - "engines": { - "node": ">=8" - } - }, - "node_modules/single-file/node_modules/escodegen": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", - "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/single-file/node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/single-file/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/single-file/node_modules/html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", - "dependencies": { - "whatwg-encoding": "^1.0.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/single-file/node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/single-file/node_modules/jsdom": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", - "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", - "dependencies": { - "abab": "^2.0.5", - "acorn": "^8.2.4", - "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", - "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", - "domexception": "^2.0.1", - "escodegen": "^2.0.0", - "form-data": "^3.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", - "https-proxy-agent": "^5.0.0", - "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.0", - "parse5": "6.0.1", - "saxes": "^5.0.1", - "symbol-tree": "^3.2.4", - "tough-cookie": "^4.0.0", - "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.6", - "xml-name-validator": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "canvas": "^2.5.0" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } - } - }, - "node_modules/single-file/node_modules/parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" - }, - "node_modules/single-file/node_modules/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.1.2" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/single-file/node_modules/tr46": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", - "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", - "dependencies": { - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/single-file/node_modules/webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", - "engines": { - "node": ">=10.4" - } - }, - "node_modules/single-file/node_modules/whatwg-url": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", - "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", - "dependencies": { - "lodash": "^4.7.0", - "tr46": "^2.1.0", - "webidl-conversions": "^6.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/single-file/node_modules/ws": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "dependencies": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stealthy-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", - "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stream-length": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/stream-length/-/stream-length-1.0.2.tgz", - "integrity": "sha1-gnfzy+5JpNqrz9tOL0qbXp8snwA=", - "dependencies": { - "bluebird": "^2.6.2" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-direction": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/string-direction/-/string-direction-0.1.2.tgz", - "integrity": "sha1-PYRT5ydKLkShQrPchEnftk2a3jo=" - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strong-data-uri": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/strong-data-uri/-/strong-data-uri-1.0.6.tgz", - "integrity": "sha512-zhzBZev0uhT2IrFUerenXhfaE0vFUYwAZsnG0gIKGpfM/Gi6jOUQ3cmcvyTsXeDLIPiTubHESeO7EbD6FoPmzw==", - "dependencies": { - "truncate": "^2.0.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/symbol-tree": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" - }, - "node_modules/tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, - "node_modules/tmp": { - "version": "0.0.30", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.30.tgz", - "integrity": "sha1-ckGdSovn1s51FI/YsyTlk6cRwu0=", - "dependencies": { - "os-tmpdir": "~1.0.1" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/tr46": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", - "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/truncate": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/truncate/-/truncate-2.1.0.tgz", - "integrity": "sha512-em3E3SUDONOjTBcZ36DTm3RvDded3IRU9rX32oHwwXNt3rJD5MVaFlJTQvs8tJoHRoeYP36OuQ1eL/Q7bNEWIQ==", - "engines": { - "node": "*" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/turndown": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/turndown/-/turndown-5.0.3.tgz", - "integrity": "sha512-popfGXEiedpq6F5saRIAThKxq/bbEPVFnsDnUdjaDGIre9f3/OL9Yi/yPbPcZ7RYUDpekghr666bBfZPrwNnhQ==", - "dependencies": { - "jsdom": "^11.9.0" - } - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dependencies": { - "prelude-ls": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/unbzip2-stream": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", - "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "dependencies": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, - "node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "engines": { - "node": ">= 4.0.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/url": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, - "node_modules/url/node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/valid-url": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", - "integrity": "sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA=" - }, - "node_modules/verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "engines": [ - "node >=0.6.0" - ], - "dependencies": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "node_modules/w3c-hr-time": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", - "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", - "dependencies": { - "browser-process-hrtime": "^1.0.0" - } - }, - "node_modules/w3c-xmlserializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", - "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", - "dependencies": { - "xml-name-validator": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/webidl-conversions": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", - "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" - }, - "node_modules/whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", - "dependencies": { - "iconv-lite": "0.4.24" - } - }, - "node_modules/whatwg-encoding/node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" - }, - "node_modules/whatwg-url": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", - "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", - "dependencies": { - "lodash.sortby": "^4.7.0", - "tr46": "^1.0.1", - "webidl-conversions": "^4.0.2" - } - }, - "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "node_modules/ws": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.3.tgz", - "integrity": "sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA==", - "dependencies": { - "async-limiter": "~1.0.0" - } - }, - "node_modules/wuzzy": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/wuzzy/-/wuzzy-0.1.8.tgz", - "integrity": "sha512-FUzKQepFSTnANsDYwxpIzGJ/dIJaqxuMre6tzzbvWwFAiUHPsI1nVQVCLK4Xqr67KO7oYAK0kaCcI/+WYj/7JA==", - "dependencies": { - "lodash": "^4.17.15" - } - }, - "node_modules/xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" - }, - "node_modules/xmlchars": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-14.0.0.tgz", - "integrity": "sha512-zn/Mnx+tbFjkCFUodEpjXckNS65NfpB5oyqOkDDEG/8uxlfLZJu2IoBLQFjukUkn9rBbGkVYNzrDh6qy4NUd3g==", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } - }, - "node_modules/yargs/node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "engines": { - "node": ">=10" - } - }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } - } - }, "dependencies": { "@babel/runtime-corejs2": { "version": "7.16.5", @@ -2965,38 +53,16 @@ "yargs-parser": "^14.0.0" }, "dependencies": { - "browser-request": { - "version": "0.3.2", - "bundled": true, - "requires": { - "http-headers": "^3.0.1" - } - }, "http-headers": { "version": "3.0.2", - "bundled": true, + "resolved": false, "requires": { "next-line": "^1.1.0" } }, - "jquery": { - "version": "3.4.1", - "bundled": true - }, - "moment": { - "version": "2.23.0", - "bundled": true - }, - "moment-timezone": { - "version": "0.5.26", - "bundled": true, - "requires": { - "moment": ">= 2.9.0" - } - }, "next-line": { "version": "1.1.0", - "bundled": true + "resolved": false } } }, @@ -3199,6 +265,13 @@ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, + "browser-request": { + "version": "github:postlight/browser-request#38faa5b85741aabfca61aa37d1ef044d68969ddf", + "from": "github:postlight/browser-request#feat-add-headers-to-response", + "requires": { + "http-headers": "^3.0.1" + } + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -3390,9 +463,8 @@ "integrity": "sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg==" }, "difflib": { - "version": "git+ssh://git@github.com/postlight/difflib.js.git#32e8e38c7fcd935241b9baab71bb432fd9b166ed", - "integrity": "sha512-uFNs7czGYLWdMP22WQhD/vlFen/CuKzC+KiajNCj+ik2Ah/I9i2AFyMWkBjFgbVFGhv95kBHOtx7tgF6IVngqA==", - "from": "difflib@github:postlight/difflib.js", + "version": "github:postlight/difflib.js#32e8e38c7fcd935241b9baab71bb432fd9b166ed", + "from": "github:postlight/difflib.js", "requires": { "heap": ">= 0.2.0" } @@ -3667,6 +739,14 @@ "readable-stream": "^3.1.1" } }, + "http-headers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-headers/-/http-headers-3.0.2.tgz", + "integrity": "sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==", + "requires": { + "next-line": "^1.1.0" + } + }, "http-proxy-agent": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", @@ -3753,6 +833,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jquery": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz", + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -3983,16 +1068,34 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "moment": { + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz", + "integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==" + }, "moment-parseformat": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/moment-parseformat/-/moment-parseformat-3.0.0.tgz", "integrity": "sha512-dVgXe6b6DLnv4CHG7a1zUe5mSXaIZ3c6lSHm/EKeVeQI2/4pwe0VRde8OyoCE1Ro2lKT5P6uT9JElF7KDLV+jw==" }, + "moment-timezone": { + "version": "0.5.26", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz", + "integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "next-line": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", + "integrity": "sha1-/K5XhTBStqm66CCOQN19PC0wRgM=" + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -4234,8 +1337,7 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "requires": {} + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -4250,9 +1352,8 @@ "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, "readability-extractor": { - "version": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", - "integrity": "sha512-B+oZuG4FwPYg5hxEafuhrwNOS8uiv/gYKlLKbIaeXXHlyznARYOqHpkHumiLMU6vkbZ3VAC7WucnWwh5jVOaBQ==", - "from": "readability-extractor@git+https://github.com/ArchiveBox/readability-extractor.git", + "version": "git+https://github.com/ArchiveBox/readability-extractor.git#42b243843c724a5d7a6b364d23985ff6acaeb55a", + "from": "git+https://github.com/ArchiveBox/readability-extractor.git", "requires": { "@mozilla/readability": "^0.4.1", "dompurify": "^2.2.7", @@ -4440,8 +1541,7 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "requires": {} + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -4608,9 +1708,8 @@ "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" }, "single-file": { - "version": "git+ssh://git@github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", - "integrity": "sha512-NwwtloJdZlPG2zKrTAlcPRowIcwQx+1U39wcyPnfsbpVZKi6FAyLpo+CV0/xVJmvdOO0DSTvimuZtnc8/gJfTw==", - "from": "single-file@git+https://github.com/gildas-lormeau/SingleFile.git", + "version": "git+https://github.com/gildas-lormeau/SingleFile.git#ec9dbc7c2272bff0dc2415a44d6cdfb2b48aa7d2", + "from": "git+https://github.com/gildas-lormeau/SingleFile.git", "requires": { "file-url": "^3.0.0", "iconv-lite": "^0.6.2", @@ -4810,8 +1909,7 @@ "ws": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz", - "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==", - "requires": {} + "integrity": "sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==" } } }, @@ -4850,14 +1948,6 @@ "bluebird": "^2.6.2" } }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - } - }, "string-direction": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/string-direction/-/string-direction-0.1.2.tgz", @@ -4873,6 +1963,14 @@ "strip-ansi": "^6.0.1" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", From 8ebf3e2f93188013b494dcfea29e939f1fd7f348 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:31:41 -0700 Subject: [PATCH 057/124] add config option PREVIEW_ORIGINALS to hide original iframes in snapshot detail pages --- archivebox/config.py | 1 + archivebox/index/html.py | 2 ++ archivebox/templates/core/snapshot.html | 2 ++ 3 files changed, 5 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index 21e48ff2..3da4b304 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -94,6 +94,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40}, 'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None}, 'TIME_ZONE': {'type': str, 'default': 'UTC'}, + 'PREVIEW_ORIGINALS': {'type': bool, 'default': True}, }, 'ARCHIVE_METHOD_TOGGLES': { diff --git a/archivebox/index/html.py b/archivebox/index/html.py index d45f66ea..66e26fab 100644 --- a/archivebox/index/html.py +++ b/archivebox/index/html.py @@ -24,6 +24,7 @@ from ..config import ( FOOTER_INFO, HTML_INDEX_FILENAME, SAVE_ARCHIVE_DOT_ORG, + PREVIEW_ORIGINALS, ) MAIN_INDEX_TEMPLATE = 'static_index.html' @@ -105,6 +106,7 @@ def link_details_template(link: Link) -> str: 'status_color': 'success' if link.is_archived else 'danger', 'oldest_archive_date': ts_to_date_str(link.oldest_archive_date), 'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG, + 'PREVIEW_ORIGINALS': PREVIEW_ORIGINALS, }) @enforce_types diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index ccb74227..9b8b1e53 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -414,6 +414,7 @@ {% endif %} + {% if PREVIEW_ORIGINALS %}
@@ -427,6 +428,7 @@
+ {{% endif %}}
From ecbcb6a1b3e75095f9c6d7e3659363bf7f56b56b Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:56:08 -0700 Subject: [PATCH 058/124] fix bracing in template tag for PREVIEW_ORIGINALS --- archivebox/templates/core/snapshot.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index 9b8b1e53..d4f43d3d 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -428,7 +428,7 @@
- {{% endif %}} + {% endif %}
From 38e54b93fe36cfdb5d3f5b529e26a286e6d242f1 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 19:56:24 -0700 Subject: [PATCH 059/124] allow parsing to continue even when fetching URL contents fails --- archivebox/main.py | 7 +++++-- archivebox/parsers/__init__.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index ed1df69b..d4e7d9c3 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -594,8 +594,11 @@ def add(urls: Union[str, List[str]], if new_links and depth == 1: log_crawl_started(new_links) for new_link in new_links: - downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir) - new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url) + try: + downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir) + new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url) + except Exception as err: + stderr('[!] Failed to get contents of URL {new_link.url}', err, color='red') imported_links = list({link.url: link for link in (new_links + new_links_depth)}.values()) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 2451f0f5..0ae958f2 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -176,7 +176,7 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba ANSI['reset'], )) print(' ', e) - raise SystemExit(1) + raise e else: # Source is a path to a local file on the filesystem From 8cfe6f4afbcc839fc497dd5e0a536cab0c4091d4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 20:15:55 -0700 Subject: [PATCH 060/124] cleanup update flag handling and show better logging to clarify when its working --- archivebox/cli/archivebox_add.py | 9 ++++++++- archivebox/main.py | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 2c3d7ce3..ed05584c 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -30,11 +30,17 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3", ) parser.add_argument( - '--update-all', #'-n', + '--update', #'-u', action='store_true', default=not ONLY_NEW, # when ONLY_NEW=True we skip updating old links help="Also retry previously skipped/failed links when adding new links", ) + parser.add_argument( + '--update-all', #'-n', + action='store_true', + default=False, + help="Also update ALL links in index when finished adding new links", + ) parser.add_argument( '--index-only', #'-o', action='store_true', @@ -104,6 +110,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional urls=stdin_urls or urls, depth=command.depth, tag=command.tag, + update=command.update, update_all=command.update_all, index_only=command.index_only, overwrite=command.overwrite, diff --git a/archivebox/main.py b/archivebox/main.py index d4e7d9c3..b330d496 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -4,8 +4,9 @@ import os import sys import shutil import platform +from django.utils import timezone from pathlib import Path -from datetime import date +from datetime import date, datetime from typing import Dict, List, Optional, Iterable, IO, Union from crontab import CronTab, CronSlices @@ -554,7 +555,8 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR): def add(urls: Union[str, List[str]], tag: str='', depth: int=0, - update_all: bool=not ONLY_NEW, + update: bool=not ONLY_NEW, + update_all: bool=False, index_only: bool=False, overwrite: bool=False, # duplicate: bool=False, # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically @@ -621,11 +623,21 @@ def add(urls: Union[str, List[str]], if extractors: archive_kwargs["methods"] = extractors - if update_all: + stderr() + + ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') + + if update: + stderr(f'[*] [{ts}] Archiving + updating', len(imported_links), 'URLs from added set...', color='green') + archive_links(imported_links, overwrite=overwrite, **archive_kwargs) + elif update_all: + stderr(f'[*] [{ts}] Archiving + updating ', len(all_links), 'URLs from entire library...', color='green') archive_links(all_links, overwrite=overwrite, **archive_kwargs) elif overwrite: + stderr(f'[*] [{ts}] Archiving + overwriting', len(imported_links), 'URLs from added set...', color='green') archive_links(imported_links, overwrite=True, **archive_kwargs) elif new_links: + stderr(f'[*] [{ts}] Archiving', len(imported_links), 'URLs from added set...', color='green') archive_links(new_links, overwrite=False, **archive_kwargs) From 4b8b17e788e4e0efa26db87b84b4e123941b1cff Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 20:18:43 -0700 Subject: [PATCH 061/124] add update flag support to archivebox schedule --- archivebox/cli/archivebox_schedule.py | 6 ++++++ archivebox/main.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index f528e6a6..d4747906 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -51,6 +51,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional action='store_true', help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots', ) + parser.add_argument( + '--update', + action='store_true', + help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults', + ) group.add_argument( '--clear', # '-c' action='store_true', @@ -94,6 +99,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional every=command.every, depth=command.depth, overwrite=command.overwrite, + update=command.update, import_path=command.import_path, out_dir=pwd or OUTPUT_DIR, ) diff --git a/archivebox/main.py b/archivebox/main.py index b330d496..5f672656 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -1128,6 +1128,7 @@ def schedule(add: bool=False, every: Optional[str]=None, depth: int=0, overwrite: bool=False, + update: bool=not ONLY_NEW, import_path: Optional[str]=None, out_dir: Path=OUTPUT_DIR): """Set ArchiveBox to regularly import URLs at specific times using cron""" @@ -1157,6 +1158,7 @@ def schedule(add: bool=False, *([ 'add', *(['--overwrite'] if overwrite else []), + *(['--update'] if update else []), f'--depth={depth}', f'"{import_path}"', ] if import_path else ['update']), From d05510f844ae82dc738f55c27100defca47618ef Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:21:08 -0700 Subject: [PATCH 062/124] fix version string parsing on macOS in some cases where LANG C is not supported --- archivebox/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index 3da4b304..6cfee837 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -653,6 +653,8 @@ def bin_version(binary: Optional[str]) -> Optional[str]: try: version_str = run([abspath, "--version"], stdout=PIPE, env={'LANG': 'C'}).stdout.strip().decode() + if not version_str: + version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode() # take first 3 columns of first line of version info return ' '.join(version_str.split('\n')[0].strip().split()[:3]) except OSError: From f6d6a06c7872f6c8ccdf08ff65eef06e139742ad Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:21:26 -0700 Subject: [PATCH 063/124] always show all totals in log output --- archivebox/main.py | 9 +++++---- archivebox/parsers/__init__.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 5f672656..12016dbc 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -589,6 +589,7 @@ def add(urls: Union[str, List[str]], # save verbatim args to sources write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir) + new_links += parse_links_from_source(write_ahead_log, root_url=None, parser=parser) # If we're going one level deeper, download each link and look for more links @@ -628,16 +629,16 @@ def add(urls: Union[str, List[str]], ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') if update: - stderr(f'[*] [{ts}] Archiving + updating', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving + updating {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green') archive_links(imported_links, overwrite=overwrite, **archive_kwargs) elif update_all: - stderr(f'[*] [{ts}] Archiving + updating ', len(all_links), 'URLs from entire library...', color='green') + stderr(f'[*] [{ts}] Archiving + updating {len(all_links)}/{len(all_links)}', len(all_links), 'URLs from entire library...', color='green') archive_links(all_links, overwrite=overwrite, **archive_kwargs) elif overwrite: - stderr(f'[*] [{ts}] Archiving + overwriting', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving + overwriting {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green') archive_links(imported_links, overwrite=True, **archive_kwargs) elif new_links: - stderr(f'[*] [{ts}] Archiving', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving {len(new_links)}/{len(all_links)} URLs from added set...', color='green') archive_links(new_links, overwrite=False, **archive_kwargs) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 0ae958f2..2ce01e1a 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -8,6 +8,7 @@ For examples of supported import formats see tests/. __package__ = 'archivebox.parsers' import re +import os from io import StringIO from typing import IO, Tuple, List, Optional From a6767671fb68e25f67edcf16afafe5234d2826dd Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:21:39 -0700 Subject: [PATCH 064/124] append content of referenced files to imports --- archivebox/parsers/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 2ce01e1a..fef1efe1 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -150,7 +150,17 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None, def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str: ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0] source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts)) - atomic_write(source_path, raw_text) + + referenced_texts = '' + + for entry in raw_text.split(): + try: + if Path(entry).exists: + referenced_texts += Path(entry).read_text() + except Exception as err: + print(err) + + atomic_write(source_path, raw_text + '\n' + referenced_texts) log_source_saved(source_file=source_path) return source_path From d581a5081f802fd9c242096616654d33af93e12d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:29:37 -0700 Subject: [PATCH 065/124] correctly handle bytes strings in hints --- archivebox/logging_util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index e2403850..542e2b78 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -432,7 +432,9 @@ def log_archive_method_finished(result: "ArchiveResult"): # Prettify error output hints string and limit to five lines hints = getattr(result.output, 'hints', None) or () if hints: - if not isinstance(hints, (list, tuple)): + if isinstance(hints, (list, tuple)): + hints = (hint.decode() for hint in hints if isinstance(hint, bytes)) + else: if isinstance(hints, bytes): hints = hints.decode() hints = hints.split('\n') From e96c1bcf13cd5e4b1f9310a7db9343edf2fc2be0 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:48:41 -0700 Subject: [PATCH 066/124] bump mercury parser to git head version --- archivebox/config.py | 2 +- package-lock.json | 52 ++++++++++++++++---------------------------- package.json | 2 +- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 6cfee837..e06fc9a6 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -378,7 +378,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'READABILITY_VERSION': {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None}, 'USE_MERCURY': {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']}, - 'MERCURY_VERSION': {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None}, # mercury is unversioned + 'MERCURY_VERSION': {'default': lambda c: (bin_version(c['MERCURY_BINARY']) or '1.0.0') if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None}, # mercury is unversioned 'USE_GIT': {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']}, 'GIT_VERSION': {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None}, diff --git a/package-lock.json b/package-lock.json index fed13aaf..eab7b512 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,11 +5,11 @@ "requires": true, "dependencies": { "@babel/runtime-corejs2": { - "version": "7.16.5", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.16.5.tgz", - "integrity": "sha512-GHejyoK+JQqna9rUTkybaGahZOOM+EDcUsbWaLye1g4ZOQMotrMiY8VTBbX1gycZTMNG/YiWTo4WTABvyPUgOg==", + "version": "7.17.11", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.17.11.tgz", + "integrity": "sha512-pJe8Aerb88TGVi1Xe/AE36aRCPrg+h6ktZPGl6xaJvOfTLcMMuogQu3BYcxeXPTNHhSYbmsDVYBs8CfAxeFFTg==", "requires": { - "core-js": "^2.6.5", + "core-js": "^2.6.12", "regenerator-runtime": "^0.13.4" } }, @@ -28,9 +28,8 @@ } }, "@postlight/mercury-parser": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@postlight/mercury-parser/-/mercury-parser-2.2.1.tgz", - "integrity": "sha512-WsXHcVfbwlAb0HO1nZ3tS2ipNvLeLk9frJj3y5hn/NAeQfEJeRm9AtMrxpOJ11X8L4he+bRyjr1bpIPvQPWvCA==", + "version": "git+https://github.com/postlight/mercury-parser.git#9cd9662bcbfea00b773fad691a4f6e53394ff543", + "from": "git+https://github.com/postlight/mercury-parser.git", "requires": { "@babel/runtime-corejs2": "^7.2.0", "@postlight/ci-failed-test-reporter": "^1.0", @@ -50,20 +49,7 @@ "url": "^0.11.0", "valid-url": "^1.0.9", "wuzzy": "^0.1.4", - "yargs-parser": "^14.0.0" - }, - "dependencies": { - "http-headers": { - "version": "3.0.2", - "resolved": false, - "requires": { - "next-line": "^1.1.0" - } - }, - "next-line": { - "version": "1.1.0", - "resolved": false - } + "yargs-parser": "^15.0.1" } }, "@postman/form-data": { @@ -1234,9 +1220,9 @@ "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" }, "postman-request": { - "version": "2.88.1-postman.30", - "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.30.tgz", - "integrity": "sha512-zsGvs8OgNeno1Q44zTgGP2IL7kCqUy4DAtl8/ms0AQpqkIoysrxzR/Zg4kM1Kz8/duBvwxt8NN717wB7SMNm6w==", + "version": "2.88.1-postman.31", + "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.31.tgz", + "integrity": "sha512-OJbYqP7ItxQ84yHyuNpDywCZB0HYbpHJisMQ9lb1cSL3N5H3Td6a2+3l/a74UMd3u82BiGC5yQyYmdOIETP/nQ==", "requires": { "@postman/form-data": "~3.1.1", "@postman/tunnel-agent": "^0.6.3", @@ -1342,9 +1328,9 @@ } }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==" }, "querystring": { "version": "0.2.0", @@ -1920,9 +1906,9 @@ "optional": true }, "sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", + "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -2269,9 +2255,9 @@ } }, "yargs-parser": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-14.0.0.tgz", - "integrity": "sha512-zn/Mnx+tbFjkCFUodEpjXckNS65NfpB5oyqOkDDEG/8uxlfLZJu2IoBLQFjukUkn9rBbGkVYNzrDh6qy4NUd3g==", + "version": "15.0.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.3.tgz", + "integrity": "sha512-/MVEVjTXy/cGAjdtQf8dW3V9b97bPN7rNn8ETj6BmAQL7ibC7O1Q9SPJbGjgh3SlwoBNXMzj/ZGIj8mBgl12YA==", "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" diff --git a/package.json b/package.json index 194713e9..405a2ed1 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { - "@postlight/mercury-parser": "^2.2.1", + "@postlight/mercury-parser": "git+https://github.com/postlight/mercury-parser.git", "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git", "single-file": "git+https://github.com/gildas-lormeau/SingleFile.git" } From 5e4ddbbf25978383d3501b3bdf46eadc0b46e671 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:58:17 -0700 Subject: [PATCH 067/124] fix mercury bin parsing back --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index e06fc9a6..6cfee837 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -378,7 +378,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'READABILITY_VERSION': {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None}, 'USE_MERCURY': {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']}, - 'MERCURY_VERSION': {'default': lambda c: (bin_version(c['MERCURY_BINARY']) or '1.0.0') if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None}, # mercury is unversioned + 'MERCURY_VERSION': {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None}, # mercury is unversioned 'USE_GIT': {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']}, 'GIT_VERSION': {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None}, From 74b3c5931427ed3eb7600a78244a65f0982607ff Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:58:30 -0700 Subject: [PATCH 068/124] fix helper install script handling of python3 version --- bin/setup.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/setup.sh b/bin/setup.sh index 30a3ea54..5f4b4103 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -120,8 +120,8 @@ if which apt-get > /dev/null; then sudo apt-get install -y archivebox sudo apt-get --only-upgrade install -y archivebox echo "" - echo "[+] Installing ArchiveBox python dependencies using pip..." - sudo python3.7 -m pip install --upgrade --ignore-installed archivebox + echo "[+] Installing ArchiveBox python dependencies using pip3..." + sudo python3 -m pip install --upgrade --ignore-installed archivebox # On Mac: elif which brew > /dev/null; then echo "[+] Installing ArchiveBox system dependencies using brew..." @@ -129,16 +129,15 @@ elif which brew > /dev/null; then brew update brew install --fetch-HEAD -f archivebox echo "" - echo "[+] Installing ArchiveBox python dependencies using pip..." + echo "[+] Installing ArchiveBox python dependencies using pip3..." python3 -m pip install --upgrade --ignore-installed archivebox elif which pkg > /dev/null; then - echo "[+] Installing ArchiveBox system dependencies using pkg..." + echo "[+] Installing ArchiveBox system dependencies using pkg and pip37 (python3.7)..." sudo pkg install -y python37 py37-pip py37-sqlite3 node npm wget curl youtube_dl ffmpeg git ripgrep sudo pkg install -y chromium echo "" - echo "[+] Installing ArchiveBox python dependencies using pip..." - sudo python3.7 -m pip install --upgrade --ignore-installed archivebox - alias python3=python3.7 + echo "[+] Installing ArchiveBox python dependencies using pip37..." + sudo python3 -m pip install --upgrade --ignore-installed archivebox else echo "[!] Warning: Could not find aptitude/homebrew/pkg! May not be able to install all dependencies automatically." echo "" From 0b4df768bac61d246a0e13397095e451e5af2887 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 23:50:56 -0700 Subject: [PATCH 070/124] hack to check for generator type cause too lazy to import --- archivebox/logging_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index 542e2b78..acf62511 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -432,7 +432,7 @@ def log_archive_method_finished(result: "ArchiveResult"): # Prettify error output hints string and limit to five lines hints = getattr(result.output, 'hints', None) or () if hints: - if isinstance(hints, (list, tuple)): + if isinstance(hints, (list, tuple, type(_ for _ in ()))): hints = (hint.decode() for hint in hints if isinstance(hint, bytes)) else: if isinstance(hints, bytes): From c5fc3e1e65a4fc76caa70ebfa581c5537b0c5717 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 23:59:27 -0700 Subject: [PATCH 071/124] --ammend --- archivebox/logging_util.py | 3 ++- archivebox/parsers/__init__.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index acf62511..e00d4cbb 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -433,11 +433,12 @@ def log_archive_method_finished(result: "ArchiveResult"): hints = getattr(result.output, 'hints', None) or () if hints: if isinstance(hints, (list, tuple, type(_ for _ in ()))): - hints = (hint.decode() for hint in hints if isinstance(hint, bytes)) + hints = [hint.decode() for hint in hints if isinstance(hint, bytes)] else: if isinstance(hints, bytes): hints = hints.decode() hints = hints.split('\n') + hints = ( ' {}{}{}'.format(ANSI['lightyellow'], line.strip(), ANSI['reset']) for line in hints[:5] if line.strip() diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index fef1efe1..992446ba 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -8,7 +8,6 @@ For examples of supported import formats see tests/. __package__ = 'archivebox.parsers' import re -import os from io import StringIO from typing import IO, Tuple, List, Optional From 5a7828fb08a36db13c540d1e986cfdbeb9115da3 Mon Sep 17 00:00:00 2001 From: Manu Date: Tue, 10 May 2022 15:28:10 +0400 Subject: [PATCH 073/124] Add PikaPods as hosting option --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index faf046d0..228d1480 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,7 @@ None of these hosting providers are officially endorsed:
(most still require manual setup or manual periodic updating using the methods above)

  • (USD $29-250/mo, pricing)
    • +
  • (from USD $2.6/mo)
  • (USD $5-50+/mo, 🎗  referral link, instructions)
    • From 880ab07f10648f5261f99cb066f350c1edf14811 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 16:59:42 -0700 Subject: [PATCH 074/124] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 228d1480..46427e04 100644 --- a/README.md +++ b/README.md @@ -1081,7 +1081,7 @@ Extractors take the URL of a page to archive, write their output to the filesyst
    -This project is maintained mostly in my spare time with the help from generous contributors and Monadical (✨ hire them for dev work!). +This project is maintained mostly in my spare time with the help from generous contributors and Monadical (✨ hire them for dev work!).

    From c245d36e44bf6ee559352bcaef726ca4def0b96d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 17:42:52 -0700 Subject: [PATCH 075/124] add PUID and PGID as config options in archivebox --- archivebox/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index 6cfee837..11b7c926 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -65,6 +65,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'USE_COLOR': {'type': bool, 'default': lambda c: c['IS_TTY']}, 'SHOW_PROGRESS': {'type': bool, 'default': lambda c: (c['IS_TTY'] and platform.system() != 'Darwin')}, # progress bars are buggy on mac, disable for now 'IN_DOCKER': {'type': bool, 'default': False}, + 'PUID': {'type': int, 'default': os.getuid()}, + 'PGID': {'type': int, 'default': os.getgid()}, # TODO: 'SHOW_HINTS': {'type: bool, 'default': True}, }, From 89175ccb22bf1d7cf27e6a23d268077c7d53434a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:24:17 -0700 Subject: [PATCH 076/124] check SQLite3 version and enabled extensions on startup --- archivebox/config.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index 11b7c926..28efcee9 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -355,6 +355,11 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'DJANGO_BINARY': {'default': lambda c: django.__file__.replace('__init__.py', 'bin/django-admin.py')}, 'DJANGO_VERSION': {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)}, + + 'SQLITE_BINARY': {'default': lambda c: 'sqlite3'}, + 'SQLITE_VERSION': {'default': lambda c: None}, + 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, + 'SQLITE_EXTENSIONS': {'default': lambda c: []}, 'USE_CURL': {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])}, 'CURL_VERSION': {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None}, @@ -850,6 +855,13 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue: 'enabled': True, 'is_valid': bool(config['DJANGO_VERSION']), }, + 'SQLITE_BINARY': { + 'path': bin_path(config['SQLITE_BINARY']), + 'version': config['SQLITE_VERSION'], + 'hash': bin_hash(config['SQLITE_BINARY']), + 'enabled': True, + 'is_valid': bool(config['SQLITE_VERSION']) and ('JSON1' in config['SQLITE_EXTENSIONS']), + }, 'CURL_BINARY': { 'path': bin_path(config['CURL_BINARY']), 'version': config['CURL_VERSION'], @@ -1159,7 +1171,6 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, # Otherwise use default sqlite3 file-based database and initialize django # without running migrations automatically (user runs them manually by calling init) django.setup() - from django.conf import settings @@ -1192,6 +1203,11 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, except django.db.utils.OperationalError: call_command("createcachetable", verbosity=0) + + with connection.cursor() as cursor: + config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] + config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] + config['SQLITE_EXTENSIONS'] = ['JSON1'] if ('ENABLE_JSON1',) in cursor.execute('PRAGMA compile_options;').fetchall() else [] # if archivebox gets imported multiple times, we have to close # the sqlite3 whenever we init from scratch to avoid multiple threads From ca16c88a3dfb152734627781665f945a0ae915f5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:24:58 -0700 Subject: [PATCH 077/124] show PUID, PGID, ENFORCE_ATOMIC_WRITES, and OUTPUT_PERMISSIONS in version output header --- archivebox/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/archivebox/main.py b/archivebox/main.py index 12016dbc..f3360db2 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -71,7 +71,11 @@ from .config import ( IS_TTY, DEBUG, IN_DOCKER, + PUID, + PGID, USER, + ENFORCE_ATOMIC_WRITES, + OUTPUT_PERMISSIONS, PYTHON_BINARY, ARCHIVEBOX_BINARY, ONLY_NEW, @@ -210,6 +214,7 @@ def version(quiet: bool=False, else: # ArchiveBox v0.5.6 # Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) + # IN_DOCKER=True USER=501:20 DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND_ENGINE=ripgrep print('ArchiveBox v{}'.format(VERSION)) p = platform.uname() print( @@ -223,6 +228,8 @@ def version(quiet: bool=False, f'DEBUG={DEBUG}', f'IS_TTY={IS_TTY}', f'TZ={os.environ.get("TZ", "UTC")}', + f'DB=SQLite v{CONFIG["SQLITE_VERSION"]} ({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"])', + f'FS={"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) print() From 0c7d7deb3235719ebbee336754c76c2c97feda60 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:26:42 -0700 Subject: [PATCH 078/124] add missing brace --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index f3360db2..32832580 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -228,7 +228,7 @@ def version(quiet: bool=False, f'DEBUG={DEBUG}', f'IS_TTY={IS_TTY}', f'TZ={os.environ.get("TZ", "UTC")}', - f'DB=SQLite v{CONFIG["SQLITE_VERSION"]} ({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"])', + f'DB=SQLite v{CONFIG["SQLITE_VERSION"]} ({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"]})', f'FS={"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) From d0f129295f95da3728434bae7ea1257edcb93e80 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:29:53 -0700 Subject: [PATCH 079/124] move sqlite3 checks up a level --- archivebox/config.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 28efcee9..6afd9c4a 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1203,12 +1203,6 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, except django.db.utils.OperationalError: call_command("createcachetable", verbosity=0) - - with connection.cursor() as cursor: - config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] - config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] - config['SQLITE_EXTENSIONS'] = ['JSON1'] if ('ENABLE_JSON1',) in cursor.execute('PRAGMA compile_options;').fetchall() else [] - # if archivebox gets imported multiple times, we have to close # the sqlite3 whenever we init from scratch to avoid multiple threads # sharing the same connection by accident @@ -1220,5 +1214,10 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, assert sql_index_path.exists(), ( f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)') + with connection.cursor() as cursor: + config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] + config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] + config['SQLITE_EXTENSIONS'] = ['JSON1'] if ('ENABLE_JSON1',) in cursor.execute('PRAGMA compile_options;').fetchall() else [] + except KeyboardInterrupt: raise SystemExit(2) From e06717419c2e93bf7bf7824787812b8b40aa3ccb Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:35:31 -0700 Subject: [PATCH 080/124] fix sqlite3 version detection --- archivebox/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 6afd9c4a..d6cbe54a 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -26,11 +26,12 @@ import io import re import sys import json +import inspect import getpass import platform import shutil -import sqlite3 import django +from sqlite3 import dbapi2 as sqlite3 from hashlib import md5 from pathlib import Path @@ -356,8 +357,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'DJANGO_BINARY': {'default': lambda c: django.__file__.replace('__init__.py', 'bin/django-admin.py')}, 'DJANGO_VERSION': {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)}, - 'SQLITE_BINARY': {'default': lambda c: 'sqlite3'}, - 'SQLITE_VERSION': {'default': lambda c: None}, + 'SQLITE_BINARY': {'default': lambda c: inspect.getfile(sqlite3)}, + 'SQLITE_VERSION': {'default': lambda c: sqlite3.version_info}, 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, 'SQLITE_EXTENSIONS': {'default': lambda c: []}, From 3e3c011f86b6c4ba2cbebc13c62a78936e6d2832 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:40:48 -0700 Subject: [PATCH 081/124] enforce UTC timezone on server --- archivebox/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index d6cbe54a..84e0e4d1 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -96,7 +96,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'}, 'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40}, 'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None}, - 'TIME_ZONE': {'type': str, 'default': 'UTC'}, + 'TIMEZONE': {'type': str, 'default': 'UTC'}, 'PREVIEW_ORIGINALS': {'type': bool, 'default': True}, }, @@ -989,7 +989,8 @@ globals().update(CONFIG) # Set timezone to UTC and umask to OUTPUT_PERMISSIONS -os.environ["TZ"] = 'UTC' +assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC' # we may allow this to change later +os.environ["TZ"] = TIMEZONE os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8)) # noqa: F821 # add ./node_modules/.bin to $PATH so we can use node scripts in extractors From 1fd5830f58e7cdf6dbb4510c8adce5a179556b43 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:41:22 -0700 Subject: [PATCH 082/124] enforce UTC timezone on server --- archivebox/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 32832580..f11244cb 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -74,6 +74,7 @@ from .config import ( PUID, PGID, USER, + TIMEZONE, ENFORCE_ATOMIC_WRITES, OUTPUT_PERMISSIONS, PYTHON_BINARY, @@ -209,6 +210,8 @@ def version(quiet: bool=False, out_dir: Path=OUTPUT_DIR) -> None: """Print the ArchiveBox version and dependency information""" + from django.conf import settings + if quiet: print(VERSION) else: @@ -227,8 +230,8 @@ def version(quiet: bool=False, f'IN_DOCKER={IN_DOCKER}', f'DEBUG={DEBUG}', f'IS_TTY={IS_TTY}', - f'TZ={os.environ.get("TZ", "UTC")}', - f'DB=SQLite v{CONFIG["SQLITE_VERSION"]} ({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"]})', + f'TZ={TIMEZONE}', + f'DB={settings.DATABASES["default"]["engine"]} (({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"]})', f'FS={"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) From f9c82841ad5bbe116a5e73070490b28276665d26 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:58:15 -0700 Subject: [PATCH 083/124] fix sqlite option detection --- archivebox/config.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 84e0e4d1..fef08fce 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -359,8 +359,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'SQLITE_BINARY': {'default': lambda c: inspect.getfile(sqlite3)}, 'SQLITE_VERSION': {'default': lambda c: sqlite3.version_info}, - 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, - 'SQLITE_EXTENSIONS': {'default': lambda c: []}, + 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, # set at runtime below + 'SQLITE_OPTIONS': {'default': lambda c: []}, # set at runtime below 'USE_CURL': {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])}, 'CURL_VERSION': {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None}, @@ -861,7 +861,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue: 'version': config['SQLITE_VERSION'], 'hash': bin_hash(config['SQLITE_BINARY']), 'enabled': True, - 'is_valid': bool(config['SQLITE_VERSION']) and ('JSON1' in config['SQLITE_EXTENSIONS']), + 'is_valid': bool(config['SQLITE_VERSION']) and ('ENABLE_JSON1' in config['SQLITE_OPTIONS']), }, 'CURL_BINARY': { 'path': bin_path(config['CURL_BINARY']), @@ -1004,6 +1004,11 @@ if not CONFIG['CHECK_SSL_VALIDITY']: requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +# get SQLite database version, compile options, and runtime options +with sqlite3.connect(':memory:').cursor() as cursor: + config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] + config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] + config['SQLITE_OPTIONS'] = [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] ########################### Config Validity Checkers ########################### @@ -1099,6 +1104,7 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None: stderr(' If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:') stderr(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_media') stderr() + def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=CONFIG) -> None: output_dir = out_dir or config['OUTPUT_DIR'] @@ -1216,10 +1222,5 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, assert sql_index_path.exists(), ( f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)') - with connection.cursor() as cursor: - config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] - config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] - config['SQLITE_EXTENSIONS'] = ['JSON1'] if ('ENABLE_JSON1',) in cursor.execute('PRAGMA compile_options;').fetchall() else [] - except KeyboardInterrupt: raise SystemExit(2) From c752c7053dbac257cd1fdb6edfadbe50dc69c842 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 18:59:08 -0700 Subject: [PATCH 084/124] Update main.py --- archivebox/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index f11244cb..55b9ae7b 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -209,8 +209,6 @@ def help(out_dir: Path=OUTPUT_DIR) -> None: def version(quiet: bool=False, out_dir: Path=OUTPUT_DIR) -> None: """Print the ArchiveBox version and dependency information""" - - from django.conf import settings if quiet: print(VERSION) @@ -226,13 +224,14 @@ def version(quiet: bool=False, platform.platform(), p.machine, ) + fs_is_mount = os.path.ismount(ARCHIVE_DIR) print( f'IN_DOCKER={IN_DOCKER}', f'DEBUG={DEBUG}', f'IS_TTY={IS_TTY}', f'TZ={TIMEZONE}', - f'DB={settings.DATABASES["default"]["engine"]} (({CONFIG["SQLITE_JOURNAL_MODE"]} {CONFIG["SQLITE_EXTENSIONS"]})', - f'FS={"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', + f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', + f'FS={"remote" if fs_is_mount else "local" {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) print() From 6b019da3e91e9d5a24563b09cfef616750b56094 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:01:55 -0700 Subject: [PATCH 085/124] Update config.py --- archivebox/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index fef08fce..a94c94c9 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1005,10 +1005,11 @@ if not CONFIG['CHECK_SSL_VALIDITY']: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # get SQLite database version, compile options, and runtime options -with sqlite3.connect(':memory:').cursor() as cursor: - config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] - config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] - config['SQLITE_OPTIONS'] = [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] +cursor = sqlite3.connect(':memory:').cursor() +config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] +config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] +config['SQLITE_OPTIONS'] = [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] +cursor.close() ########################### Config Validity Checkers ########################### From 31d5fbbf17edb8de06ca2afa92ad00b18a01d7f0 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:04:06 -0700 Subject: [PATCH 086/124] Update config.py --- archivebox/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index a94c94c9..7cecc7d2 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1006,9 +1006,9 @@ if not CONFIG['CHECK_SSL_VALIDITY']: # get SQLite database version, compile options, and runtime options cursor = sqlite3.connect(':memory:').cursor() -config['SQLITE_VERSION'] = cursor.execute("SELECT sqlite_version();").fetchone()[0] -config['SQLITE_JOURNAL_MODE'] = cursor.execute('PRAGMA journal_mode;').fetchone()[0] -config['SQLITE_OPTIONS'] = [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] +DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0] +DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0] +DYNAMIC_CONFIG_SCHEMA['SQLITE_OPTIONS'] = lambda c: [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] cursor.close() ########################### Config Validity Checkers ########################### From 19b88d30b2976b92fb1b8f699d2ab6af1f736e9d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:06:03 -0700 Subject: [PATCH 087/124] fix missing brace --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index 55b9ae7b..c319e9d1 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -231,7 +231,7 @@ def version(quiet: bool=False, f'IS_TTY={IS_TTY}', f'TZ={TIMEZONE}', f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', - f'FS={"remote" if fs_is_mount else "local" {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', + f'FS={"remote" if fs_is_mount else "local"} {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) print() From 5509b5cd8be09660473ddc2986e8f0ffeff354e3 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:08:33 -0700 Subject: [PATCH 088/124] Update main.py --- archivebox/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index c319e9d1..2b827e8c 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -230,8 +230,8 @@ def version(quiet: bool=False, f'DEBUG={DEBUG}', f'IS_TTY={IS_TTY}', f'TZ={TIMEZONE}', - f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', - f'FS={"remote" if fs_is_mount else "local"} {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID} ({OUTPUT_PERMISSIONS})', + #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', # add this if we have more useful info to show eventually + f'FS={"remote" if fs_is_mount else "local"} {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID};({OUTPUT_PERMISSIONS})', f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', ) print() From f9c580894090cea2744b3d3d35a65705c48c7aad Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:09:11 -0700 Subject: [PATCH 089/124] Update config.py --- archivebox/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 7cecc7d2..61562d7e 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -861,7 +861,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue: 'version': config['SQLITE_VERSION'], 'hash': bin_hash(config['SQLITE_BINARY']), 'enabled': True, - 'is_valid': bool(config['SQLITE_VERSION']) and ('ENABLE_JSON1' in config['SQLITE_OPTIONS']), + 'is_valid': bool(config['SQLITE_VERSION']), }, 'CURL_BINARY': { 'path': bin_path(config['CURL_BINARY']), @@ -1006,7 +1006,7 @@ if not CONFIG['CHECK_SSL_VALIDITY']: # get SQLite database version, compile options, and runtime options cursor = sqlite3.connect(':memory:').cursor() -DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0] +#DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0] DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0] DYNAMIC_CONFIG_SCHEMA['SQLITE_OPTIONS'] = lambda c: [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] cursor.close() From 0c6d4c82c32a361eeb11600bf636f82d8e48dcab Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:11:02 -0700 Subject: [PATCH 090/124] Update config.py --- archivebox/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index 61562d7e..b9b2c275 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -358,7 +358,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'DJANGO_VERSION': {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)}, 'SQLITE_BINARY': {'default': lambda c: inspect.getfile(sqlite3)}, - 'SQLITE_VERSION': {'default': lambda c: sqlite3.version_info}, + 'SQLITE_VERSION': {'default': lambda c: sqlite3.version}, 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, # set at runtime below 'SQLITE_OPTIONS': {'default': lambda c: []}, # set at runtime below @@ -1005,6 +1005,7 @@ if not CONFIG['CHECK_SSL_VALIDITY']: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # get SQLite database version, compile options, and runtime options +# TODO: move this somewhere better, e.g. setup_django cursor = sqlite3.connect(':memory:').cursor() #DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0] DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0] From dd29e1bf78d7ef9577e9c2eac56c9eea62897739 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:46:09 -0700 Subject: [PATCH 091/124] clean up first line of CLI version output for easier downstream parsing --- archivebox/main.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 2b827e8c..3c6c3084 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -210,29 +210,41 @@ def version(quiet: bool=False, out_dir: Path=OUTPUT_DIR) -> None: """Print the ArchiveBox version and dependency information""" - if quiet: + print(VERSION) + + if not quiet: + # 0.5.6 + # ArchiveBox v0.5.6 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) + # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep + # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep print(VERSION) - else: - # ArchiveBox v0.5.6 - # Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) - # IN_DOCKER=True USER=501:20 DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND_ENGINE=ripgrep - print('ArchiveBox v{}'.format(VERSION)) + + COMMIT_HASH = None + try: + COMMIT_HASH = list(Path('.git/refs/heads/').glob('*'))[0].read_text() or list((SOURCE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() + except Exception: + pass + p = platform.uname() print( + 'ArchiveBox v{}'.format(VERSION), + *((COMMIT_HASH[7:],) if COMMIT_HASH else ()), sys.implementation.name.title(), p.system, platform.platform(), p.machine, ) - fs_is_mount = os.path.ismount(ARCHIVE_DIR) + OUTPUT_IS_REMOTE_FS = bool(os.path.ismount(OUTPUT_DIR) or os.path.ismount(ARCHIVE_DIR)) print( - f'IN_DOCKER={IN_DOCKER}', f'DEBUG={DEBUG}', + f'IN_DOCKER={IN_DOCKER}', f'IS_TTY={IS_TTY}', f'TZ={TIMEZONE}', #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', # add this if we have more useful info to show eventually - f'FS={"remote" if fs_is_mount else "local"} {"atomic" if ENFORCE_ATOMIC_WRITES else "non-atomic"} {PUID}:{PGID};({OUTPUT_PERMISSIONS})', - f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}', + f'FS_ATOMIC={ENFORCE_ATOMIC_WRITES}', + f'FS_REMOTE={OUTPUT_IS_REMOTE_FS}', + f'FS_PERMS={OUTPUT_PERMISSIONS} {PUID}:{PGID}', + f'SEARCH_BACKEND={SEARCH_BACKEND_ENGINE}', ) print() From 33ec2117e963c5fd39c949d0cad658a166ea93b5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:50:45 -0700 Subject: [PATCH 092/124] Update main.py --- archivebox/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 3c6c3084..1011e033 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -216,13 +216,13 @@ def version(quiet: bool=False, # 0.5.6 # ArchiveBox v0.5.6 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep - # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep - print(VERSION) + # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrepw COMMIT_HASH = None try: - COMMIT_HASH = list(Path('.git/refs/heads/').glob('*'))[0].read_text() or list((SOURCE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() - except Exception: + COMMIT_HASH = list((SOURCE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() + except Exception as e: + print(e) pass p = platform.uname() From 413aa2ef04da90013fe97731aca2a1fe0f8cdb47 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:51:46 -0700 Subject: [PATCH 093/124] fix commit hash detection --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index 1011e033..f8a781e8 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -220,7 +220,7 @@ def version(quiet: bool=False, COMMIT_HASH = None try: - COMMIT_HASH = list((SOURCE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() + COMMIT_HASH = list((PACKAGE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() except Exception as e: print(e) pass From cb3ebbe69ab35b1386182e40dde60bd479c17c80 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:52:48 -0700 Subject: [PATCH 094/124] fix git commit hash location --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index f8a781e8..86d2d172 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -220,7 +220,7 @@ def version(quiet: bool=False, COMMIT_HASH = None try: - COMMIT_HASH = list((PACKAGE_DIR / '.git/refs/heads/').glob('*'))[0].read_text() + COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text() except Exception as e: print(e) pass From ae5c8f2bf8ac701dd49b01d06efc28756dc936ff Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 19:57:38 -0700 Subject: [PATCH 095/124] fix newline included in commit hash --- archivebox/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 86d2d172..beeb9e84 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -220,15 +220,14 @@ def version(quiet: bool=False, COMMIT_HASH = None try: - COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text() + COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip() except Exception as e: - print(e) pass p = platform.uname() print( 'ArchiveBox v{}'.format(VERSION), - *((COMMIT_HASH[7:],) if COMMIT_HASH else ()), + *((COMMIT_HASH[:7],) if COMMIT_HASH else ()), sys.implementation.name.title(), p.system, platform.platform(), @@ -251,6 +250,10 @@ def version(quiet: bool=False, print('{white}[i] Dependency versions:{reset}'.format(**ANSI)) for name, dependency in DEPENDENCIES.items(): print(printable_dependency_version(name, dependency)) + + # add a newline between core dependencies and extractor dependencies for easier reading + if 'sqlite' in name.lower(): + print() print() print('{white}[i] Source-code locations:{reset}'.format(**ANSI)) From 375ba9d13571363a440e64cef70e42984853e123 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:00:29 -0700 Subject: [PATCH 096/124] Update settings.py --- archivebox/core/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 550c6077..ddcd632c 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -19,7 +19,7 @@ from ..config import ( SQL_INDEX_FILENAME, OUTPUT_DIR, LOGS_DIR, - TIME_ZONE, + TIMEZONE, ) IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3] @@ -154,7 +154,7 @@ DATABASES = { 'timeout': 60, 'check_same_thread': False, }, - 'TIME_ZONE': 'UTC', + 'TIME_ZONE': TIMEZONE, # DB setup is sometimes modified at runtime by setup_django() in config.py } } @@ -224,7 +224,8 @@ USE_L10N = True USE_TZ = True DATETIME_FORMAT = 'Y-m-d g:iA' SHORT_DATETIME_FORMAT = 'Y-m-d h:iA' -TIME_ZONE = TIME_ZONE # noqa +TIME_ZONE = TIMEZONE # django convention is TIME_ZONE, archivebox config uses TIMEZONE, they are equivalent + from django.conf.locale.en import formats as en_formats From c78a2edc42092e1bf90ade4cdd222e3758ef800b Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:04:01 -0700 Subject: [PATCH 097/124] add is_mount and COMMIT_HASH to config.py --- archivebox/config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index b9b2c275..9dbec950 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -49,6 +49,9 @@ from .config_stubs import ( ConfigDefaultDict, ) + +### Pre-Fetch Minimal System Config + SYSTEM_USER = getpass.getuser() or os.getlogin() try: @@ -58,6 +61,13 @@ except ModuleNotFoundError: # pwd is only needed for some linux systems, doesn't exist on windows pass +COMMIT_HASH = None +try: + COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip() +except Exception as e: + pass + + ############################### Config Schema ################################## CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { @@ -805,6 +815,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': config['OUTPUT_DIR'].resolve(), 'enabled': True, 'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(), + 'is_mount': os.path.ismount(OUTPUT_DIR), }, 'SOURCES_DIR': { 'path': config['SOURCES_DIR'].resolve(), @@ -820,6 +831,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': config['ARCHIVE_DIR'].resolve(), 'enabled': True, 'is_valid': config['ARCHIVE_DIR'].exists(), + 'is_mount': os.path.ismount(ARCHIVE_DIR), }, 'CONFIG_FILE': { 'path': config['CONFIG_FILE'].resolve(), @@ -830,6 +842,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve(), 'enabled': True, 'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(), + 'is_mount': os.path.ismount(config['OUTPUT_DIR'] / SQL_INDEX_FILENAME), }, } From 9f90a2d60d9dc2f8d69704fc647eaf70e796bd0d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:12:55 -0700 Subject: [PATCH 098/124] disable unused sqlite3 stuff --- archivebox/config.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 9dbec950..adb41f69 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -364,13 +364,13 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'PYTHON_ENCODING': {'default': lambda c: sys.stdout.encoding.upper()}, 'PYTHON_VERSION': {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])}, - 'DJANGO_BINARY': {'default': lambda c: django.__file__.replace('__init__.py', 'bin/django-admin.py')}, + 'DJANGO_BINARY': {'default': lambda c: inspect.getfile(django)}, 'DJANGO_VERSION': {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)}, 'SQLITE_BINARY': {'default': lambda c: inspect.getfile(sqlite3)}, 'SQLITE_VERSION': {'default': lambda c: sqlite3.version}, - 'SQLITE_JOURNAL_MODE': {'default': lambda c: None}, # set at runtime below - 'SQLITE_OPTIONS': {'default': lambda c: []}, # set at runtime below + #'SQLITE_JOURNAL_MODE': {'default': lambda c: 'wal'}, # set at runtime below + #'SQLITE_OPTIONS': {'default': lambda c: ['JSON1']}, # set at runtime below 'USE_CURL': {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])}, 'CURL_VERSION': {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None}, @@ -1010,6 +1010,14 @@ os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8)) # noqa: F821 NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin')) sys.path.append(NODE_BIN_PATH) +# OPTIONAL: also look around the host system for node modules to use +# avoid enabling this unless absolutely needed, +# having overlapping potential sources of libs is a big source of bugs/confusing to users +# DEV_NODE_BIN_PATH = str((Path(CONFIG["PACKAGE_DIR"]).absolute() / '..' / 'node_modules' / '.bin')) +# sys.path.append(DEV_NODE_BIN_PATH) +# USER_NODE_BIN_PATH = str(Path('~/.node_modules/.bin').resolve()) +# sys.path.append(USER_NODE_BIN_PATH) + # disable stderr "you really shouldnt disable ssl" warnings with library config if not CONFIG['CHECK_SSL_VALIDITY']: import urllib3 @@ -1018,12 +1026,12 @@ if not CONFIG['CHECK_SSL_VALIDITY']: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # get SQLite database version, compile options, and runtime options -# TODO: move this somewhere better, e.g. setup_django -cursor = sqlite3.connect(':memory:').cursor() +# TODO: make this a less hacky proper assertion checker helper function in somewhere like setup_django +#cursor = sqlite3.connect(':memory:').cursor() #DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0] -DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0] -DYNAMIC_CONFIG_SCHEMA['SQLITE_OPTIONS'] = lambda c: [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] -cursor.close() +#DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0] +#DYNAMIC_CONFIG_SCHEMA['SQLITE_OPTIONS'] = lambda c: [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()] +#cursor.close() ########################### Config Validity Checkers ########################### From ccce4a6a2f467ca70961cadb42b4603d23a3db26 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:13:22 -0700 Subject: [PATCH 099/124] use new is_mount and COMMIT_HASH config options --- archivebox/main.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index beeb9e84..1c53b4cc 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -96,6 +96,7 @@ from .config import ( check_data_folder, write_config_file, VERSION, + COMMIT_HASH, CODE_LOCATIONS, EXTERNAL_LOCATIONS, DATA_LOCATIONS, @@ -213,16 +214,9 @@ def version(quiet: bool=False, print(VERSION) if not quiet: - # 0.5.6 - # ArchiveBox v0.5.6 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) + # 0.6.3 + # ArchiveBox v0.6.3 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY) # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep - # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrepw - - COMMIT_HASH = None - try: - COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip() - except Exception as e: - pass p = platform.uname() print( @@ -233,7 +227,7 @@ def version(quiet: bool=False, platform.platform(), p.machine, ) - OUTPUT_IS_REMOTE_FS = bool(os.path.ismount(OUTPUT_DIR) or os.path.ismount(ARCHIVE_DIR)) + OUTPUT_IS_REMOTE_FS = DATA_LOCATIONS['OUTPUT_DIR']['is_mount'] or DATA_LOCATIONS['ARCHIVE_DIR']['is_mount'] print( f'DEBUG={DEBUG}', f'IN_DOCKER={IN_DOCKER}', From e2fa68dba636a6c702af141259960a99b8406c7e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:15:22 -0700 Subject: [PATCH 100/124] resolve config paths before using --- archivebox/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index adb41f69..af6458e0 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -815,7 +815,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': config['OUTPUT_DIR'].resolve(), 'enabled': True, 'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(), - 'is_mount': os.path.ismount(OUTPUT_DIR), + 'is_mount': os.path.ismount(config['OUTPUT_DIR'].resolve()), }, 'SOURCES_DIR': { 'path': config['SOURCES_DIR'].resolve(), @@ -831,7 +831,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': config['ARCHIVE_DIR'].resolve(), 'enabled': True, 'is_valid': config['ARCHIVE_DIR'].exists(), - 'is_mount': os.path.ismount(ARCHIVE_DIR), + 'is_mount': os.path.ismount(onfig['ARCHIVE_DIR'].resolve()), }, 'CONFIG_FILE': { 'path': config['CONFIG_FILE'].resolve(), @@ -842,7 +842,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve(), 'enabled': True, 'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(), - 'is_mount': os.path.ismount(config['OUTPUT_DIR'] / SQL_INDEX_FILENAME), + 'is_mount': os.path.ismount((config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve()), }, } From 2bbc742017e1c6f9c74914681c3815ba3e5ffdaf Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:16:08 -0700 Subject: [PATCH 101/124] typo fix --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index af6458e0..cf9aa3ea 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -831,7 +831,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'path': config['ARCHIVE_DIR'].resolve(), 'enabled': True, 'is_valid': config['ARCHIVE_DIR'].exists(), - 'is_mount': os.path.ismount(onfig['ARCHIVE_DIR'].resolve()), + 'is_mount': os.path.ismount(config['ARCHIVE_DIR'].resolve()), }, 'CONFIG_FILE': { 'path': config['CONFIG_FILE'].resolve(), From 01555dfe345bafd43c32b88e20d7af007ed06b8f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:17:31 -0700 Subject: [PATCH 102/124] Update main.py --- archivebox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/main.py b/archivebox/main.py index 1c53b4cc..0499f73f 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -246,7 +246,7 @@ def version(quiet: bool=False, print(printable_dependency_version(name, dependency)) # add a newline between core dependencies and extractor dependencies for easier reading - if 'sqlite' in name.lower(): + if name == 'ARCHIVEBOX_BINARY': print() print() From 319ea481b8541996f86660b5e3ab84bfc9eb2fd5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:17:38 -0700 Subject: [PATCH 103/124] Update config.py --- archivebox/config.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index cf9aa3ea..cfe41b53 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -848,13 +848,6 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: def get_dependency_info(config: ConfigDict) -> ConfigValue: return { - 'ARCHIVEBOX_BINARY': { - 'path': bin_path(config['ARCHIVEBOX_BINARY']), - 'version': config['VERSION'], - 'hash': bin_hash(config['ARCHIVEBOX_BINARY']), - 'enabled': True, - 'is_valid': True, - }, 'PYTHON_BINARY': { 'path': bin_path(config['PYTHON_BINARY']), 'version': config['PYTHON_VERSION'], @@ -862,13 +855,6 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue: 'enabled': True, 'is_valid': bool(config['PYTHON_VERSION']), }, - 'DJANGO_BINARY': { - 'path': bin_path(config['DJANGO_BINARY']), - 'version': config['DJANGO_VERSION'], - 'hash': bin_hash(config['DJANGO_BINARY']), - 'enabled': True, - 'is_valid': bool(config['DJANGO_VERSION']), - }, 'SQLITE_BINARY': { 'path': bin_path(config['SQLITE_BINARY']), 'version': config['SQLITE_VERSION'], @@ -876,6 +862,21 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue: 'enabled': True, 'is_valid': bool(config['SQLITE_VERSION']), }, + 'DJANGO_BINARY': { + 'path': bin_path(config['DJANGO_BINARY']), + 'version': config['DJANGO_VERSION'], + 'hash': bin_hash(config['DJANGO_BINARY']), + 'enabled': True, + 'is_valid': bool(config['DJANGO_VERSION']), + }, + 'ARCHIVEBOX_BINARY': { + 'path': bin_path(config['ARCHIVEBOX_BINARY']), + 'version': config['VERSION'], + 'hash': bin_hash(config['ARCHIVEBOX_BINARY']), + 'enabled': True, + 'is_valid': True, + }, + 'CURL_BINARY': { 'path': bin_path(config['CURL_BINARY']), 'version': config['CURL_VERSION'], From d586a8babc396aac5d3aa1ed9afdc24d71921343 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 8 Jun 2022 20:22:58 -0700 Subject: [PATCH 104/124] show mount paths with at symbol in version output --- archivebox/logging_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index e00d4cbb..49ee12d7 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -572,7 +572,7 @@ def printable_config(config: ConfigDict, prefix: str='') -> str: def printable_folder_status(name: str, folder: Dict) -> str: if folder['enabled']: if folder['is_valid']: - color, symbol, note = 'green', '√', 'valid' + color, symbol, note, num_files = 'green', '√', 'valid', '' else: color, symbol, note, num_files = 'red', 'X', 'invalid', '?' else: @@ -587,6 +587,10 @@ def printable_folder_status(name: str, folder: Dict) -> str: ) else: num_files = 'missing' + + if folder.get('is_mount'): + # add symbol @ next to filecount if path is a remote filesystem mount + num_files = f'{num_files} @' if num_files else '@' path = str(folder['path']).replace(str(OUTPUT_DIR), '.') if folder['path'] else '' if path and ' ' in path: From 8899fe0b9259748da2ef19d37028c317f39f37d3 Mon Sep 17 00:00:00 2001 From: renaisun <43090234+renaisun@users.noreply.github.com> Date: Thu, 9 Jun 2022 14:35:48 +0800 Subject: [PATCH 105/124] Add SINGLEFILE_ARGS to control single-file arguments --- archivebox/config.py | 2 ++ archivebox/extractors/singlefile.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/archivebox/config.py b/archivebox/config.py index cfe41b53..bd3a1688 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -177,6 +177,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { '--compressed' ]}, 'GIT_ARGS': {'type': list, 'default': ['--recursive']}, + 'SINGLEFILE_ARGS': {'type': list, 'default' : None} }, 'SEARCH_BACKEND_CONFIG' : { @@ -391,6 +392,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'USE_SINGLEFILE': {'default': lambda c: c['USE_SINGLEFILE'] and c['SAVE_SINGLEFILE']}, 'SINGLEFILE_VERSION': {'default': lambda c: bin_version(c['SINGLEFILE_BINARY']) if c['USE_SINGLEFILE'] else None}, + 'SINGLEFILE_ARGS': {'default': lambda c: c['SINGLEFILE_ARGS'] or []}, 'USE_READABILITY': {'default': lambda c: c['USE_READABILITY'] and c['SAVE_READABILITY']}, 'READABILITY_VERSION': {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None}, diff --git a/archivebox/extractors/singlefile.py b/archivebox/extractors/singlefile.py index 3279960e..80ad90b1 100644 --- a/archivebox/extractors/singlefile.py +++ b/archivebox/extractors/singlefile.py @@ -17,6 +17,7 @@ from ..config import ( SAVE_SINGLEFILE, DEPENDENCIES, SINGLEFILE_VERSION, + SINGLEFILE_ARGS, CHROME_BINARY, ) from ..logging_util import TimedProgress @@ -47,6 +48,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:])) cmd = [ DEPENDENCIES['SINGLEFILE_BINARY']['path'], + *SINGLEFILE_ARGS, '--browser-executable-path={}'.format(CHROME_BINARY), browser_args, link.url, From 03eb7e58758d8dcb85ed781e713fc083f8292264 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 9 Jun 2022 01:04:55 -0700 Subject: [PATCH 106/124] Update config.py --- archivebox/config.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index cfe41b53..9744cd16 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -61,13 +61,6 @@ except ModuleNotFoundError: # pwd is only needed for some linux systems, doesn't exist on windows pass -COMMIT_HASH = None -try: - COMMIT_HASH = list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip() -except Exception as e: - pass - - ############################### Config Schema ################################## CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { @@ -334,6 +327,15 @@ ALLOWED_IN_OUTPUT_DIR = { 'static_index.json', } +def get_version(config): + return json.loads((Path(config['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version'] + +def get_commit_hash(config): + try: + return list((config['PACKAGE_DIR'] / '../.git/refs/heads/').glob('*'))[0].read_text().strip() + except Exception: + return None + ############################## Derived Config ################################## @@ -358,8 +360,9 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'DIR_OUTPUT_PERMISSIONS': {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')}, 'ARCHIVEBOX_BINARY': {'default': lambda c: sys.argv[0] or bin_path('archivebox')}, - 'VERSION': {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version']}, - + 'VERSION': {'default': lambda c: get_version(c)}, + 'COMMIT_HASH': {'default': lambda c: get_commit_hash(c)}, + 'PYTHON_BINARY': {'default': lambda c: sys.executable}, 'PYTHON_ENCODING': {'default': lambda c: sys.stdout.encoding.upper()}, 'PYTHON_VERSION': {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])}, @@ -369,7 +372,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'SQLITE_BINARY': {'default': lambda c: inspect.getfile(sqlite3)}, 'SQLITE_VERSION': {'default': lambda c: sqlite3.version}, - #'SQLITE_JOURNAL_MODE': {'default': lambda c: 'wal'}, # set at runtime below + #'SQLITE_JOURNAL_MODE': {'default': lambda c: 'wal'}, # set at runtime below, interesting but unused for now #'SQLITE_OPTIONS': {'default': lambda c: ['JSON1']}, # set at runtime below 'USE_CURL': {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])}, From 2b58cce43fca64865292ccb967b8800a421e05cd Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Sun, 11 Sep 2022 12:19:16 +0200 Subject: [PATCH 107/124] Attempted to warn on #984 and #1014 --- archivebox/extractors/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index dc1b9692..ce2ff365 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -7,6 +7,7 @@ from typing import Optional, List, Iterable, Union from datetime import datetime, timezone from django.db.models import QuerySet +from ..core.settings import ERROR_LOG from ..index.schema import Link from ..index.sql import write_link_to_sql_index from ..index import ( @@ -127,10 +128,25 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s # print('{black} X {}{reset}'.format(method_name, **ANSI)) stats['skipped'] += 1 except Exception as e: + # Disabled until https://github.com/ArchiveBox/ArchiveBox/issues/984 + # and https://github.com/ArchiveBox/ArchiveBox/issues/1014 + # are fixed. + """ raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format( method_name, link.url, )) from e + """ + # Instead, use the kludgy workaround from + # https://github.com/ArchiveBox/ArchiveBox/issues/984#issuecomment-1150541627 + with open(ERROR_LOG, "a", encoding='utf-8') as f: + command = ' '.join(sys.argv) + ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S') + f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={}))'.format( + method_name, + link.url, + ) + "\n")) + #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") # print(' ', stats) From e41f313fa3522c888b63b353cd5622dfbea7e573 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Sun, 11 Sep 2022 12:31:11 +0200 Subject: [PATCH 108/124] Change actions --- .github/workflows/codeql-analysis.yml | 8 +------- .github/workflows/lint.yml | 4 +--- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 66e331b2..0eb2db0e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,12 +1,6 @@ name: "CodeQL" -on: - push: - branches: [ dev ] - pull_request: - branches: [ dev ] - schedule: - - cron: '43 1 * * 2' +on: [push] jobs: analyze: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5a402b25..28375994 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,8 +1,6 @@ name: Run linters -on: - workflow_dispatch: - push: +on: [push] env: MAX_LINE_LENGTH: 110 From cc56785695e25288529380cb0a62b0c962c3b48b Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Sun, 11 Sep 2022 22:11:13 +0200 Subject: [PATCH 109/124] Added standalone dockerfile instructions --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7d422628..4b718230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ # This is the Dockerfile for ArchiveBox, it bundles the following dependencies: # python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file # Usage: +# git submodule update --init --recursive +# git pull --recurse-submodules # docker build . -t archivebox --no-cache # docker run -v "$PWD/data":/data archivebox init # docker run -v "$PWD/data":/data archivebox add 'https://example.com' From 35a052ae8b004221098f6e75075a7aaf2bd3d252 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Sun, 11 Sep 2022 22:13:22 +0200 Subject: [PATCH 110/124] Another comment --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4b718230..5ff97708 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,9 @@ # Multi-arch build: # docker buildx create --use # docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev +# +# Read more about [developing +# Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development). FROM python:3.10-slim-bullseye From 40659b5e9d345309515873f61e07c213f6b21ac8 Mon Sep 17 00:00:00 2001 From: notevenaperson <66701832+notevenaperson@users.noreply.github.com> Date: Sun, 11 Sep 2022 17:23:15 +0000 Subject: [PATCH 111/124] singlefile.py: Code to ensure options are deduplicated --- archivebox/extractors/singlefile.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/archivebox/extractors/singlefile.py b/archivebox/extractors/singlefile.py index 80ad90b1..f29d59c3 100644 --- a/archivebox/extractors/singlefile.py +++ b/archivebox/extractors/singlefile.py @@ -46,11 +46,31 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:])) - cmd = [ - DEPENDENCIES['SINGLEFILE_BINARY']['path'], + options = [ *SINGLEFILE_ARGS, '--browser-executable-path={}'.format(CHROME_BINARY), browser_args, + ] + + # Deduplicate options (single-file doesn't like when you use the same option two times) + # + # NOTE: Options names that come first clobber conflicting names that come later + # My logic is SINGLEFILE_ARGS is the option that affects the singlefile command with most + # specificity, therefore the user sets it with a lot intent, therefore it should take precedence + # kind of like the ergonomic principle of lexical scope in programming languages. + seen_option_names = [] + def test_seen(argument): + option_name = argument.split("=")[0] + if option_name in seen_option_names: + return False + else: + seen_option_names.append(option_name) + return True + deduped_options = list(filter(test_seen, options)) + + cmd = [ + DEPENDENCIES['SINGLEFILE_BINARY']['path'], + *deduped_options link.url, output, ] From 0ea955b3edbb9d8fdf60f40b448b7653ff20ada3 Mon Sep 17 00:00:00 2001 From: renaisun <43090234+renaisun@users.noreply.github.com> Date: Mon, 12 Sep 2022 08:54:15 +0800 Subject: [PATCH 112/124] add a missing comma --- archivebox/extractors/singlefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/extractors/singlefile.py b/archivebox/extractors/singlefile.py index f29d59c3..f7b1b686 100644 --- a/archivebox/extractors/singlefile.py +++ b/archivebox/extractors/singlefile.py @@ -70,7 +70,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO cmd = [ DEPENDENCIES['SINGLEFILE_BINARY']['path'], - *deduped_options + *deduped_options, link.url, output, ] From f5f7aff3b4dec51a001110227946f083897dae72 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 20:34:02 +0000 Subject: [PATCH 113/124] Added yt-dlp everywhere --- .gitignore | 3 +++ Dockerfile | 2 +- README.md | 6 +++--- archivebox/config.py | 3 ++- archivebox/extractors/__init__.py | 1 + archivebox/extractors/media.py | 1 + bin/setup.sh | 8 ++++---- etc/ArchiveBox.conf.default | 2 +- setup.py | 1 + stdeb.cfg | 2 +- 10 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index a80c30ba..f8fefbfb 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ data1/ data2/ data3/ output/ + +# vim +*.sw? diff --git a/Dockerfile b/Dockerfile index 7d422628..e147e56e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # This is the Dockerfile for ArchiveBox, it bundles the following dependencies: -# python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file +# python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file # Usage: # docker build . -t archivebox --no-cache # docker run -v "$PWD/data":/data archivebox init diff --git a/README.md b/README.md index 46427e04..039dd6bb 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ ls ./archive/*/index.json # or browse directly via the filesyste - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up online, stores all data locally - [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) -- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl), articles (readability), code (git), etc.](#output-formats) +- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl or yt-dlp), articles (readability), code (git), etc.](#output-formats) - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats) - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA) @@ -469,7 +469,7 @@ Inside each Snapshot folder, ArchiveBox save these different types of extractor - **DOM Dump:** `output.html` DOM Dump of the HTML after rendering using headless chrome - **Article Text:** `article.html/json` Article text extraction using Readability & Mercury - **Archive.org Permalink:** `archive.org.txt` A link to the saved site on archive.org -- **Audio & Video:** `media/` all audio/video files + playlists, including subtitles & metadata with youtube-dl +- **Audio & Video:** `media/` all audio/video files + playlists, including subtitles & metadata with youtube-dl (or yt-dlp) - **Source Code:** `git/` clone of any repository found on GitHub, Bitbucket, or GitLab links - _More coming soon! See the [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap)..._ @@ -529,7 +529,7 @@ To achieve high fidelity archives in as many situations as possible, ArchiveBox - `node` & `npm` (for readability, mercury, and singlefile) - `wget` (for plain HTML, static files, and WARC saving) - `curl` (for fetching headers, favicon, and posting to Archive.org) -- `youtube-dl` (for audio, video, and subtitles) +- `youtube-dl` or `yt-dlp` (for audio, video, and subtitles) - `git` (for cloning git repos) - and more as we grow... diff --git a/archivebox/config.py b/archivebox/config.py index 9744cd16..4d839805 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -203,7 +203,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'SINGLEFILE_BINARY': {'type': str, 'default': lambda c: bin_path('single-file')}, 'READABILITY_BINARY': {'type': str, 'default': lambda c: bin_path('readability-extractor')}, 'MERCURY_BINARY': {'type': str, 'default': lambda c: bin_path('mercury-parser')}, - 'YOUTUBEDL_BINARY': {'type': str, 'default': 'youtube-dl'}, + #'YOUTUBEDL_BINARY': {'type': str, 'default': 'youtube-dl'}, + 'YOUTUBEDL_BINARY': {'type': str, 'default': 'yt-dlp'}, 'NODE_BINARY': {'type': str, 'default': 'node'}, 'RIPGREP_BINARY': {'type': str, 'default': 'rg'}, 'CHROME_BINARY': {'type': str, 'default': None}, diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index ce2ff365..2f5b3b73 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -1,6 +1,7 @@ __package__ = 'archivebox.extractors' import os +import sys from pathlib import Path from typing import Optional, List, Iterable, Union diff --git a/archivebox/extractors/media.py b/archivebox/extractors/media.py index 1b093e8a..17e7a6a6 100644 --- a/archivebox/extractors/media.py +++ b/archivebox/extractors/media.py @@ -72,6 +72,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME timer.end() # add video description and subtitles to full-text index + # Let's try a few different index_texts = [ text_file.read_text(encoding='utf-8').strip() for text_file in ( diff --git a/bin/setup.sh b/bin/setup.sh index 5f4b4103..37d7937c 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -91,9 +91,9 @@ echo " This is a helper script which installs the ArchiveBox dependencies on echo " You may be prompted for a sudo password in order to install the following:" echo "" echo " - archivebox" -echo " - python3, pip, nodejs, npm (languages used by ArchiveBox, and its extractor modules)" -echo " - curl, wget, git, youtube-dl (used for extracting title, favicon, git, media, and more)" -echo " - chromium (skips this if any Chrome/Chromium version is already installed)" +echo " - python3, pip, nodejs, npm (languages used by ArchiveBox, and its extractor modules)" +echo " - curl, wget, git, youtube-dl, yt-dlp (used for extracting title, favicon, git, media, and more)" +echo " - chromium (skips this if any Chrome/Chromium version is already installed)" echo "" echo " If you'd rather install these manually as-needed, you can find detailed documentation here:" echo " https://github.com/ArchiveBox/ArchiveBox/wiki/Install" @@ -115,7 +115,7 @@ if which apt-get > /dev/null; then fi echo echo "[+] Installing ArchiveBox system dependencies using apt..." - sudo apt-get install -y git python3 python3-pip python3-distutils wget curl youtube-dl ffmpeg git nodejs npm ripgrep + sudo apt-get install -y git python3 python3-pip python3-distutils wget curl youtube-dl yt-dlp ffmpeg git nodejs npm ripgrep sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true sudo apt-get install -y archivebox sudo apt-get --only-upgrade install -y archivebox diff --git a/etc/ArchiveBox.conf.default b/etc/ArchiveBox.conf.default index 982a1931..03048a42 100644 --- a/etc/ArchiveBox.conf.default +++ b/etc/ArchiveBox.conf.default @@ -55,7 +55,7 @@ # CURL_BINARY = curl # GIT_BINARY = git # WGET_BINARY = wget -# YOUTUBEDL_BINARY = youtube-dl +# YOUTUBEDL_BINARY = yt-dlp # CHROME_BINARY = chromium # CHROME_USER_DATA_DIR="~/.config/google-chrome/Default" diff --git a/setup.py b/setup.py index a9d8a509..346d3b62 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ INSTALL_REQUIRES = [ "django-extensions>=3.0.3", "dateparser>=1.0.0", "youtube-dl>=2021.04.17", + "yt-dlp>=2021.4.11", "python-crontab>=2.5.1", "croniter>=0.3.34", "w3lib>=1.22.0", diff --git a/stdeb.cfg b/stdeb.cfg index 6664c6c7..571d4245 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -5,7 +5,7 @@ Package3: archivebox Suite: focal Suite3: focal Build-Depends: debhelper, dh-python, python3-all, python3-pip, python3-setuptools, python3-wheel, python3-stdeb -Depends3: nodejs, wget, curl, git, ffmpeg, youtube-dl, python3-all, python3-pip, python3-setuptools, python3-croniter, python3-crontab, python3-dateparser, python3-django, python3-django-extensions, python3-django-jsonfield, python3-mypy-extensions, python3-requests, python3-w3lib, ripgrep +Depends3: nodejs, wget, curl, git, ffmpeg, youtube-dl, yt-dlp, python3-all, python3-pip, python3-setuptools, python3-croniter, python3-crontab, python3-dateparser, python3-django, python3-django-extensions, python3-django-jsonfield, python3-mypy-extensions, python3-requests, python3-w3lib, ripgrep X-Python3-Version: >= 3.7 XS-Python-Version: >= 3.7 Setup-Env-Vars: DEB_BUILD_OPTIONS=nocheck From dba423a56865e14741c83af94d0222a263f3e6aa Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 20:36:23 +0000 Subject: [PATCH 114/124] A few more youtube-dl tweaks --- archivebox/config.py | 2 ++ archivebox/extractors/media.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 4d839805..d5666e87 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -149,6 +149,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { '--yes-playlist', '--continue', '--ignore-errors', + # This flag doesn't exist in youtube-dl + # only in yt-dlp '--no-abort-on-error', '--geo-bypass', '--add-metadata', diff --git a/archivebox/extractors/media.py b/archivebox/extractors/media.py index 17e7a6a6..c6388a1f 100644 --- a/archivebox/extractors/media.py +++ b/archivebox/extractors/media.py @@ -33,7 +33,7 @@ def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optio @enforce_types def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: - """Download playlists or individual video, audio, and subtitles using youtube-dl""" + """Download playlists or individual video, audio, and subtitles using youtube-dl or yt-dlp""" out_dir = out_dir or Path(link.link_dir) output: ArchiveOutput = 'media' @@ -61,7 +61,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME pass else: hints = ( - 'Got youtube-dl response code: {}.'.format(result.returncode), + 'Got youtube-dl (or yt-dlp) response code: {}.'.format(result.returncode), *result.stderr.decode().split('\n'), ) raise ArchiveError('Failed to save media', hints) From b864c38d9e9eb661bde0c0c267784b71cb8bd95e Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 20:40:45 +0000 Subject: [PATCH 115/124] Don't be strict on unicode errors --- archivebox/extractors/media.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/archivebox/extractors/media.py b/archivebox/extractors/media.py index c6388a1f..7d73024f 100644 --- a/archivebox/extractors/media.py +++ b/archivebox/extractors/media.py @@ -74,7 +74,16 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME # add video description and subtitles to full-text index # Let's try a few different index_texts = [ - text_file.read_text(encoding='utf-8').strip() + # errors: + # * 'strict' to raise a ValueError exception if there is an + # encoding error. The default value of None has the same effect. + # * 'ignore' ignores errors. Note that ignoring encoding errors + # can lead to data loss. + # * 'xmlcharrefreplace' is only supported when writing to a + # file. Characters not supported by the encoding are replaced with + # the appropriate XML character reference &#nnn;. + # There are a few more options described in https://docs.python.org/3/library/functions.html#open + text_file.read_text(encoding='utf-8', errors='xmlcharrefreplace').strip() for text_file in ( *output_path.glob('*.description'), *output_path.glob('*.srt'), From 983f485cc07e4d241718a063e8a835e583670f4e Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 21:29:43 +0000 Subject: [PATCH 116/124] flake8 --- archivebox/extractors/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index 2f5b3b73..3fe58082 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -128,7 +128,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s else: # print('{black} X {}{reset}'.format(method_name, **ANSI)) stats['skipped'] += 1 - except Exception as e: + except Exception: # Disabled until https://github.com/ArchiveBox/ArchiveBox/issues/984 # and https://github.com/ArchiveBox/ArchiveBox/issues/1014 # are fixed. @@ -138,7 +138,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s link.url, )) from e """ - # Instead, use the kludgy workaround from + # Instead, use the kludgy workaround from # https://github.com/ArchiveBox/ArchiveBox/issues/984#issuecomment-1150541627 with open(ERROR_LOG, "a", encoding='utf-8') as f: command = ' '.join(sys.argv) @@ -147,7 +147,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s method_name, link.url, ) + "\n")) - #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") + f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") # print(' ', stats) From daef48e59bb534709aa72627532f46813c9db9df Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 21:31:33 +0000 Subject: [PATCH 117/124] flake8 --- archivebox/extractors/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index 3fe58082..e9d1347b 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -143,11 +143,12 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s with open(ERROR_LOG, "a", encoding='utf-8') as f: command = ' '.join(sys.argv) ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S') - f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={}))'.format( + f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={})) command={}'.format( method_name, link.url, + command ) + "\n")) - f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") + #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") # print(' ', stats) From 081a12b0799b1a585c2274e8a033a2c1151d4248 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 21:32:47 +0000 Subject: [PATCH 118/124] Add ts --- archivebox/extractors/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index e9d1347b..8623a15b 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -143,10 +143,11 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s with open(ERROR_LOG, "a", encoding='utf-8') as f: command = ' '.join(sys.argv) ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S') - f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={})) command={}'.format( + f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={})) command={}; ts={}'.format( method_name, link.url, - command + command, + ts ) + "\n")) #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") From 4ce392846cebb7471c8fc6cb919a48f50d2a4e79 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 21:51:15 +0000 Subject: [PATCH 119/124] Fix actions --- .github/workflows/codeql-analysis.yml | 8 +++++++- .github/workflows/lint.yml | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0eb2db0e..66e331b2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,6 +1,12 @@ name: "CodeQL" -on: [push] +on: + push: + branches: [ dev ] + pull_request: + branches: [ dev ] + schedule: + - cron: '43 1 * * 2' jobs: analyze: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 28375994..5a402b25 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,8 @@ name: Run linters -on: [push] +on: + workflow_dispatch: + push: env: MAX_LINE_LENGTH: 110 From caa8b782fbceeb7913246d4dbb23272d32a2eee5 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Mon, 12 Sep 2022 21:52:01 +0000 Subject: [PATCH 120/124] Remove tab --- bin/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/setup.sh b/bin/setup.sh index 37d7937c..395b43f9 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -91,7 +91,7 @@ echo " This is a helper script which installs the ArchiveBox dependencies on echo " You may be prompted for a sudo password in order to install the following:" echo "" echo " - archivebox" -echo " - python3, pip, nodejs, npm (languages used by ArchiveBox, and its extractor modules)" +echo " - python3, pip, nodejs, npm (languages used by ArchiveBox, and its extractor modules)" echo " - curl, wget, git, youtube-dl, yt-dlp (used for extracting title, favicon, git, media, and more)" echo " - chromium (skips this if any Chrome/Chromium version is already installed)" echo "" From f729bbe122c96fb1fbaf0d0f39cae30aa513b5d0 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Wed, 14 Sep 2022 06:27:58 +0200 Subject: [PATCH 121/124] yt-dlp fixes --- archivebox/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 9744cd16..b6f7e7c1 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -144,12 +144,17 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { '--no-call-home', '--write-sub', '--all-subs', - '--write-auto-sub', + # There are too many of these and youtube + # throttles you with HTTP error 429 + #'--write-auto-sub', '--convert-subs=srt', '--yes-playlist', '--continue', - '--ignore-errors', '--no-abort-on-error', + # --ignore-errors must come AFTER + # --no-abort-on-error + # https://github.com/yt-dlp/yt-dlp/issues/4914 + '--ignore-errors', '--geo-bypass', '--add-metadata', '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']), From 22d8e5763794895cde0fe2880b45f424b24722f4 Mon Sep 17 00:00:00 2001 From: Joseph Turian Date: Wed, 14 Sep 2022 09:36:17 +0000 Subject: [PATCH 122/124] Add missing migration 0021 --- .../core/migrations/0021_auto_20220914_0934.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 archivebox/core/migrations/0021_auto_20220914_0934.py diff --git a/archivebox/core/migrations/0021_auto_20220914_0934.py b/archivebox/core/migrations/0021_auto_20220914_0934.py new file mode 100644 index 00000000..4ef09034 --- /dev/null +++ b/archivebox/core/migrations/0021_auto_20220914_0934.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2022-09-14 09:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0020_auto_20210410_1031'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='extractor', + field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32), + ), + ] From 2db830c6a8d30be951056dce0f1c0881e83611de Mon Sep 17 00:00:00 2001 From: SnZ <3882467+EsEnZeT@users.noreply.github.com> Date: Sun, 20 Nov 2022 01:51:16 +0100 Subject: [PATCH 123/124] Method typo? Fixes '[Errno 2] No such file or directory' error during add --- archivebox/parsers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 992446ba..c033ab28 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -154,7 +154,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: for entry in raw_text.split(): try: - if Path(entry).exists: + if Path(entry).exists(): referenced_texts += Path(entry).read_text() except Exception as err: print(err) From 86208c0267839e99176c827017ddb3957f118859 Mon Sep 17 00:00:00 2001 From: Matthew West Date: Sun, 25 Dec 2022 17:36:01 +0000 Subject: [PATCH 124/124] get setup.sh to run on FreeBSD again (13.x) --- bin/setup.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/setup.sh b/bin/setup.sh index 5f4b4103..e90923e5 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -132,12 +132,13 @@ elif which brew > /dev/null; then echo "[+] Installing ArchiveBox python dependencies using pip3..." python3 -m pip install --upgrade --ignore-installed archivebox elif which pkg > /dev/null; then - echo "[+] Installing ArchiveBox system dependencies using pkg and pip37 (python3.7)..." - sudo pkg install -y python37 py37-pip py37-sqlite3 node npm wget curl youtube_dl ffmpeg git ripgrep + echo "[+] Installing ArchiveBox system dependencies using pkg and pip (python3.9)..." + sudo pkg install -y python3 py39-pip py39-sqlite3 npm wget curl youtube_dl ffmpeg git ripgrep sudo pkg install -y chromium echo "" - echo "[+] Installing ArchiveBox python dependencies using pip37..." - sudo python3 -m pip install --upgrade --ignore-installed archivebox + echo "[+] Installing ArchiveBox python dependencies using pip..." + # don't use sudo here so that pip installs in $HOME/.local instead of into /usr/local + python3 -m pip install --upgrade --ignore-installed archivebox else echo "[!] Warning: Could not find aptitude/homebrew/pkg! May not be able to install all dependencies automatically." echo "" @@ -191,7 +192,7 @@ echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized echo " cd ~/archivebox" echo " ps aux | grep archivebox" echo " pkill -f archivebox" -echo " pip3 install --upgrade archivebox" +echo " python3 -m pip install --upgrade archivebox" echo " archivebox server --quick-init 0.0.0.0:8000" echo " archivebox manage createsuperuser" echo " archivebox add 'https://example.com'"