diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 814b2f5e..97ac7712 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -1,19 +1,31 @@
-from django.contrib import admin
-from django.utils.html import format_html
+__package__ = 'archivebox.core'
+
+from io import StringIO
+from contextlib import redirect_stdout
+
+from django.contrib import admin
+from django.urls import path
+from django.utils.html import format_html
+from django.shortcuts import render
+from django.contrib.auth import get_user_model
-from util import htmldecode, urldecode
from core.models import Snapshot
-from archivebox.logging_util import printable_filesize
+from core.forms import AddLinkForm
+
+from ..util import htmldecode, urldecode, ansi_to_html
+from ..logging_util import printable_filesize
+from ..main import add
+from ..config import OUTPUT_DIR
# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
class SnapshotAdmin(admin.ModelAdmin):
- list_display = ('title_str', 'url_str', 'tags', 'files', 'size', 'added', 'updated')
+ list_display = ('added', 'title_str', 'url_str', 'tags', 'files', 'size', 'updated')
sort_fields = ('title_str', 'url_str', 'tags', 'added', 'updated')
- readonly_fields = ('id', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
+ readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
search_fields = ('url', 'timestamp', 'title', 'tags')
- fields = ('url', 'timestamp', 'title', 'tags', *readonly_fields)
+ fields = ('title', 'tags', *readonly_fields)
list_filter = ('added', 'updated', 'tags')
ordering = ['-added']
@@ -27,15 +39,16 @@ class SnapshotAdmin(admin.ModelAdmin):
canon = obj.as_link().canonical_outputs()
return format_html(
''
- '
'
- ' '
+ '
'
''
''
- '{}',
+ '{}'
+ '',
obj.archive_path,
obj.archive_path, canon['favicon_path'],
obj.archive_path, canon['wget_path'] or '',
- urldecode(htmldecode(obj.latest_title or obj.title or '-'))[:128],
+ 'fetched' if obj.latest_title or obj.title else 'pending',
+ urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...',
)
def files(self, obj):
@@ -68,17 +81,68 @@ class SnapshotAdmin(admin.ModelAdmin):
def url_str(self, obj):
return format_html(
- '{}
',
+ '{}',
obj.url,
obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
)
id_str.short_description = 'ID'
title_str.short_description = 'Title'
- url_str.short_description = 'URL'
+ url_str.short_description = 'Original URL'
id_str.admin_order_field = 'id'
title_str.admin_order_field = 'title'
url_str.admin_order_field = 'url'
+
+
+class ArchiveBoxAdmin(admin.AdminSite):
+ site_header = 'ArchiveBox'
+ index_title = 'Links'
+ site_title = 'Index'
+
+ def get_urls(self):
+ return [
+ path('core/snapshot/add/', self.add_view, name='add'),
+ ] + super().get_urls()
+
+ def add_view(self, request):
+ request.current_app = self.name
+ context = {
+ **self.each_context(request),
+ 'title': 'Add URLs',
+ }
+
+ if request.method == 'GET':
+ context['form'] = AddLinkForm()
+
+ elif request.method == 'POST':
+ form = AddLinkForm(request.POST)
+ if form.is_valid():
+ url = form.cleaned_data["url"]
+ print(f'[+] Adding URL: {url}')
+ depth = 0 if form.cleaned_data["depth"] == "0" else 1
+ input_kwargs = {
+ "urls": url,
+ "depth": depth,
+ "update_all": False,
+ "out_dir": OUTPUT_DIR,
+ }
+ add_stdout = StringIO()
+ with redirect_stdout(add_stdout):
+ add(**input_kwargs)
+ print(add_stdout.getvalue())
+
+ context.update({
+ "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+ "form": AddLinkForm()
+ })
+ else:
+ context["form"] = form
+
+ return render(template_name='add_links.html', request=request, context=context)
+
+
+admin.site = ArchiveBoxAdmin()
+admin.site.register(get_user_model())
admin.site.register(Snapshot, SnapshotAdmin)
diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py
index 8bf0cbd0..f641298a 100644
--- a/archivebox/core/forms.py
+++ b/archivebox/core/forms.py
@@ -1,10 +1,14 @@
+__package__ = 'archivebox.core'
+
from django import forms
+from ..util import URL_REGEX
+
CHOICES = (
- ('0', 'depth=0 (archive just this url)'),
- ('1', 'depth=1 (archive this url and all sites one link away)'),
+ ('0', 'depth = 0 (archive just these URLs)'),
+ ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
)
class AddLinkForm(forms.Form):
- url = forms.URLField()
- depth = forms.ChoiceField(choices=CHOICES, widget=forms.RadioSelect, initial='0')
+ url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
+ depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
diff --git a/archivebox/core/migrations/0005_auto_20200728_0326.py b/archivebox/core/migrations/0005_auto_20200728_0326.py
new file mode 100644
index 00000000..f367aeb1
--- /dev/null
+++ b/archivebox/core/migrations/0005_auto_20200728_0326.py
@@ -0,0 +1,28 @@
+# Generated by Django 3.0.7 on 2020-07-28 03:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0004_auto_20200713_1552'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='snapshot',
+ name='tags',
+ field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
+ ),
+ migrations.AlterField(
+ model_name='snapshot',
+ name='title',
+ field=models.CharField(blank=True, db_index=True, max_length=128, null=True),
+ ),
+ migrations.AlterField(
+ model_name='snapshot',
+ name='updated',
+ field=models.DateTimeField(blank=True, db_index=True, null=True),
+ ),
+ ]
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index 7ac9427b..95638bc1 100644
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -15,11 +15,11 @@ class Snapshot(models.Model):
url = models.URLField(unique=True)
timestamp = models.CharField(max_length=32, unique=True, db_index=True)
- title = models.CharField(max_length=128, null=True, default=None, db_index=True)
- tags = models.CharField(max_length=256, null=True, default=None, db_index=True)
+ title = models.CharField(max_length=128, null=True, blank=True, db_index=True)
+ tags = models.CharField(max_length=256, null=True, blank=True, db_index=True)
added = models.DateTimeField(auto_now_add=True, db_index=True)
- updated = models.DateTimeField(null=True, default=None, db_index=True)
+ updated = models.DateTimeField(null=True, blank=True, db_index=True)
# bookmarked = models.DateTimeField()
keys = ('url', 'timestamp', 'title', 'tags', 'updated')
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index e9cc0dc8..a0da8b92 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -5,16 +5,16 @@ import sys
from django.utils.crypto import get_random_string
-from ..config import (
- OUTPUT_DIR,
+from ..config import ( # noqa: F401
+ DEBUG,
SECRET_KEY,
ALLOWED_HOSTS,
PYTHON_DIR,
ACTIVE_THEME,
SQL_INDEX_FILENAME,
+ OUTPUT_DIR,
)
-
ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -25,8 +25,8 @@ INSTALLED_APPS = [
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
- 'django.contrib.admin',
'django.contrib.staticfiles',
+ 'django.contrib.admin',
'core',
@@ -121,5 +121,4 @@ STATIC_URL = '/static/'
STATICFILES_DIRS = [
os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME, 'static'),
os.path.join(PYTHON_DIR, 'themes', 'default', 'static'),
- os.path.join(PYTHON_DIR, 'themes', 'static'),
]
diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py
index 70ebaf63..0c1f8131 100644
--- a/archivebox/core/urls.py
+++ b/archivebox/core/urls.py
@@ -3,15 +3,12 @@ from django.contrib import admin
from django.urls import path, include
from django.views import static
from django.conf import settings
-from django.contrib.staticfiles.views import serve as serve_static
from django.views.generic.base import RedirectView
-from core.views import MainIndex, AddLinks, LinkDetails
+from core.views import MainIndex, LinkDetails
-admin.site.site_header = 'ArchiveBox'
-admin.site.index_title = 'Links'
-admin.site.site_title = 'Index'
+# print('DEBUG', settings.DEBUG)
urlpatterns = [
path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
@@ -19,14 +16,11 @@ urlpatterns = [
path('archive/', RedirectView.as_view(url='/')),
path('archive/', LinkDetails.as_view(), name='LinkAssets'),
- path('add/', AddLinks.as_view(), name='AddLinks'),
-
- path('static/', serve_static),
+ path('add/', RedirectView.as_view(url='/admin/core/snapshot/add/')),
path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
- path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
path('accounts/', include('django.contrib.auth.urls')),
path('admin/', admin.site.urls),
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index c411e98b..1eb8fc20 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -7,9 +7,6 @@ from django.views import View, static
from core.models import Snapshot
-from contextlib import redirect_stdout
-from io import StringIO
-
from ..index import load_main_index, load_main_index_meta
from ..config import (
OUTPUT_DIR,
@@ -18,10 +15,7 @@ from ..config import (
PUBLIC_INDEX,
PUBLIC_SNAPSHOTS,
)
-from ..util import base_url, ansi_to_html
-from .. main import add
-
-from .forms import AddLinkForm
+from ..util import base_url
class MainIndex(View):
@@ -45,48 +39,6 @@ class MainIndex(View):
return render(template_name=self.template, request=request, context=context)
-class AddLinks(View):
- template = 'add_links.html'
-
- def get(self, request):
- if not request.user.is_authenticated and not PUBLIC_INDEX:
- return redirect(f'/admin/login/?next={request.path}')
-
- context = {
- "form": AddLinkForm()
- }
-
- return render(template_name=self.template, request=request, context=context)
-
- def post(self, request):
- if not request.user.is_authenticated and not PUBLIC_INDEX:
- return redirect(f'/admin/login/?next={request.path}')
- form = AddLinkForm(request.POST)
- if form.is_valid():
- url = form.cleaned_data["url"]
- print(f'[+] Adding URL: {url}')
- depth = 0 if form.cleaned_data["depth"] == "0" else 0
- input_kwargs = {
- "urls": url,
- "depth": depth,
- "update_all": False,
- "out_dir": OUTPUT_DIR,
- }
- add_stdout = StringIO()
- with redirect_stdout(add_stdout):
- add(**input_kwargs)
- print(add_stdout.getvalue())
-
- context = {
- "stdout": ansi_to_html(add_stdout.getvalue()),
- "form": AddLinkForm()
- }
- else:
- context = {"form": form}
-
- return render(template_name=self.template, request=request, context=context)
-
-
class LinkDetails(View):
def get(self, request, path):
# missing trailing slash -> redirect to index
diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index e21ae576..4c6ae8bb 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -136,8 +136,8 @@ def link_details_template(link: Link) -> str:
'url_str': htmlencode(urldecode(link.base_url)),
'archive_url': urlencode(
wget_output_path(link)
- or (link.domain if link.is_archived else 'about:blank')
- ),
+ or (link.domain if link.is_archived else '')
+ ) or 'about:blank',
'extension': link.extension or 'html',
'tags': link.tags or 'untagged',
'status': 'archived' if link.is_archived else 'not yet archived',
diff --git a/archivebox/main.py b/archivebox/main.py
index 1cb34b30..141fe34c 100644
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -83,6 +83,7 @@ from .config import (
EXTERNAL_LOCATIONS,
DATA_LOCATIONS,
DEPENDENCIES,
+ DEBUG,
load_all_config,
CONFIG,
USER_CONFIG,
@@ -987,13 +988,19 @@ def server(runserver_args: Optional[List[str]]=None,
"""Run the ArchiveBox HTTP server"""
runserver_args = runserver_args or []
- check_data_folder(out_dir=out_dir)
+
+ from . import config
+ config.SHOW_PROGRESS = False
if debug:
- os.environ['DEBUG'] = 'True'
+ # if --debug is passed, patch config.DEBUG to be True for this run
+ config.DEBUG = True
else:
+ # force staticfiles to be served when DEBUG=False
+ # TODO: do this using nginx or another server instead of django?
runserver_args.append('--insecure')
+ check_data_folder(out_dir=out_dir)
setup_django(out_dir)
from django.core.management import call_command
from django.contrib.auth.models import User
diff --git a/archivebox/themes/admin/base.html b/archivebox/themes/admin/base.html
index 2160dfeb..2a67873e 100644
--- a/archivebox/themes/admin/base.html
+++ b/archivebox/themes/admin/base.html
@@ -2,7 +2,7 @@
{% get_current_language as LANGUAGE_CODE %}{% get_current_language_bidi as LANGUAGE_BIDI %}
-{% block title %}{% endblock %}
+{% block title %}{% endblock %} | ArchiveBox
{% block extrastyle %}{% endblock %}
{% if LANGUAGE_BIDI %}{% endif %}
@@ -13,6 +13,7 @@
{% if LANGUAGE_BIDI %}{% endif %}
{% endblock %}
{% block blockbots %}{% endblock %}
+
{% load i18n %}
@@ -26,13 +27,14 @@
-
+