fix pending titles and favicons, improve add page, custom admin

archivebox/core/admin.py

@@ -1,19 +1,31 @@
-from django.contrib import admin
-from django.utils.html import format_html
+__package__ = 'archivebox.core'
+
+from io import StringIO
+from contextlib import redirect_stdout
+
+from django.contrib import admin
+from django.urls import path
+from django.utils.html import format_html
+from django.shortcuts import render
+from django.contrib.auth import get_user_model
 
-from util import htmldecode, urldecode
 from core.models import Snapshot
-from archivebox.logging_util import printable_filesize
+from core.forms import AddLinkForm
+
+from ..util import htmldecode, urldecode, ansi_to_html
+from ..logging_util import printable_filesize
+from ..main import add
+from ..config import OUTPUT_DIR
 
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
 
 class SnapshotAdmin(admin.ModelAdmin):
-    list_display = ('title_str', 'url_str', 'tags', 'files', 'size', 'added', 'updated')
+    list_display = ('added', 'title_str', 'url_str', 'tags', 'files', 'size', 'updated')
     sort_fields = ('title_str', 'url_str', 'tags', 'added', 'updated')
-    readonly_fields = ('id', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
+    readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
     search_fields = ('url', 'timestamp', 'title', 'tags')
-    fields = ('url', 'timestamp', 'title', 'tags', *readonly_fields)
+    fields = ('title', 'tags', *readonly_fields)
     list_filter = ('added', 'updated', 'tags')
     ordering = ['-added']
 
@@ -27,15 +39,16 @@ class SnapshotAdmin(admin.ModelAdmin):
         canon = obj.as_link().canonical_outputs()
         return format_html(
             '<a href="/{}">'
-            '<img src="/{}/{}" style="height: 20px; width: 20px;" onerror="this.remove()">'
-            ' &nbsp; &nbsp; '
+                '<img src="/{}/{}" class="favicon" onerror="this.remove()">'
             '</a>'
             '<a href="/{}/{}">'
-            '<b>{}</b></a>',
+                '<b class="status-{}">{}</b>'
+            '</a>',
             obj.archive_path,
             obj.archive_path, canon['favicon_path'],
             obj.archive_path, canon['wget_path'] or '',
-            urldecode(htmldecode(obj.latest_title or obj.title or '-'))[:128],
+            'fetched' if obj.latest_title or obj.title else 'pending',
+            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...',
         )
 
     def files(self, obj):
@@ -68,17 +81,68 @@ class SnapshotAdmin(admin.ModelAdmin):
 
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code>{}</code></a>',
+            '<a href="{}">{}</a>',
             obj.url,
             obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
         )
 
     id_str.short_description = 'ID'
     title_str.short_description = 'Title'
-    url_str.short_description = 'URL'
+    url_str.short_description = 'Original URL'
 
     id_str.admin_order_field = 'id'
     title_str.admin_order_field = 'title'
     url_str.admin_order_field = 'url'
 
+
+
+class ArchiveBoxAdmin(admin.AdminSite):
+    site_header = 'ArchiveBox'
+    index_title = 'Links'
+    site_title = 'Index'
+
+    def get_urls(self):
+        return [
+            path('core/snapshot/add/', self.add_view, name='add'),
+        ] + super().get_urls()
+
+    def add_view(self, request):
+        request.current_app = self.name
+        context = {
+            **self.each_context(request),
+            'title': 'Add URLs',
+        }
+
+        if request.method == 'GET':
+            context['form'] = AddLinkForm()
+
+        elif request.method == 'POST':
+            form = AddLinkForm(request.POST)
+            if form.is_valid():
+                url = form.cleaned_data["url"]
+                print(f'[+] Adding URL: {url}')
+                depth = 0 if form.cleaned_data["depth"] == "0" else 1
+                input_kwargs = {
+                    "urls": url,
+                    "depth": depth,
+                    "update_all": False,
+                    "out_dir": OUTPUT_DIR,
+                }
+                add_stdout = StringIO()
+                with redirect_stdout(add_stdout):
+                   add(**input_kwargs)
+                print(add_stdout.getvalue())
+
+                context.update({
+                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+                    "form": AddLinkForm()
+                })
+            else:
+                context["form"] = form
+
+        return render(template_name='add_links.html', request=request, context=context)
+
+
+admin.site = ArchiveBoxAdmin()
+admin.site.register(get_user_model())
 admin.site.register(Snapshot, SnapshotAdmin)

archivebox/core/forms.py

@@ -1,10 +1,14 @@
+__package__ = 'archivebox.core'
+
 from django import forms
 
+from ..util import URL_REGEX
+
 CHOICES = (
-    ('0', 'depth=0 (archive just this url)'),
-    ('1', 'depth=1 (archive this url and all sites one link away)'),
+    ('0', 'depth = 0 (archive just these URLs)'),
+    ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
 )
 
 class AddLinkForm(forms.Form):
-    url = forms.URLField()
-    depth = forms.ChoiceField(choices=CHOICES, widget=forms.RadioSelect, initial='0')
+    url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
+    depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')

archivebox/core/migrations/0005_auto_20200728_0326.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.7 on 2020-07-28 03:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_auto_20200713_1552'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='tags',
+            field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='updated',
+            field=models.DateTimeField(blank=True, db_index=True, null=True),
+        ),
+    ]

archivebox/core/models.py

@@ -15,11 +15,11 @@ class Snapshot(models.Model):
     url = models.URLField(unique=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True)
 
-    title = models.CharField(max_length=128, null=True, default=None, db_index=True)
-    tags = models.CharField(max_length=256, null=True, default=None, db_index=True)
+    title = models.CharField(max_length=128, null=True, blank=True, db_index=True)
+    tags = models.CharField(max_length=256, null=True, blank=True, db_index=True)
 
     added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(null=True, default=None, db_index=True)
+    updated = models.DateTimeField(null=True, blank=True, db_index=True)
     # bookmarked = models.DateTimeField()
 
     keys = ('url', 'timestamp', 'title', 'tags', 'updated')

archivebox/core/settings.py

@@ -5,16 +5,16 @@ import sys
 from django.utils.crypto import get_random_string
 
 
-from ..config import (
-    OUTPUT_DIR,
+from ..config import (                                                          # noqa: F401
+    DEBUG,
     SECRET_KEY,
     ALLOWED_HOSTS,
     PYTHON_DIR,
     ACTIVE_THEME,
     SQL_INDEX_FILENAME,
+    OUTPUT_DIR,
 )
 
-
 ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
 
@@ -25,8 +25,8 @@ INSTALLED_APPS = [
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
-    'django.contrib.admin',
     'django.contrib.staticfiles',
+    'django.contrib.admin',
 
     'core',
 
@@ -121,5 +121,4 @@ STATIC_URL = '/static/'
 STATICFILES_DIRS = [
     os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME, 'static'),
     os.path.join(PYTHON_DIR, 'themes', 'default', 'static'),
-    os.path.join(PYTHON_DIR, 'themes', 'static'),
 ]

archivebox/core/urls.py

@@ -3,15 +3,12 @@ from django.contrib import admin
 from django.urls import path, include
 from django.views import static
 from django.conf import settings
-from django.contrib.staticfiles.views import serve as serve_static
 from django.views.generic.base import RedirectView
 
-from core.views import MainIndex, AddLinks, LinkDetails
+from core.views import MainIndex, LinkDetails
 
-admin.site.site_header = 'ArchiveBox'
-admin.site.index_title = 'Links' 
-admin.site.site_title = 'Index'
 
+# print('DEBUG', settings.DEBUG)
 
 urlpatterns = [
     path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
@@ -19,14 +16,11 @@ urlpatterns = [
 
     path('archive/', RedirectView.as_view(url='/')),
     path('archive/<path:path>', LinkDetails.as_view(), name='LinkAssets'),
-    path('add/', AddLinks.as_view(), name='AddLinks'),
-    
-    path('static/<path>', serve_static),
+    path('add/', RedirectView.as_view(url='/admin/core/snapshot/add/')),
     
     path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
     path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
 
-    path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
 
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', admin.site.urls),

archivebox/core/views.py

@@ -7,9 +7,6 @@ from django.views import View, static
 
 from core.models import Snapshot
 
-from contextlib import redirect_stdout
-from io import StringIO
-
 from ..index import load_main_index, load_main_index_meta
 from ..config import (
     OUTPUT_DIR,
@@ -18,10 +15,7 @@ from ..config import (
     PUBLIC_INDEX,
     PUBLIC_SNAPSHOTS,
 )
-from ..util import base_url, ansi_to_html
-from .. main import add
-
-from .forms import AddLinkForm
+from ..util import base_url
 
 
 class MainIndex(View):
@@ -45,48 +39,6 @@ class MainIndex(View):
         return render(template_name=self.template, request=request, context=context)
 
 
-class AddLinks(View):
-    template = 'add_links.html'
-
-    def get(self, request):
-        if not request.user.is_authenticated and not PUBLIC_INDEX:
-            return redirect(f'/admin/login/?next={request.path}')
-
-        context = {
-            "form": AddLinkForm()
-        }
-
-        return render(template_name=self.template, request=request, context=context)
-
-    def post(self, request):
-        if not request.user.is_authenticated and not PUBLIC_INDEX:
-            return redirect(f'/admin/login/?next={request.path}')
-        form = AddLinkForm(request.POST)
-        if form.is_valid():
-            url = form.cleaned_data["url"]
-            print(f'[+] Adding URL: {url}')
-            depth = 0 if form.cleaned_data["depth"] == "0" else 0
-            input_kwargs = {
-                "urls": url,
-                "depth": depth,
-                "update_all": False,
-                "out_dir": OUTPUT_DIR,
-            }
-            add_stdout = StringIO()
-            with redirect_stdout(add_stdout):
-               add(**input_kwargs)
-            print(add_stdout.getvalue())
-
-            context = {
-                "stdout": ansi_to_html(add_stdout.getvalue()),
-                "form": AddLinkForm()
-            }
-        else:
-            context = {"form": form}
-
-        return render(template_name=self.template, request=request, context=context)
-
-
 class LinkDetails(View):
     def get(self, request, path):
         # missing trailing slash -> redirect to index