API improvements

archivebox/api/v1_api.py

@@ -37,9 +37,9 @@ html_description=f'''
 
 
 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
+    # api.add_router('/auth/',     'api.v1_auth.router')
     api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/crawls/',   'api.v1_core.router')
+    api.add_router('/crawls/',   'api.v1_crawls.router')
     api.add_router('/cli/',      'api.v1_cli.router')
     api.add_router('/jobs/',     'api.v1_actors.router')
     return api
@@ -83,7 +83,7 @@ class NinjaAPIWithIOCapture(NinjaAPI):
 api = NinjaAPIWithIOCapture(
     title='ArchiveBox API',
     description=html_description,
-    version='1.0.0',
+    version=VERSION,
     csrf=False,
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",

archivebox/api/v1_cli.py

@@ -17,10 +17,10 @@ from archivebox.misc.util import ansi_to_html
 from archivebox.config.common import ARCHIVING_CONFIG
 
 
-from .auth import API_AUTH_METHODS
+# from .auth import API_AUTH_METHODS
 
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])
 
 
 # Schemas

archivebox/api/v1_core.py

@@ -16,12 +16,13 @@ from ninja.errors import HttpError
 
 from core.models import Snapshot, ArchiveResult, Tag
 from api.models import APIToken, OutboundWebhook
-from crawls.models import Crawl
-from seeds.models import Seed
+from api.v1_crawls import CrawlSchema, SeedSchema
 
-from .auth import API_AUTH_METHODS
+# from .auth import API_AUTH_METHODS
 
-router = Router(tags=['Core Models'], auth=API_AUTH_METHODS)
+
+
+router = Router(tags=['Core Models'])
 
 
 
@@ -397,108 +398,6 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):
         pass
     return tag
 
-
-
-class SeedSchema(Schema):
-    TYPE: str = 'seeds.models.Seed'
-
-    id: UUID
-    abid: str
-    
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    
-    uri: str
-    tags_str: str
-    config: dict
-    
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-    
-@router.get("/seeds", response=List[SeedSchema], url_name="get_seeds")
-def get_seeds(request):
-    return Seed.objects.all().distinct()
-
-@router.get("/seed/{seed_id}", response=SeedSchema, url_name="get_seed")
-def get_seed(request, seed_id: str):
-    seed = None
-    request.with_snapshots = False
-    request.with_archiveresults = False
-    
-    try:
-        seed = Seed.objects.get(Q(abid__icontains=seed_id) | Q(id__icontains=seed_id))
-    except Exception:
-        pass
-    return seed
-
-
-class CrawlSchema(Schema):
-    TYPE: str = 'core.models.Crawl'
-
-    id: UUID
-    abid: str
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    
-    status: str
-    retry_at: datetime | None
-
-    seed: SeedSchema
-    max_depth: int
-    
-    # snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-    
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if context['request'].with_snapshots:
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-@router.get("/crawls", response=List[CrawlSchema], url_name="get_crawls")
-def get_crawls(request):
-    return Crawl.objects.all().distinct()
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request, crawl_id: str, with_snapshots: bool=False, with_archiveresults: bool=False):
-    """Get a specific Crawl by id or abid."""
-    
-    crawl = None
-    request.with_snapshots = with_snapshots
-    request.with_archiveresults = with_archiveresults
-    
-    try:
-        crawl = Crawl.objects.get(abid__icontains=crawl_id)
-    except Exception:
-        pass
-
-    try:
-        crawl = crawl or Crawl.objects.get(id__icontains=crawl_id)
-    except Exception:
-        pass
-    return crawl
-
-
 @router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, SeedSchema, CrawlSchema], url_name="get_any", summary="Get any object by its ABID or ID (e.g. snapshot, archiveresult, tag, seed, crawl, etc.)")
 def get_any(request, abid: str):
     """Get any object by its ABID or ID (e.g. snapshot, archiveresult, tag, seed, crawl, etc.)."""
@@ -529,11 +428,13 @@ def get_any(request, abid: str):
         pass
     
     try:
+        from api.v1_crawls import get_seed
         response = response or get_seed(request, abid)
     except Exception:
         pass
     
     try:
+        from api.v1_crawls import get_crawl
         response = response or get_crawl(request, abid)
     except Exception:
         pass

archivebox/api/v1_crawls.py

@@ -0,0 +1,119 @@
+__package__ = 'archivebox.api'
+
+from uuid import UUID
+from typing import List
+from datetime import datetime
+
+from django.db.models import Q
+from django.contrib.auth import get_user_model
+
+from ninja import Router, Schema
+
+from core.models import Snapshot
+from crawls.models import Crawl
+from seeds.models import Seed
+
+from .auth import API_AUTH_METHODS
+
+router = Router(tags=['Crawl Models'], auth=API_AUTH_METHODS)
+
+
+class SeedSchema(Schema):
+    TYPE: str = 'seeds.models.Seed'
+
+    id: UUID
+    abid: str
+    
+    modified_at: datetime
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+    
+    uri: str
+    tags_str: str
+    config: dict
+    
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+    
+@router.get("/seeds", response=List[SeedSchema], url_name="get_seeds")
+def get_seeds(request):
+    return Seed.objects.all().distinct()
+
+@router.get("/seed/{seed_id}", response=SeedSchema, url_name="get_seed")
+def get_seed(request, seed_id: str):
+    seed = None
+    request.with_snapshots = False
+    request.with_archiveresults = False
+    
+    try:
+        seed = Seed.objects.get(Q(abid__icontains=seed_id) | Q(id__icontains=seed_id))
+    except Exception:
+        pass
+    return seed
+
+
+class CrawlSchema(Schema):
+    TYPE: str = 'core.models.Crawl'
+
+    id: UUID
+    abid: str
+
+    modified_at: datetime
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+    
+    status: str
+    retry_at: datetime | None
+
+    seed: SeedSchema
+    max_depth: int
+    
+    # snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+    
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if context['request'].with_snapshots:
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+
+@router.get("/crawls", response=List[CrawlSchema], url_name="get_crawls")
+def get_crawls(request):
+    return Crawl.objects.all().distinct()
+
+@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
+def get_crawl(request, crawl_id: str, with_snapshots: bool=False, with_archiveresults: bool=False):
+    """Get a specific Crawl by id or abid."""
+    
+    crawl = None
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = with_archiveresults
+    
+    try:
+        crawl = Crawl.objects.get(abid__icontains=crawl_id)
+    except Exception:
+        pass
+
+    try:
+        crawl = crawl or Crawl.objects.get(id__icontains=crawl_id)
+    except Exception:
+        pass
+    return crawl
+