move everything out of legacy folder

archivebox/index/__init__.py

@@ -0,0 +1,592 @@
+__package__ = 'archivebox.index'
+
+import re
+import os
+import shutil
+import json as pyjson
+
+from itertools import chain
+from typing import List, Tuple, Dict, Optional, Iterable
+from collections import OrderedDict
+from contextlib import contextmanager
+
+from ..parsers import parse_links
+from ..util import (
+    scheme,
+    enforce_types,
+    TimedProgress,
+    atomic_write,
+    ExtendedEncoder,
+)
+from ..config import (
+    ARCHIVE_DIR_NAME,
+    SQL_INDEX_FILENAME,
+    JSON_INDEX_FILENAME,
+    HTML_INDEX_FILENAME,
+    OUTPUT_DIR,
+    TIMEOUT,
+    URL_BLACKLIST_PTN,
+    ANSI,
+    stderr,
+)
+from ..cli.logging import (
+    log_indexing_process_started,
+    log_indexing_process_finished,
+    log_indexing_started,
+    log_indexing_finished,
+    log_parsing_started,
+    log_parsing_finished,
+)
+
+from .schema import Link, ArchiveResult
+from .html import (
+    write_html_main_index,
+    write_html_link_details,
+)
+from .json import (
+    parse_json_main_index,
+    write_json_main_index,
+    parse_json_link_details, 
+    write_json_link_details,
+)
+from .sql import (
+    write_sql_main_index,
+    parse_sql_main_index,
+)
+
+### Link filtering and checking
+
+@enforce_types
+def merge_links(a: Link, b: Link) -> Link:
+    """deterministially merge two links, favoring longer field values over shorter,
+    and "cleaner" values over worse ones.
+    """
+    assert a.base_url == b.base_url, 'Cannot merge two links with different URLs'
+
+    # longest url wins (because a fuzzy url will always be shorter)
+    url = a.url if len(a.url) > len(b.url) else b.url
+
+    # best title based on length and quality
+    possible_titles = [
+        title
+        for title in (a.title, b.title)
+        if title and title.strip() and '://' not in title
+    ]
+    title = None
+    if len(possible_titles) == 2:
+        title = max(possible_titles, key=lambda t: len(t))
+    elif len(possible_titles) == 1:
+        title = possible_titles[0]
+
+    # earliest valid timestamp
+    timestamp = (
+        a.timestamp
+        if float(a.timestamp or 0) < float(b.timestamp or 0) else
+        b.timestamp
+    )
+
+    # all unique, truthy tags
+    tags_set = (
+        set(tag.strip() for tag in (a.tags or '').split(','))
+        | set(tag.strip() for tag in (b.tags or '').split(','))
+    )
+    tags = ','.join(tags_set) or None
+
+    # all unique source entries
+    sources = list(set(a.sources + b.sources))
+
+    # all unique history entries for the combined archive methods
+    all_methods = set(list(a.history.keys()) + list(a.history.keys()))
+    history = {
+        method: (a.history.get(method) or []) + (b.history.get(method) or [])
+        for method in all_methods
+    }
+    for method in all_methods:
+        deduped_jsons = {
+            pyjson.dumps(result, sort_keys=True, cls=ExtendedEncoder)
+            for result in history[method]
+        }
+        history[method] = list(reversed(sorted(
+            (ArchiveResult.from_json(pyjson.loads(result)) for result in deduped_jsons),
+            key=lambda result: result.start_ts,
+        )))
+
+    return Link(
+        url=url,
+        timestamp=timestamp,
+        title=title,
+        tags=tags,
+        sources=sources,
+        history=history,
+    )
+
+
+@enforce_types
+def validate_links(links: Iterable[Link]) -> List[Link]:
+    links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
+    links = sorted_links(links)      # deterministically sort the links based on timstamp, url
+    links = uniquefied_links(links)  # merge/dedupe duplicate timestamps & urls
+
+    if not links:
+        stderr('{red}[X] No links found in index.{reset}'.format(**ANSI))
+        stderr('    To add a link to your archive, run:')
+        stderr("        archivebox add 'https://example.com'")
+        stderr()
+        stderr('    For more usage and examples, run:')
+        stderr('        archivebox help')
+        raise SystemExit(1)
+
+    return list(links)
+
+
+@enforce_types
+def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
+    """remove chrome://, about:// or other schemed links that cant be archived"""
+    for link in links:
+        scheme_is_valid = scheme(link.url) in ('http', 'https', 'ftp')
+        not_blacklisted = (not URL_BLACKLIST_PTN.match(link.url)) if URL_BLACKLIST_PTN else True
+        if scheme_is_valid and not_blacklisted:
+            yield link
+
+
+@enforce_types
+def uniquefied_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
+    """
+    ensures that all non-duplicate links have monotonically increasing timestamps
+    """
+
+    unique_urls: OrderedDict[str, Link] = OrderedDict()
+
+    for link in sorted_links:
+        if link.base_url in unique_urls:
+            # merge with any other links that share the same url
+            link = merge_links(unique_urls[link.base_url], link)
+        unique_urls[link.base_url] = link
+
+    unique_timestamps: OrderedDict[str, Link] = OrderedDict()
+    for link in unique_urls.values():
+        new_link = link.overwrite(
+            timestamp=lowest_uniq_timestamp(unique_timestamps, link.timestamp),
+        )
+        unique_timestamps[new_link.timestamp] = new_link
+
+    return unique_timestamps.values()
+
+
+@enforce_types
+def sorted_links(links: Iterable[Link]) -> Iterable[Link]:
+    sort_func = lambda link: (link.timestamp.split('.', 1)[0], link.url)
+    return sorted(links, key=sort_func, reverse=True)
+
+
+@enforce_types
+def links_after_timestamp(links: Iterable[Link], resume: Optional[float]=None) -> Iterable[Link]:
+    if not resume:
+        yield from links
+        return
+
+    for link in links:
+        try:
+            if float(link.timestamp) <= resume:
+                yield link
+        except (ValueError, TypeError):
+            print('Resume value and all timestamp values must be valid numbers.')
+
+
+@enforce_types
+def lowest_uniq_timestamp(used_timestamps: OrderedDict, timestamp: str) -> str:
+    """resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2"""
+
+    timestamp = timestamp.split('.')[0]
+    nonce = 0
+
+    # first try 152323423 before 152323423.0
+    if timestamp not in used_timestamps:
+        return timestamp
+
+    new_timestamp = '{}.{}'.format(timestamp, nonce)
+    while new_timestamp in used_timestamps:
+        nonce += 1
+        new_timestamp = '{}.{}'.format(timestamp, nonce)
+
+    return new_timestamp
+
+
+
+### Main Links Index
+
+@contextmanager
+@enforce_types
+def timed_index_update(out_path: str):
+    log_indexing_started(out_path)
+    timer = TimedProgress(TIMEOUT * 2, prefix='      ')
+    try:
+        yield
+    finally:
+        timer.end()
+
+    assert os.path.exists(out_path), f'Failed to write index file: {out_path}'
+    log_indexing_finished(out_path)
+
+
+@enforce_types
+def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
+    """create index.html file for a given list of links"""
+
+    log_indexing_process_started(len(links))
+
+    with timed_index_update(os.path.join(out_dir, SQL_INDEX_FILENAME)):
+        write_sql_main_index(links, out_dir=out_dir)
+
+    with timed_index_update(os.path.join(out_dir, JSON_INDEX_FILENAME)):
+        write_json_main_index(links, out_dir=out_dir)
+
+    with timed_index_update(os.path.join(out_dir, HTML_INDEX_FILENAME)):
+        write_html_main_index(links, out_dir=out_dir, finished=finished)
+
+    log_indexing_process_finished()
+
+
+@enforce_types
+def load_main_index(out_dir: str=OUTPUT_DIR, warn: bool=True) -> List[Link]:
+    """parse and load existing index with any new links from import_path merged in"""
+
+    all_links: List[Link] = []
+    all_links = list(parse_json_main_index(out_dir))
+    links_from_sql = list(parse_sql_main_index(out_dir))
+
+    if warn and not set(l.url for l in all_links) == set(l.url for l in links_from_sql):
+        stderr('{red}[!] Warning: SQL index does not match JSON index!{reset}'.format(**ANSI))
+        stderr('    To repair the index and re-import any orphaned links run:')
+        stderr('        archivebox init')
+
+    return all_links
+
+@enforce_types
+def load_main_index_meta(out_dir: str=OUTPUT_DIR) -> Optional[dict]:
+    index_path = os.path.join(out_dir, JSON_INDEX_FILENAME)
+    if os.path.exists(index_path):
+        with open(index_path, 'r', encoding='utf-8') as f:
+            meta_dict = pyjson.load(f)
+            meta_dict.pop('links')
+            return meta_dict
+
+    return None
+
+@enforce_types
+def import_new_links(existing_links: List[Link],
+                     import_path: str,
+                     out_dir: str=OUTPUT_DIR) -> Tuple[List[Link], List[Link]]:
+
+    new_links: List[Link] = []
+
+    # parse and validate the import file
+    log_parsing_started(import_path)
+    raw_links, parser_name = parse_links(import_path)
+    new_links = validate_links(raw_links)
+
+    # merge existing links in out_dir and new links
+    all_links = validate_links(existing_links + new_links)
+    all_link_urls = {link.url for link in existing_links}
+
+    new_links = [
+        link for link in new_links
+        if link.url not in all_link_urls
+    ]
+
+    if parser_name:
+        num_parsed = len(raw_links)
+        num_new_links = len(all_links) - len(existing_links)
+        log_parsing_finished(num_parsed, num_new_links, parser_name)
+
+    return all_links, new_links
+
+
+@enforce_types
+def patch_main_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
+    """hack to in-place update one row's info in the generated index files"""
+
+    # TODO: remove this ASAP, it's ugly, error-prone, and potentially dangerous
+
+    title = link.title or link.latest_outputs(status='succeeded')['title']
+    successful = link.num_outputs
+
+    # Patch JSON main index
+    json_file_links = parse_json_main_index(out_dir)
+    patched_links = []
+    for saved_link in json_file_links:
+        if saved_link.url == link.url:
+            patched_links.append(saved_link.overwrite(
+                title=title,
+                history=link.history,
+                updated=link.updated,
+            ))
+        else:
+            patched_links.append(saved_link)
+    
+    write_json_main_index(patched_links, out_dir=out_dir)
+
+    # Patch HTML main index
+    html_path = os.path.join(out_dir, 'index.html')
+    with open(html_path, 'r') as f:
+        html = f.read().split('\n')
+    for idx, line in enumerate(html):
+        if title and ('<span data-title-for="{}"'.format(link.url) in line):
+            html[idx] = '<span>{}</span>'.format(title)
+        elif successful and ('<span data-number-for="{}"'.format(link.url) in line):
+            html[idx] = '<span>{}</span>'.format(successful)
+            break
+
+    atomic_write('\n'.join(html), html_path)
+
+
+### Link Details Index
+
+@enforce_types
+def write_link_details(link: Link, out_dir: Optional[str]=None) -> None:
+    out_dir = out_dir or link.link_dir
+
+    write_json_link_details(link, out_dir=out_dir)
+    write_html_link_details(link, out_dir=out_dir)
+
+
+@enforce_types
+def load_link_details(link: Link, out_dir: Optional[str]=None) -> Link:
+    """check for an existing link archive in the given directory, 
+       and load+merge it into the given link dict
+    """
+    out_dir = out_dir or link.link_dir
+
+    existing_link = parse_json_link_details(out_dir)
+    if existing_link:
+        return merge_links(existing_link, link)
+
+    return link
+
+
+
+LINK_FILTERS = {
+    'exact': lambda link, pattern: (link.url == pattern) or (link.base_url == pattern),
+    'substring': lambda link, pattern: pattern in link.url,
+    'regex': lambda link, pattern: bool(re.match(pattern, link.url)),
+    'domain': lambda link, pattern: link.domain == pattern,
+}
+
+@enforce_types
+def link_matches_filter(link: Link, filter_patterns: List[str], filter_type: str='exact') -> bool:
+    for pattern in filter_patterns:
+        try:
+            if LINK_FILTERS[filter_type](link, pattern):
+                return True
+        except Exception:
+            stderr()
+            stderr(
+                f'[X] Got invalid pattern for --filter-type={filter_type}:',
+                color='red',
+            )
+            stderr(f'    {pattern}')
+            raise SystemExit(2)
+
+    return False
+
+
+def get_indexed_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """indexed links without checking archive status or data directory validity"""
+    return {
+        link.link_dir: link
+        for link in links
+    }
+
+def get_archived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """indexed links that are archived with a valid data directory"""
+    return {
+        link.link_dir: link
+        for link in filter(is_archived, links)
+    }
+
+def get_unarchived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """indexed links that are unarchived with no data directory or an empty data directory"""
+    return {
+        link.link_dir: link
+        for link in filter(is_unarchived, links)
+    }
+
+def get_present_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that are expected to exist based on the main index"""
+    all_folders = {}
+
+    for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
+        if entry.is_dir(follow_symlinks=True):
+            link = None
+            try:
+                link = parse_json_link_details(entry.path)
+            except Exception:
+                pass
+
+            all_folders[entry.path] = link
+
+    return all_folders
+
+def get_valid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs with a valid index matched to the main index and archived content"""
+    return {
+        link.link_dir: link
+        for link in filter(is_valid, links)
+    }
+
+def get_invalid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized"""
+    duplicate = get_duplicate_folders(links, out_dir=OUTPUT_DIR)
+    orphaned = get_orphaned_folders(links, out_dir=OUTPUT_DIR)
+    corrupted = get_corrupted_folders(links, out_dir=OUTPUT_DIR)
+    unrecognized = get_unrecognized_folders(links, out_dir=OUTPUT_DIR)
+    return {**duplicate, **orphaned, **corrupted, **unrecognized}
+
+
+def get_duplicate_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that conflict with other directories that have the same link URL or timestamp"""
+    links = list(links)
+    by_url = {link.url: 0 for link in links}
+    by_timestamp = {link.timestamp: 0 for link in links}
+
+    duplicate_folders = {}
+
+    indexed_folders = {link.link_dir for link in links}
+    data_folders = (
+        entry.path
+        for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME))
+        if entry.is_dir(follow_symlinks=True) and entry.path not in indexed_folders
+    )
+
+    for path in chain(sorted(indexed_folders), sorted(data_folders)):
+        link = None
+        try:
+            link = parse_json_link_details(path)
+        except Exception:
+            pass
+
+        if link:
+            # link folder has same timestamp as different link folder
+            by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1
+            if by_timestamp[link.timestamp] > 1:
+                duplicate_folders[path] = link
+
+            # link folder has same url as different link folder
+            by_url[link.url] = by_url.get(link.url, 0) + 1
+            if by_url[link.url] > 1:
+                duplicate_folders[path] = link
+
+    return duplicate_folders
+
+def get_orphaned_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that contain a valid index but aren't listed in the main index"""
+    links = list(links)
+    indexed_folders = {link.link_dir: link for link in links}
+    orphaned_folders = {}
+
+    for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
+        if entry.is_dir(follow_symlinks=True):
+            link = None
+            try:
+                link = parse_json_link_details(entry.path)
+            except Exception:
+                pass
+
+            if link and entry.path not in indexed_folders:
+                # folder is a valid link data dir with index details, but it's not in the main index
+                orphaned_folders[entry.path] = link
+
+    return orphaned_folders
+
+def get_corrupted_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that don't contain a valid index and aren't listed in the main index"""
+    return {
+        link.link_dir: link
+        for link in filter(is_corrupt, links)
+    }
+
+def get_unrecognized_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+    """dirs that don't contain recognizable archive data and aren't listed in the main index"""
+    by_timestamp = {link.timestamp: 0 for link in links}
+    unrecognized_folders: Dict[str, Optional[Link]] = {}
+
+    for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
+        if entry.is_dir(follow_symlinks=True):
+            index_exists = os.path.exists(os.path.join(entry.path, 'index.json'))
+            link = None
+            try:
+                link = parse_json_link_details(entry.path)
+            except Exception:
+                pass
+
+            if index_exists and link is None:
+                # index exists but it's corrupted or unparseable
+                unrecognized_folders[entry.path] = link
+            
+            elif not index_exists:
+                # link details index doesn't exist and the folder isn't in the main index
+                timestamp = entry.path.rsplit('/', 1)[-1]
+                if timestamp not in by_timestamp:
+                    unrecognized_folders[entry.path] = link
+
+    return unrecognized_folders
+
+
+def is_valid(link: Link) -> bool:
+    dir_exists = os.path.exists(link.link_dir)
+    index_exists = os.path.exists(os.path.join(link.link_dir, 'index.json'))
+    if not dir_exists:
+        # unarchived links are not included in the valid list
+        return False
+    if dir_exists and not index_exists:
+        return False
+    if dir_exists and index_exists:
+        try:
+            parsed_link = parse_json_link_details(link.link_dir)
+            return link.url == parsed_link.url
+        except Exception:
+            pass
+    return False
+
+def is_corrupt(link: Link) -> bool:
+    if not os.path.exists(link.link_dir):
+        # unarchived links are not considered corrupt
+        return False
+
+    if is_valid(link):
+        return False
+
+    return True
+
+def is_archived(link: Link) -> bool:
+    return is_valid(link) and link.is_archived
+    
+def is_unarchived(link: Link) -> bool:
+    if not os.path.exists(link.link_dir):
+        return True
+    return not link.is_archived
+
+
+def fix_invalid_folder_locations(out_dir: str=OUTPUT_DIR) -> Tuple[List[str], List[str]]:
+    fixed = []
+    cant_fix = []
+    for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
+        if entry.is_dir(follow_symlinks=True):
+            if os.path.exists(os.path.join(entry.path, 'index.json')):
+                link = parse_json_link_details(entry.path)
+                if not link:
+                    continue
+
+                if not entry.path.endswith(f'/{link.timestamp}'):
+                    dest = os.path.join(out_dir, ARCHIVE_DIR_NAME, link.timestamp)
+                    if os.path.exists(dest):
+                        cant_fix.append(entry.path)
+                    else:
+                        shutil.move(entry.path, dest)
+                        fixed.append(dest)
+
+                if link.link_dir != entry.path:
+                    link = link.overwrite(link_dir=entry.path)
+                    write_json_link_details(link, out_dir=entry.path)
+
+    return fixed, cant_fix

archivebox/index/html.py

@@ -0,0 +1,144 @@
+__package__ = 'archivebox.index'
+
+import os
+
+from datetime import datetime
+from typing import List, Optional, Iterator
+
+from .schema import Link
+from ..util import (
+    enforce_types,
+    ts_to_date,
+    urlencode,
+    htmlencode,
+    urldecode,
+    wget_output_path,
+    render_template,
+    atomic_write,
+    copy_and_overwrite,
+)
+from ..config import (
+    OUTPUT_DIR,
+    TEMPLATES_DIR,
+    VERSION,
+    GIT_SHA,
+    FOOTER_INFO,
+    ARCHIVE_DIR_NAME,
+    HTML_INDEX_FILENAME,
+    STATIC_DIR_NAME,
+    ROBOTS_TXT_FILENAME,
+    FAVICON_FILENAME,
+)
+
+join = lambda *paths: os.path.join(*paths)
+MAIN_INDEX_TEMPLATE = join(TEMPLATES_DIR, 'main_index.html')
+MAIN_INDEX_ROW_TEMPLATE = join(TEMPLATES_DIR, 'main_index_row.html')
+LINK_DETAILS_TEMPLATE = join(TEMPLATES_DIR, 'link_details.html')
+TITLE_LOADING_MSG = 'Not yet archived...'
+
+
+### Main Links Index
+
+@enforce_types
+def parse_html_main_index(out_dir: str=OUTPUT_DIR) -> Iterator[str]:
+    """parse an archive index html file and return the list of urls"""
+
+    index_path = os.path.join(out_dir, HTML_INDEX_FILENAME)
+    if os.path.exists(index_path):
+        with open(index_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                if 'class="link-url"' in line:
+                    yield line.split('"')[1]
+    return ()
+
+@enforce_types
+def write_html_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
+    """write the html link index to a given path"""
+
+    copy_and_overwrite(join(TEMPLATES_DIR, FAVICON_FILENAME), join(out_dir, FAVICON_FILENAME))
+    copy_and_overwrite(join(TEMPLATES_DIR, ROBOTS_TXT_FILENAME), join(out_dir, ROBOTS_TXT_FILENAME))
+    copy_and_overwrite(join(TEMPLATES_DIR, STATIC_DIR_NAME), join(out_dir, STATIC_DIR_NAME))
+    
+    rendered_html = main_index_template(links, finished=finished)
+    atomic_write(rendered_html, join(out_dir, HTML_INDEX_FILENAME))
+
+
+@enforce_types
+def main_index_template(links: List[Link], finished: bool=True) -> str:
+    """render the template for the entire main index"""
+
+    return render_template(MAIN_INDEX_TEMPLATE, {
+        'version': VERSION,
+        'git_sha': GIT_SHA,
+        'num_links': str(len(links)),
+        'status': 'finished' if finished else 'running',
+        'date_updated': datetime.now().strftime('%Y-%m-%d'),
+        'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
+        'rows': '\n'.join(
+            main_index_row_template(link)
+            for link in links
+        ),
+        'footer_info': FOOTER_INFO,
+    })
+
+
+@enforce_types
+def main_index_row_template(link: Link) -> str:
+    """render the template for an individual link row of the main index"""
+
+    return render_template(MAIN_INDEX_ROW_TEMPLATE, {
+        **link._asdict(extended=True),
+        
+        # before pages are finished archiving, show loading msg instead of title
+        'title': (
+            link.title
+            or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
+        ),
+
+        # before pages are finished archiving, show fallback loading favicon
+        'favicon_url': (
+            join(ARCHIVE_DIR_NAME, link.timestamp, 'favicon.ico')
+            # if link['is_archived'] else 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs='
+        ),
+
+        # before pages are finished archiving, show the details page instead
+        'wget_url': urlencode(wget_output_path(link) or 'index.html'),
+        
+        # replace commas in tags with spaces, or file extension if it's static
+        'tags': (link.tags or '') + (' {}'.format(link.extension) if link.is_static else ''),
+    })
+
+
+### Link Details Index
+
+@enforce_types
+def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
+    out_dir = out_dir or link.link_dir
+
+    rendered_html = link_details_template(link)
+    atomic_write(rendered_html, join(out_dir, HTML_INDEX_FILENAME))
+
+
+@enforce_types
+def link_details_template(link: Link) -> str:
+
+    link_info = link._asdict(extended=True)
+
+    return render_template(LINK_DETAILS_TEMPLATE, {
+        **link_info,
+        **link_info['canonical'],
+        'title': (
+            link.title
+            or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
+        ),
+        'url_str': htmlencode(urldecode(link.base_url)),
+        'archive_url': urlencode(
+            wget_output_path(link)
+            or (link.domain if link.is_archived else 'about:blank')
+        ),
+        'extension': link.extension or 'html',
+        'tags': link.tags or 'untagged',
+        'status': 'archived' if link.is_archived else 'not yet archived',
+        'status_color': 'success' if link.is_archived else 'danger',
+        'oldest_archive_date': ts_to_date(link.oldest_archive_date),
+    })

archivebox/index/json.py

@@ -0,0 +1,113 @@
+__package__ = 'archivebox.index'
+
+import os
+import sys
+import json
+
+from datetime import datetime
+from typing import List, Optional, Iterator
+
+from .schema import Link, ArchiveResult
+from ..util import enforce_types, atomic_write
+from ..config import (
+    VERSION,
+    OUTPUT_DIR,
+    FOOTER_INFO,
+    GIT_SHA,
+    DEPENDENCIES,
+    JSON_INDEX_FILENAME,
+    ARCHIVE_DIR_NAME,
+)
+
+
+MAIN_INDEX_HEADER = {
+    'info': 'This is an index of site data archived by ArchiveBox: The self-hosted web archive.',
+    'schema': 'archivebox.index.json',
+    'copyright_info': FOOTER_INFO,
+    'meta': {
+        'project': 'ArchiveBox',
+        'version': VERSION,
+        'git_sha': GIT_SHA,
+        'website': 'https://ArchiveBox.io',
+        'docs': 'https://github.com/pirate/ArchiveBox/wiki',
+        'source': 'https://github.com/pirate/ArchiveBox',
+        'issues': 'https://github.com/pirate/ArchiveBox/issues',
+        'dependencies': DEPENDENCIES,
+    },
+}
+
+
+### Main Links Index
+
+@enforce_types
+def parse_json_main_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]:
+    """parse an archive index json file and return the list of links"""
+
+    index_path = os.path.join(out_dir, JSON_INDEX_FILENAME)
+    if os.path.exists(index_path):
+        with open(index_path, 'r', encoding='utf-8') as f:
+            links = json.load(f)['links']
+            for link_json in links:
+                yield Link.from_json(link_json)
+
+    return ()
+
+@enforce_types
+def write_json_main_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None:
+    """write the json link index to a given path"""
+
+    assert isinstance(links, List), 'Links must be a list, not a generator.'
+    assert not links or isinstance(links[0].history, dict)
+    assert not links or isinstance(links[0].sources, list)
+
+    if links and links[0].history.get('title'):
+        assert isinstance(links[0].history['title'][0], ArchiveResult)
+
+    if links and links[0].sources:
+        assert isinstance(links[0].sources[0], str)
+
+    main_index_json = {
+        **MAIN_INDEX_HEADER,
+        'num_links': len(links),
+        'updated': datetime.now(),
+        'last_run_cmd': sys.argv,
+        'links': links,
+    }
+    atomic_write(main_index_json, os.path.join(out_dir, JSON_INDEX_FILENAME))
+
+
+### Link Details Index
+
+@enforce_types
+def write_json_link_details(link: Link, out_dir: Optional[str]=None) -> None:
+    """write a json file with some info about the link"""
+    
+    out_dir = out_dir or link.link_dir
+    path = os.path.join(out_dir, JSON_INDEX_FILENAME)
+
+    atomic_write(link._asdict(extended=True), path)
+
+
+@enforce_types
+def parse_json_link_details(out_dir: str) -> Optional[Link]:
+    """load the json link index from a given directory"""
+    existing_index = os.path.join(out_dir, JSON_INDEX_FILENAME)
+    if os.path.exists(existing_index):
+        with open(existing_index, 'r', encoding='utf-8') as f:
+            try:
+                link_json = json.load(f)
+                return Link.from_json(link_json)
+            except json.JSONDecodeError:
+                pass
+    return None
+
+@enforce_types
+def parse_json_links_details(out_dir: str) -> Iterator[Link]:
+    """read through all the archive data folders and return the parsed links"""
+
+    for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
+        if entry.is_dir(follow_symlinks=True):
+            if os.path.exists(os.path.join(entry.path, 'index.json')):
+                link = parse_json_link_details(entry.path)
+                if link:
+                    yield link

archivebox/index/schema.py

@@ -0,0 +1,384 @@
+__package__ = 'archivebox.index'
+
+import os
+
+from datetime import datetime
+
+from typing import List, Dict, Any, Optional, Union
+
+from dataclasses import dataclass, asdict, field, fields
+
+
+class ArchiveError(Exception):
+    def __init__(self, message, hints=None):
+        super().__init__(message)
+        self.hints = hints
+
+LinkDict = Dict[str, Any]
+
+ArchiveOutput = Union[str, Exception, None]
+
+@dataclass(frozen=True)
+class ArchiveResult:
+    cmd: List[str]
+    pwd: Optional[str]
+    cmd_version: Optional[str]
+    output: ArchiveOutput
+    status: str
+    start_ts: datetime
+    end_ts: datetime
+    schema: str = 'ArchiveResult'
+
+    def __post_init__(self):
+        self.typecheck()
+
+    def _asdict(self):
+        return asdict(self)
+
+    def typecheck(self) -> None:
+        assert self.schema == self.__class__.__name__
+        assert isinstance(self.status, str) and self.status
+        assert isinstance(self.start_ts, datetime)
+        assert isinstance(self.end_ts, datetime)
+        assert isinstance(self.cmd, list)
+        assert all(isinstance(arg, str) and arg for arg in self.cmd)
+        assert self.pwd is None or isinstance(self.pwd, str) and self.pwd
+        assert self.cmd_version is None or isinstance(self.cmd_version, str) and self.cmd_version
+        assert self.output is None or isinstance(self.output, (str, Exception))
+        if isinstance(self.output, str):
+            assert self.output
+
+    @classmethod
+    def from_json(cls, json_info):
+        from ..util import parse_date
+
+        info = {
+            key: val
+            for key, val in json_info.items()
+            if key in cls.field_names()
+        }
+        info['start_ts'] = parse_date(info['start_ts'])
+        info['end_ts'] = parse_date(info['end_ts'])
+        return cls(**info)
+
+    def to_json(self, indent=4, sort_keys=True):
+        from ..util import to_json
+
+        return to_json(self, indent=indent, sort_keys=sort_keys)
+
+    def to_csv(self, cols=None, ljust: int=0, separator: str=','):
+        from ..util import to_json
+
+        cols = cols or self.field_names()
+        return separator.join(
+            to_json(getattr(self, col), indent=None).ljust(ljust)
+            for col in cols
+        )
+    
+    @classmethod
+    def field_names(cls):
+        return [f.name for f in fields(cls)]
+
+    @property
+    def duration(self) -> int:
+        return (self.end_ts - self.start_ts).seconds
+
+@dataclass(frozen=True)
+class Link:
+    timestamp: str
+    url: str
+    title: Optional[str]
+    tags: Optional[str]
+    sources: List[str]
+    history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {})
+    updated: Optional[datetime] = None
+    schema: str = 'Link'
+
+    def __str__(self) -> str:
+        return f'[{self.timestamp}] {self.base_url} "{self.title}"'
+
+    def __post_init__(self):
+        self.typecheck()
+
+    def overwrite(self, **kwargs):
+        """pure functional version of dict.update that returns a new instance"""
+        return Link(**{**self._asdict(), **kwargs})
+
+    def __eq__(self, other):
+        if not isinstance(other, Link):
+            return NotImplemented
+        return self.url == other.url
+
+    def __gt__(self, other):
+        if not isinstance(other, Link):
+            return NotImplemented
+        if not self.timestamp or not other.timestamp:
+            return 
+        return float(self.timestamp) > float(other.timestamp)
+
+    def typecheck(self) -> None:
+        from ..config import stderr, ANSI
+        try:
+            assert self.schema == self.__class__.__name__
+            assert isinstance(self.timestamp, str) and self.timestamp
+            assert self.timestamp.replace('.', '').isdigit()
+            assert isinstance(self.url, str) and '://' in self.url
+            assert self.updated is None or isinstance(self.updated, datetime)
+            assert self.title is None or (isinstance(self.title, str) and self.title)
+            assert self.tags is None or (isinstance(self.tags, str) and self.tags)
+            assert isinstance(self.sources, list)
+            assert all(isinstance(source, str) and source for source in self.sources)
+            assert isinstance(self.history, dict)
+            for method, results in self.history.items():
+                assert isinstance(method, str) and method
+                assert isinstance(results, list)
+                assert all(isinstance(result, ArchiveResult) for result in results)
+        except Exception:
+            stderr('{red}[X] Error while loading link! [{}] {} "{}"{reset}'.format(self.timestamp, self.url, self.title, **ANSI))
+            raise
+    
+    def _asdict(self, extended=False):
+        info = {
+            'schema': 'Link',
+            'url': self.url,
+            'title': self.title or None,
+            'timestamp': self.timestamp,
+            'updated': self.updated or None,
+            'tags': self.tags or None,
+            'sources': self.sources or [],
+            'history': self.history or {},
+        }
+        if extended:
+            info.update({
+                'link_dir': self.link_dir,
+                'archive_path': self.archive_path,
+                
+                'hash': self.url_hash,
+                'base_url': self.base_url,
+                'scheme': self.scheme,
+                'domain': self.domain,
+                'path': self.path,
+                'basename': self.basename,
+                'extension': self.extension,
+                'is_static': self.is_static,
+
+                'bookmarked_date': self.bookmarked_date,
+                'updated_date': self.updated_date,
+                'oldest_archive_date': self.oldest_archive_date,
+                'newest_archive_date': self.newest_archive_date,
+        
+                'is_archived': self.is_archived,
+                'num_outputs': self.num_outputs,
+                'num_failures': self.num_failures,
+                
+                'latest': self.latest_outputs(),
+                'canonical': self.canonical_outputs(),
+            })
+        return info
+
+    @classmethod
+    def from_json(cls, json_info):
+        from ..util import parse_date
+        
+        info = {
+            key: val
+            for key, val in json_info.items()
+            if key in cls.field_names()
+        }
+        info['updated'] = parse_date(info['updated'])
+        info['sources'] = info.get('sources') or []
+
+        json_history = info.get('history') or {}
+        cast_history = {}
+
+        for method, method_history in json_history.items():
+            cast_history[method] = []
+            for json_result in method_history:
+                assert isinstance(json_result, dict), 'Items in Link["history"][method] must be dicts'
+                cast_result = ArchiveResult.from_json(json_result)
+                cast_history[method].append(cast_result)
+
+        info['history'] = cast_history
+        return cls(**info)
+
+    def to_json(self, indent=4, sort_keys=True):
+        from ..util import to_json
+
+        return to_json(self, indent=indent, sort_keys=sort_keys)
+
+    def to_csv(self, csv_cols: List[str], ljust: int=0, separator: str=','):
+        from ..util import to_json
+
+        return separator.join(
+            to_json(getattr(self, col), indent=None).ljust(ljust)
+            for col in csv_cols
+        )
+
+    @classmethod
+    def field_names(cls):
+        return [f.name for f in fields(cls)]
+
+    @property
+    def link_dir(self) -> str:
+        from ..config import CONFIG
+        return os.path.join(CONFIG['ARCHIVE_DIR'], self.timestamp)
+
+    @property
+    def archive_path(self) -> str:
+        from ..config import ARCHIVE_DIR_NAME
+        return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
+    
+    ### URL Helpers
+    @property
+    def url_hash(self):
+        from ..util import hashurl
+
+        return hashurl(self.url)
+
+    @property
+    def scheme(self) -> str:
+        from ..util import scheme
+        return scheme(self.url)
+
+    @property
+    def extension(self) -> str:
+        from ..util import extension
+        return extension(self.url)
+
+    @property
+    def domain(self) -> str:
+        from ..util import domain
+        return domain(self.url)
+
+    @property
+    def path(self) -> str:
+        from ..util import path
+        return path(self.url)
+
+    @property
+    def basename(self) -> str:
+        from ..util import basename
+        return basename(self.url)
+
+    @property
+    def base_url(self) -> str:
+        from ..util import base_url
+        return base_url(self.url)
+
+    ### Pretty Printing Helpers
+    @property
+    def bookmarked_date(self) -> Optional[str]:
+        from ..util import ts_to_date
+        return ts_to_date(self.timestamp) if self.timestamp else None
+
+    @property
+    def updated_date(self) -> Optional[str]:
+        from ..util import ts_to_date
+        return ts_to_date(self.updated) if self.updated else None
+
+    @property
+    def archive_dates(self) -> List[datetime]:
+        return [
+            result.start_ts
+            for method in self.history.keys()
+                for result in self.history[method]
+        ]
+
+    @property
+    def oldest_archive_date(self) -> Optional[datetime]:
+        return min(self.archive_dates, default=None)
+
+    @property
+    def newest_archive_date(self) -> Optional[datetime]:
+        return max(self.archive_dates, default=None)
+
+    ### Archive Status Helpers
+    @property
+    def num_outputs(self) -> int:
+        return len(tuple(filter(None, self.latest_outputs().values())))
+
+    @property
+    def num_failures(self) -> int:
+        return sum(1
+                   for method in self.history.keys()
+                       for result in self.history[method]
+                            if result.status == 'failed')
+
+    @property
+    def is_static(self) -> bool:
+        from ..util import is_static_file
+        return is_static_file(self.url)
+
+    @property
+    def is_archived(self) -> bool:
+        from ..config import ARCHIVE_DIR
+        from ..util import domain
+
+        output_paths = (
+            domain(self.url),
+            'output.pdf',
+            'screenshot.png',
+            'output.html',
+            'media',
+        )
+
+        return any(
+            os.path.exists(os.path.join(ARCHIVE_DIR, self.timestamp, path))
+            for path in output_paths
+        )
+
+    def latest_outputs(self, status: str=None) -> Dict[str, ArchiveOutput]:
+        """get the latest output that each archive method produced for link"""
+        
+        ARCHIVE_METHODS = (
+            'title', 'favicon', 'wget', 'warc', 'pdf',
+            'screenshot', 'dom', 'git', 'media', 'archive_org',
+        )
+        latest: Dict[str, ArchiveOutput] = {}
+        for archive_method in ARCHIVE_METHODS:
+            # get most recent succesful result in history for each archive method
+            history = self.history.get(archive_method) or []
+            history = list(filter(lambda result: result.output, reversed(history)))
+            if status is not None:
+                history = list(filter(lambda result: result.status == status, history))
+
+            history = list(history)
+            if history:
+                latest[archive_method] = history[0].output
+            else:
+                latest[archive_method] = None
+
+        return latest
+
+
+    def canonical_outputs(self) -> Dict[str, Optional[str]]:
+        """predict the expected output paths that should be present after archiving"""
+
+        from ..util import wget_output_path
+        canonical = {
+            'index_path': 'index.html',
+            'favicon_path': 'favicon.ico',
+            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
+            'wget_path': wget_output_path(self),
+            'warc_path': 'warc',
+            'pdf_path': 'output.pdf',
+            'screenshot_path': 'screenshot.png',
+            'dom_path': 'output.html',
+            'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
+            'git_path': 'git',
+            'media_path': 'media',
+        }
+        if self.is_static:
+            # static binary files like PDF and images are handled slightly differently.
+            # they're just downloaded once and aren't archived separately multiple times, 
+            # so the wget, screenshot, & pdf urls should all point to the same file
+
+            static_path = wget_output_path(self)
+            canonical.update({
+                'title': self.basename,
+                'wget_path': static_path,
+                'pdf_path': static_path,
+                'screenshot_path': static_path,
+                'dom_path': static_path,
+            })
+        return canonical

archivebox/index/sql.py

@@ -0,0 +1,75 @@
+__package__ = 'archivebox.index'
+
+from io import StringIO
+from typing import List, Tuple, Iterator
+
+from .schema import Link
+from ..util import enforce_types
+from ..config import setup_django, OUTPUT_DIR
+
+
+### Main Links Index
+
+@enforce_types
+def parse_sql_main_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]:
+    setup_django(out_dir, check_db=True)
+    from core.models import Page
+
+    return (
+        Link.from_json(page.as_json(*Page.keys))
+        for page in Page.objects.all()
+    )
+
+@enforce_types
+def write_sql_main_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None:
+    setup_django(out_dir, check_db=True)
+    from core.models import Page
+
+    all_urls = {link.url: link for link in links}
+
+    for page in Page.objects.all():
+        if page.url in all_urls:
+            info = {k: v for k, v in all_urls.pop(page.url)._asdict().items() if k in Page.keys}
+            Page.objects.update(**info)
+        else:
+            page.delete()
+
+    for url, link in all_urls.items():
+        info = {k: v for k, v in link._asdict().items() if k in Page.keys}
+        Page.objects.update_or_create(url=url, defaults=info)
+
+
+
+@enforce_types
+def list_migrations(out_dir: str=OUTPUT_DIR) -> List[Tuple[bool, str]]:
+    setup_django(out_dir, check_db=False)
+    from django.core.management import call_command
+    out = StringIO()
+    call_command("showmigrations", list=True, stdout=out)
+    out.seek(0)
+    migrations = []
+    for line in out.readlines():
+        if line.strip() and ']' in line:
+            status_str, name_str = line.strip().split(']', 1)
+            is_applied = 'X' in status_str
+            migration_name = name_str.strip()
+            migrations.append((is_applied, migration_name))
+
+    return migrations
+
+@enforce_types
+def apply_migrations(out_dir: str=OUTPUT_DIR) -> List[str]:
+    setup_django(out_dir, check_db=False)
+    from django.core.management import call_command
+    null, out = StringIO(), StringIO()
+    call_command("makemigrations", interactive=False, stdout=null)
+    call_command("migrate", interactive=False, stdout=out)
+    out.seek(0)
+
+    return [line.strip() for line in out.readlines() if line.strip()]
+
+@enforce_types
+def get_admins(out_dir: str=OUTPUT_DIR) -> List[str]:
+    setup_django(out_dir, check_db=False)
+    from django.contrib.auth.models import User
+    return User.objects.filter(is_superuser=True)