diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index be156127..7e212eea 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -225,21 +225,11 @@ class ConstantsDict(Mapping): 'enabled': True, 'is_valid': STATIC_DIR.exists(), }, - 'CUSTOM_TEMPLATES_DIR': { - 'path': CUSTOM_TEMPLATES_DIR.resolve(), - 'enabled': True, - 'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(), - }, 'LIB_DIR': { 'path': LIB_DIR.resolve(), 'enabled': True, 'is_valid': LIB_DIR.is_dir(), }, - 'TMP_DIR': { - 'path': TMP_DIR.resolve(), - 'enabled': True, - 'is_valid': TMP_DIR.is_dir(), - }, }) DATA_LOCATIONS = benedict({ @@ -277,20 +267,35 @@ class ConstantsDict(Mapping): "enabled": True, "is_valid": SOURCES_DIR.exists(), }, - "PERSONAS_DIR": { - "path": PERSONAS_DIR.resolve(), - "enabled": PERSONAS_DIR.exists(), - "is_valid": PERSONAS_DIR.exists(), - }, "LOGS_DIR": { "path": LOGS_DIR.resolve(), "enabled": True, "is_valid": LOGS_DIR.is_dir(), }, - "CACHE_DIR": { - "path": CACHE_DIR.resolve(), - "enabled": True, - "is_valid": CACHE_DIR.is_dir(), + # "CACHE_DIR": { + # "path": CACHE_DIR.resolve(), + # "enabled": True, + # "is_valid": CACHE_DIR.is_dir(), + # }, + 'TMP_DIR': { + 'path': TMP_DIR.resolve(), + 'enabled': True, + 'is_valid': TMP_DIR.is_dir(), + }, + "PERSONAS_DIR": { + "path": PERSONAS_DIR.resolve(), + "enabled": PERSONAS_DIR.is_dir(), + "is_valid": PERSONAS_DIR.is_dir(), + }, + 'CUSTOM_TEMPLATES_DIR': { + 'path': CUSTOM_TEMPLATES_DIR.resolve(), + 'enabled': CUSTOM_TEMPLATES_DIR.is_dir(), + 'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(), + }, + 'USER_PLUGINS_DIR': { + 'path': USER_PLUGINS_DIR.resolve(), + 'enabled': USER_PLUGINS_DIR.is_dir(), + 'is_valid': USER_PLUGINS_DIR.is_dir(), }, }) diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index 5fddbe97..a6a07160 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -567,7 +567,7 @@ def printable_folder_status(name: str, folder: Dict) -> str: else: color, symbol, note, num_files = 'red', 'X', 'invalid', '?' else: - color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-' + color, symbol, note, num_files = 'lightyellow', '-', 'unused', '-' if folder['path']: diff --git a/archivebox/main.py b/archivebox/main.py index 0ea360c4..548799bb 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -290,52 +290,52 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat is_empty = not len(set(os.listdir(out_dir)) - CONSTANTS.ALLOWED_IN_DATA_DIR) if (out_dir / CONSTANTS.JSON_INDEX_FILENAME).exists(): - stderr("[!] This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.", color="lightyellow") - stderr(" You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.", color="lightyellow") + print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr) + print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr) existing_index = CONSTANTS.DATABASE_FILE.exists() if is_empty and not existing_index: - print('{green}[+] Initializing a new ArchiveBox v{} collection...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI)) - print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) + print(f'[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]') + print('[green]----------------------------------------------------------------------[/green]') elif existing_index: # TODO: properly detect and print the existing version in current index as well - print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI)) - print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) + print(f'[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]') + print('[green]----------------------------------------------------------------------[/green]') else: if force: - stderr('[!] This folder appears to already have files in it, but no index.sqlite3 is present.', color='lightyellow') - stderr(' Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).') + print('[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]') + print('[red] Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]') else: - stderr( - ("{red}[X] This folder appears to already have files in it, but no index.sqlite3 present.{reset}\n\n" + print( + ("[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n" " You must run init in a completely empty directory, or an existing data folder.\n\n" - " {lightred}Hint:{reset} To import an existing data folder make sure to cd into the folder first, \n" + " [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n" " then run and run 'archivebox init' to pick up where you left off.\n\n" " (Always make sure your data folder is backed up first before updating ArchiveBox)" - ).format(**SHELL_CONFIG.ANSI) + ) ) raise SystemExit(2) if existing_index: - print('\n{green}[*] Verifying archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI)) + print('\n[green][*] Verifying archive folder structure...[/green]') else: - print('\n{green}[+] Building archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI)) + print('\n[green][+] Building archive folder structure...[/green]') print(f' + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...') Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True) Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True) Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True) print(f' + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...') - write_config_file({}, out_dir=out_dir) + write_config_file({}, out_dir=str(out_dir)) if CONSTANTS.DATABASE_FILE.exists(): - print('\n{green}[*] Verifying main SQL index and running any migrations needed...{reset}'.format(**SHELL_CONFIG.ANSI)) + print('\n[green][*] Verifying main SQL index and running any migrations needed...[/green]') else: - print('\n{green}[+] Building main SQL index and running initial migrations...{reset}'.format(**SHELL_CONFIG.ANSI)) + print('\n[green][+] Building main SQL index and running initial migrations...[/green]') for migration_line in apply_migrations(out_dir): - print(f' {migration_line}') + sys.stdout.write(f' {migration_line}\n') assert CONSTANTS.DATABASE_FILE.exists() print() @@ -347,14 +347,14 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat # call_command("createsuperuser", interactive=True) print() - print('{green}[*] Checking links from indexes and archive folders (safe to Ctrl+C)...{reset}'.format(**SHELL_CONFIG.ANSI)) + print('[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]') all_links = Snapshot.objects.none() pending_links: Dict[str, Link] = {} if existing_index: all_links = load_main_index(out_dir=out_dir, warn=False) - print(' √ Loaded {} links from existing main index.'.format(all_links.count())) + print(f' √ Loaded {all_links.count()} links from existing main index.') if quick: print(' > Skipping full snapshot directory check (quick mode)') @@ -363,9 +363,9 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat # Links in data folders that dont match their timestamp fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir) if fixed: - print(' {lightyellow}√ Fixed {} data directory locations that didn\'t match their link timestamps.{reset}'.format(len(fixed), **SHELL_CONFIG.ANSI)) + print(f' [yellow]√ Fixed {len(fixed)} data directory locations that didn\'t match their link timestamps.[/yellow]') if cant_fix: - print(' {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **SHELL_CONFIG.ANSI)) + print(f' [red]! Could not fix {len(cant_fix)} data directory locations due to conflicts with existing folders.[/red]') # Links in JSON index but not in main index orphaned_json_links = { @@ -375,7 +375,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat } if orphaned_json_links: pending_links.update(orphaned_json_links) - print(' {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **SHELL_CONFIG.ANSI)) + print(f' [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]') # Links in data dir indexes but not in main index orphaned_data_dir_links = { @@ -385,7 +385,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat } if orphaned_data_dir_links: pending_links.update(orphaned_data_dir_links) - print(' {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **SHELL_CONFIG.ANSI)) + print(f' [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]') # Links in invalid/duplicate data dirs invalid_folders = { @@ -393,36 +393,36 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items() } if invalid_folders: - print(' {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **SHELL_CONFIG.ANSI)) + print(f' [red]! Skipped adding {len(invalid_folders)} invalid link data directories.[/red]') print(' X ' + '\n X '.join(f'./{Path(folder).relative_to(DATA_DIR)} {link}' for folder, link in invalid_folders.items())) print() - print(' {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**SHELL_CONFIG.ANSI)) + print(' [violet]Hint:[/violet] For more information about the link data directories that were skipped, run:') print(' archivebox status') print(' archivebox list --status=invalid') except (KeyboardInterrupt, SystemExit): - stderr() - stderr('[x] Stopped checking archive directories due to Ctrl-C/SIGTERM', color='red') - stderr(' Your archive data is safe, but you should re-run `archivebox init` to finish the process later.') - stderr() - stderr(' {lightred}Hint:{reset} In the future you can run a quick init without checking dirs like so:'.format(**SHELL_CONFIG.ANSI)) - stderr(' archivebox init --quick') + print(file=sys.stderr) + print('[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]', file=sys.stderr) + print(' Your archive data is safe, but you should re-run `archivebox init` to finish the process later.', file=sys.stderr) + print(file=sys.stderr) + print(' [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:', file=sys.stderr) + print(' archivebox init --quick', file=sys.stderr) raise SystemExit(1) write_main_index(list(pending_links.values()), out_dir=out_dir) - print('\n{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) + print('\n[green]----------------------------------------------------------------------[/green]') from django.contrib.auth.models import User if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists(): - print('{green}[+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.{reset}'.format(**SHELL_CONFIG.ANSI)) + print('[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]') User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD) if existing_index: - print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**SHELL_CONFIG.ANSI)) + print('[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]') else: - print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links) + len(pending_links), **SHELL_CONFIG.ANSI)) + print(f'[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]') json_index = out_dir / CONSTANTS.JSON_INDEX_FILENAME html_index = out_dir / CONSTANTS.HTML_INDEX_FILENAME @@ -437,7 +437,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat if Snapshot.objects.count() < 25: # hide the hints for experienced users print() - print(' {lightred}Hint:{reset} To view your archive index, run:'.format(**SHELL_CONFIG.ANSI)) + print(' [violet]Hint:[/violet] To view your archive index, run:') print(' archivebox server # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]') print() print(' To add new links, you can run:') diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 4dcf1f0a..5e324cbb 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -36,6 +36,6 @@ def check_migrations(): CONSTANTS.SOURCES_DIR.mkdir(exist_ok=True) CONSTANTS.LOGS_DIR.mkdir(exist_ok=True) - CONSTANTS.CACHE_DIR.mkdir(exist_ok=True) + # CONSTANTS.CACHE_DIR.mkdir(exist_ok=True) (CONSTANTS.LIB_DIR / 'bin').mkdir(exist_ok=True, parents=True) (CONSTANTS.PERSONAS_DIR / 'Default').mkdir(exist_ok=True, parents=True)