mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 14:44:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix archivebox init colors and dir status checking

Browse source
This commit is contained in:
Nick Sweeting 2024-10-04 21:34:19 -07:00
parent 5323953f94
commit 0c7d7a2225
No known key found for this signature in database
4 changed files with 63 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

43
archivebox/config/constants.py
View file

@ -225,21 +225,11 @@ class ConstantsDict(Mapping):
'enabled': True, 'enabled': True,
'is_valid': STATIC_DIR.exists(), 'is_valid': STATIC_DIR.exists(),
}, },
'CUSTOM_TEMPLATES_DIR': {
'path': CUSTOM_TEMPLATES_DIR.resolve(),
'enabled': True,
'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(),
},
'LIB_DIR': { 'LIB_DIR': {
'path': LIB_DIR.resolve(), 'path': LIB_DIR.resolve(),
'enabled': True, 'enabled': True,
'is_valid': LIB_DIR.is_dir(), 'is_valid': LIB_DIR.is_dir(),
}, },
'TMP_DIR': {
'path': TMP_DIR.resolve(),
'enabled': True,
'is_valid': TMP_DIR.is_dir(),
},
}) })
DATA_LOCATIONS = benedict({ DATA_LOCATIONS = benedict({
@ -277,20 +267,35 @@ class ConstantsDict(Mapping):
"enabled": True, "enabled": True,
"is_valid": SOURCES_DIR.exists(), "is_valid": SOURCES_DIR.exists(),
}, },
"PERSONAS_DIR": {
"path": PERSONAS_DIR.resolve(),
"enabled": PERSONAS_DIR.exists(),
"is_valid": PERSONAS_DIR.exists(),
},
"LOGS_DIR": { "LOGS_DIR": {
"path": LOGS_DIR.resolve(), "path": LOGS_DIR.resolve(),
"enabled": True, "enabled": True,
"is_valid": LOGS_DIR.is_dir(), "is_valid": LOGS_DIR.is_dir(),
}, },
"CACHE_DIR": { # "CACHE_DIR": {
"path": CACHE_DIR.resolve(), # "path": CACHE_DIR.resolve(),
"enabled": True, # "enabled": True,
"is_valid": CACHE_DIR.is_dir(), # "is_valid": CACHE_DIR.is_dir(),
# },
'TMP_DIR': {
'path': TMP_DIR.resolve(),
'enabled': True,
'is_valid': TMP_DIR.is_dir(),
},
"PERSONAS_DIR": {
"path": PERSONAS_DIR.resolve(),
"enabled": PERSONAS_DIR.is_dir(),
"is_valid": PERSONAS_DIR.is_dir(),
},
'CUSTOM_TEMPLATES_DIR': {
'path': CUSTOM_TEMPLATES_DIR.resolve(),
'enabled': CUSTOM_TEMPLATES_DIR.is_dir(),
'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(),
},
'USER_PLUGINS_DIR': {
'path': USER_PLUGINS_DIR.resolve(),
'enabled': USER_PLUGINS_DIR.is_dir(),
'is_valid': USER_PLUGINS_DIR.is_dir(),
}, },
}) })

2
archivebox/logging_util.py
View file

@ -567,7 +567,7 @@ def printable_folder_status(name: str, folder: Dict) -> str:
else: else:
color, symbol, note, num_files = 'red', 'X', 'invalid', '?' color, symbol, note, num_files = 'red', 'X', 'invalid', '?'
else: else:
color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-' color, symbol, note, num_files = 'lightyellow', '-', 'unused', '-'
if folder['path']: if folder['path']:

74
archivebox/main.py
View file

@ -290,52 +290,52 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
is_empty = not len(set(os.listdir(out_dir)) - CONSTANTS.ALLOWED_IN_DATA_DIR) is_empty = not len(set(os.listdir(out_dir)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
if (out_dir / CONSTANTS.JSON_INDEX_FILENAME).exists(): if (out_dir / CONSTANTS.JSON_INDEX_FILENAME).exists():
stderr("[!] This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.", color="lightyellow") print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
stderr(" You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.", color="lightyellow") print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
existing_index = CONSTANTS.DATABASE_FILE.exists() existing_index = CONSTANTS.DATABASE_FILE.exists()
if is_empty and not existing_index: if is_empty and not existing_index:
print('{green}[+] Initializing a new ArchiveBox v{} collection...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI)) print(f'[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]')
print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) print('[green]----------------------------------------------------------------------[/green]')
elif existing_index: elif existing_index:
# TODO: properly detect and print the existing version in current index as well # TODO: properly detect and print the existing version in current index as well
print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI)) print(f'[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]')
print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) print('[green]----------------------------------------------------------------------[/green]')
else: else:
if force: if force:
stderr('[!] This folder appears to already have files in it, but no index.sqlite3 is present.', color='lightyellow') print('[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]')
stderr(' Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).') print('[red] Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]')
else: else:
stderr( print(
("{red}[X] This folder appears to already have files in it, but no index.sqlite3 present.{reset}\n\n" ("[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
" You must run init in a completely empty directory, or an existing data folder.\n\n" " You must run init in a completely empty directory, or an existing data folder.\n\n"
" {lightred}Hint:{reset} To import an existing data folder make sure to cd into the folder first, \n" " [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
" then run and run 'archivebox init' to pick up where you left off.\n\n" " then run and run 'archivebox init' to pick up where you left off.\n\n"
" (Always make sure your data folder is backed up first before updating ArchiveBox)" " (Always make sure your data folder is backed up first before updating ArchiveBox)"
).format(**SHELL_CONFIG.ANSI) )
) )
raise SystemExit(2) raise SystemExit(2)
if existing_index: if existing_index:
print('\n{green}[*] Verifying archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI)) print('\n[green][*] Verifying archive folder structure...[/green]')
else: else:
print('\n{green}[+] Building archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI)) print('\n[green][+] Building archive folder structure...[/green]')
print(f' + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...') print(f' + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...')
Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True) Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True) Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True) Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
print(f' + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...') print(f' + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...')
write_config_file({}, out_dir=out_dir) write_config_file({}, out_dir=str(out_dir))
if CONSTANTS.DATABASE_FILE.exists(): if CONSTANTS.DATABASE_FILE.exists():
print('\n{green}[*] Verifying main SQL index and running any migrations needed...{reset}'.format(**SHELL_CONFIG.ANSI)) print('\n[green][*] Verifying main SQL index and running any migrations needed...[/green]')
else: else:
print('\n{green}[+] Building main SQL index and running initial migrations...{reset}'.format(**SHELL_CONFIG.ANSI)) print('\n[green][+] Building main SQL index and running initial migrations...[/green]')
for migration_line in apply_migrations(out_dir): for migration_line in apply_migrations(out_dir):
print(f' {migration_line}') sys.stdout.write(f' {migration_line}\n')
assert CONSTANTS.DATABASE_FILE.exists() assert CONSTANTS.DATABASE_FILE.exists()
print() print()
@ -347,14 +347,14 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
# call_command("createsuperuser", interactive=True) # call_command("createsuperuser", interactive=True)
print() print()
print('{green}[*] Checking links from indexes and archive folders (safe to Ctrl+C)...{reset}'.format(**SHELL_CONFIG.ANSI)) print('[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]')
all_links = Snapshot.objects.none() all_links = Snapshot.objects.none()
pending_links: Dict[str, Link] = {} pending_links: Dict[str, Link] = {}
if existing_index: if existing_index:
all_links = load_main_index(out_dir=out_dir, warn=False) all_links = load_main_index(out_dir=out_dir, warn=False)
print(' √ Loaded {} links from existing main index.'.format(all_links.count())) print(f' √ Loaded {all_links.count()} links from existing main index.')
if quick: if quick:
print(' > Skipping full snapshot directory check (quick mode)') print(' > Skipping full snapshot directory check (quick mode)')
@ -363,9 +363,9 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
# Links in data folders that dont match their timestamp # Links in data folders that dont match their timestamp
fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir) fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir)
if fixed: if fixed:
print(' {lightyellow}√ Fixed {} data directory locations that didn\'t match their link timestamps.{reset}'.format(len(fixed), **SHELL_CONFIG.ANSI)) print(f' [yellow]√ Fixed {len(fixed)} data directory locations that didn\'t match their link timestamps.[/yellow]')
if cant_fix: if cant_fix:
print(' {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **SHELL_CONFIG.ANSI)) print(f' [red]! Could not fix {len(cant_fix)} data directory locations due to conflicts with existing folders.[/red]')
# Links in JSON index but not in main index # Links in JSON index but not in main index
orphaned_json_links = { orphaned_json_links = {
@ -375,7 +375,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
} }
if orphaned_json_links: if orphaned_json_links:
pending_links.update(orphaned_json_links) pending_links.update(orphaned_json_links)
print(' {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **SHELL_CONFIG.ANSI)) print(f' [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]')
# Links in data dir indexes but not in main index # Links in data dir indexes but not in main index
orphaned_data_dir_links = { orphaned_data_dir_links = {
@ -385,7 +385,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
} }
if orphaned_data_dir_links: if orphaned_data_dir_links:
pending_links.update(orphaned_data_dir_links) pending_links.update(orphaned_data_dir_links)
print(' {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **SHELL_CONFIG.ANSI)) print(f' [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]')
# Links in invalid/duplicate data dirs # Links in invalid/duplicate data dirs
invalid_folders = { invalid_folders = {
@ -393,36 +393,36 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items() for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items()
} }
if invalid_folders: if invalid_folders:
print(' {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **SHELL_CONFIG.ANSI)) print(f' [red]! Skipped adding {len(invalid_folders)} invalid link data directories.[/red]')
print(' X ' + '\n X '.join(f'./{Path(folder).relative_to(DATA_DIR)} {link}' for folder, link in invalid_folders.items())) print(' X ' + '\n X '.join(f'./{Path(folder).relative_to(DATA_DIR)} {link}' for folder, link in invalid_folders.items()))
print() print()
print(' {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**SHELL_CONFIG.ANSI)) print(' [violet]Hint:[/violet] For more information about the link data directories that were skipped, run:')
print(' archivebox status') print(' archivebox status')
print(' archivebox list --status=invalid') print(' archivebox list --status=invalid')
except (KeyboardInterrupt, SystemExit): except (KeyboardInterrupt, SystemExit):
stderr() print(file=sys.stderr)
stderr('[x] Stopped checking archive directories due to Ctrl-C/SIGTERM', color='red') print('[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]', file=sys.stderr)
stderr(' Your archive data is safe, but you should re-run `archivebox init` to finish the process later.') print(' Your archive data is safe, but you should re-run `archivebox init` to finish the process later.', file=sys.stderr)
stderr() print(file=sys.stderr)
stderr(' {lightred}Hint:{reset} In the future you can run a quick init without checking dirs like so:'.format(**SHELL_CONFIG.ANSI)) print(' [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:', file=sys.stderr)
stderr(' archivebox init --quick') print(' archivebox init --quick', file=sys.stderr)
raise SystemExit(1) raise SystemExit(1)
write_main_index(list(pending_links.values()), out_dir=out_dir) write_main_index(list(pending_links.values()), out_dir=out_dir)
print('\n{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI)) print('\n[green]----------------------------------------------------------------------[/green]')
from django.contrib.auth.models import User from django.contrib.auth.models import User
if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists(): if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists():
print('{green}[+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.{reset}'.format(**SHELL_CONFIG.ANSI)) print('[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]')
User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD) User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
if existing_index: if existing_index:
print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**SHELL_CONFIG.ANSI)) print('[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]')
else: else:
print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links) + len(pending_links), **SHELL_CONFIG.ANSI)) print(f'[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]')
json_index = out_dir / CONSTANTS.JSON_INDEX_FILENAME json_index = out_dir / CONSTANTS.JSON_INDEX_FILENAME
html_index = out_dir / CONSTANTS.HTML_INDEX_FILENAME html_index = out_dir / CONSTANTS.HTML_INDEX_FILENAME
@ -437,7 +437,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
if Snapshot.objects.count() < 25: # hide the hints for experienced users if Snapshot.objects.count() < 25: # hide the hints for experienced users
print() print()
print(' {lightred}Hint:{reset} To view your archive index, run:'.format(**SHELL_CONFIG.ANSI)) print(' [violet]Hint:[/violet] To view your archive index, run:')
print(' archivebox server # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]') print(' archivebox server # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]')
print() print()
print(' To add new links, you can run:') print(' To add new links, you can run:')

2
archivebox/misc/checks.py
View file

@ -36,6 +36,6 @@ def check_migrations():
CONSTANTS.SOURCES_DIR.mkdir(exist_ok=True) CONSTANTS.SOURCES_DIR.mkdir(exist_ok=True)
CONSTANTS.LOGS_DIR.mkdir(exist_ok=True) CONSTANTS.LOGS_DIR.mkdir(exist_ok=True)
CONSTANTS.CACHE_DIR.mkdir(exist_ok=True) # CONSTANTS.CACHE_DIR.mkdir(exist_ok=True)
(CONSTANTS.LIB_DIR / 'bin').mkdir(exist_ok=True, parents=True) (CONSTANTS.LIB_DIR / 'bin').mkdir(exist_ok=True, parents=True)
(CONSTANTS.PERSONAS_DIR / 'Default').mkdir(exist_ok=True, parents=True) (CONSTANTS.PERSONAS_DIR / 'Default').mkdir(exist_ok=True, parents=True)