From 158f145d9a9a53bae982bbdadce06e96d56da7d5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 30 Apr 2019 23:10:48 -0400 Subject: [PATCH] move docstrings to main.py out of cli files --- archivebox/cli/__init__.py | 5 +-- archivebox/cli/archivebox.py | 3 +- archivebox/cli/archivebox_add.py | 10 ++--- archivebox/cli/archivebox_config.py | 8 ++-- archivebox/cli/archivebox_help.py | 9 +++-- archivebox/cli/archivebox_info.py | 9 +++-- archivebox/cli/archivebox_init.py | 9 +++-- archivebox/cli/archivebox_list.py | 9 +++-- archivebox/cli/archivebox_manage.py | 4 +- archivebox/cli/archivebox_remove.py | 9 +++-- archivebox/cli/archivebox_schedule.py | 9 +++-- archivebox/cli/archivebox_server.py | 9 +++-- archivebox/cli/archivebox_shell.py | 9 +++-- archivebox/cli/archivebox_update.py | 12 +++--- archivebox/cli/archivebox_version.py | 9 +++-- archivebox/main.py | 54 +++++++++++++++++++++------ 16 files changed, 107 insertions(+), 70 deletions(-) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index f4cd99b9..7972c02e 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -33,9 +33,8 @@ def list_subcommands() -> Dict[str, str]: subcommand = filename.replace('archivebox_', '').replace('.py', '') module = import_module('.archivebox_{}'.format(subcommand), __package__) assert is_valid_cli_module(module, subcommand) - COMMANDS.append((subcommand, module.__description__)) # type: ignore + COMMANDS.append((subcommand, module.main.__doc__)) globals()[subcommand] = module.main - module.main.__doc__ = module.__description__ display_order = lambda cmd: ( display_first.index(cmd[0]) @@ -50,7 +49,7 @@ def run_subcommand(subcommand: str, subcommand_args: List[str]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: - """run a given ArchiveBox subcommand with the given list of args""" + """Run a given ArchiveBox subcommand with the given list of args""" module = import_module('.archivebox_{}'.format(subcommand), __package__) module.main(args=subcommand_args, stdin=stdin, pwd=pwd) # type: ignore diff --git a/archivebox/cli/archivebox.py b/archivebox/cli/archivebox.py index d6fe207c..c8281937 100755 --- a/archivebox/cli/archivebox.py +++ b/archivebox/cli/archivebox.py @@ -3,7 +3,6 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox' -__description__ = 'ArchiveBox: The self-hosted internet archive.' import sys import argparse @@ -18,7 +17,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional subcommands = list_subcommands() parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description='ArchiveBox: The self-hosted internet archive', add_help=False, ) group = parser.add_mutually_exclusive_group() diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index d0209916..272fe5cf 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -2,29 +2,29 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox add' -__description__ = 'Add a new URL or list of URLs to your archive' import sys import argparse from typing import List, Optional, IO -from ..main import add -from ..util import SmartFormatter, accept_stdin +from ..main import add, docstring from ..config import OUTPUT_DIR, ONLY_NEW +from .logging import SmartFormatter, accept_stdin +@docstring(add.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=add.__doc__, add_help=True, formatter_class=SmartFormatter, ) parser.add_argument( '--update-all', #'-n', action='store_true', - default=not ONLY_NEW, + default=not ONLY_NEW, # when ONLY_NEW=True we skip updating old links help="Also retry previously skipped/failed links when adding new links", ) parser.add_argument( diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 2d373535..314f84f5 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -2,22 +2,22 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox config' -__description__ = 'Get and set your ArchiveBox project configuration values' import sys import argparse from typing import Optional, List, IO -from ..main import config -from ..util import SmartFormatter, accept_stdin +from ..main import config, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, accept_stdin +@docstring(config.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=config.__doc__, add_help=True, formatter_class=SmartFormatter, ) diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py index b1cf1c5b..94f65c48 100755 --- a/archivebox/cli/archivebox_help.py +++ b/archivebox/cli/archivebox_help.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox help' -__description__ = 'Print the ArchiveBox help message and usage' import sys import argparse from typing import Optional, List, IO -from ..main import help -from ..util import reject_stdin +from ..main import help, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, reject_stdin +@docstring(help.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=help.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.parse_args(args or ()) reject_stdin(__command__, stdin) diff --git a/archivebox/cli/archivebox_info.py b/archivebox/cli/archivebox_info.py index d3cc99aa..814690b8 100644 --- a/archivebox/cli/archivebox_info.py +++ b/archivebox/cli/archivebox_info.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox info' -__description__ = 'Print out some info and statistics about the archive collection' import sys import argparse from typing import Optional, List, IO -from ..main import info +from ..main import info, docstring from ..config import OUTPUT_DIR -from ..util import reject_stdin +from .logging import SmartFormatter, reject_stdin +@docstring(info.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=info.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.parse_args(args or ()) reject_stdin(__command__, stdin) diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index a66f011c..352f5c86 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox init' -__description__ = 'Initialize a new ArchiveBox collection in the current directory' import sys import argparse from typing import Optional, List, IO -from ..main import init -from ..util import reject_stdin +from ..main import init, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, reject_stdin +@docstring(init.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=init.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.parse_args(args or ()) reject_stdin(__command__, stdin) diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py index 126ad144..716688c6 100644 --- a/archivebox/cli/archivebox_list.py +++ b/archivebox/cli/archivebox_list.py @@ -2,15 +2,13 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox list' -__description__ = 'List, filter, and export information about archive entries' import sys import argparse from typing import Optional, List, IO -from ..main import list_all -from ..util import SmartFormatter, accept_stdin +from ..main import list_all, docstring from ..config import OUTPUT_DIR from ..index import ( get_indexed_folders, @@ -24,11 +22,14 @@ from ..index import ( get_corrupted_folders, get_unrecognized_folders, ) +from .logging import SmartFormatter, accept_stdin + +@docstring(list_all.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=list_all.__doc__, add_help=True, formatter_class=SmartFormatter, ) diff --git a/archivebox/cli/archivebox_manage.py b/archivebox/cli/archivebox_manage.py index f2b91cc2..cab5d565 100644 --- a/archivebox/cli/archivebox_manage.py +++ b/archivebox/cli/archivebox_manage.py @@ -2,16 +2,16 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox manage' -__description__ = 'Run an ArchiveBox Django management command' import sys from typing import Optional, List, IO -from ..main import manage +from ..main import manage, docstring from ..config import OUTPUT_DIR +@docstring(manage.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: manage( args=args, diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index c5f5ff53..735ad52f 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox remove' -__description__ = 'Remove the specified URLs from the archive.' import sys import argparse from typing import Optional, List, IO -from ..main import remove -from ..util import accept_stdin +from ..main import remove, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, accept_stdin +@docstring(remove.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=remove.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.add_argument( '--yes', # '-y', diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index b6a15e13..be9a02a9 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox schedule' -__description__ = 'Set ArchiveBox to regularly import URLs at specific times using cron' import sys import argparse from typing import Optional, List, IO -from ..main import schedule -from ..util import reject_stdin +from ..main import schedule, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, reject_stdin +@docstring(schedule.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=schedule.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.add_argument( '--quiet', '-q', diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index a5cf4b2c..d6547b8a 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox server' -__description__ = 'Run the ArchiveBox HTTP server' import sys import argparse from typing import Optional, List, IO -from ..main import server -from ..util import reject_stdin +from ..main import server, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, reject_stdin +@docstring(server.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=server.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.add_argument( 'runserver_args', diff --git a/archivebox/cli/archivebox_shell.py b/archivebox/cli/archivebox_shell.py index 04939328..a053c7be 100644 --- a/archivebox/cli/archivebox_shell.py +++ b/archivebox/cli/archivebox_shell.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox shell' -__description__ = 'Enter an interactive ArchiveBox Django shell' import sys import argparse from typing import Optional, List, IO -from ..main import shell +from ..main import shell, docstring from ..config import OUTPUT_DIR -from ..util import reject_stdin +from .logging import SmartFormatter, reject_stdin +@docstring(shell.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=shell.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.parse_args(args or ()) reject_stdin(__command__, stdin) diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 936e45ec..5088897d 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -2,15 +2,13 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox update' -__description__ = 'Import any new links from subscriptions and retry any previously failed/skipped links' import sys import argparse from typing import List, Optional, IO -from ..main import update -from ..util import SmartFormatter, accept_stdin +from ..main import update, docstring from ..config import OUTPUT_DIR from ..index import ( get_indexed_folders, @@ -24,12 +22,14 @@ from ..index import ( get_corrupted_folders, get_unrecognized_folders, ) +from .logging import SmartFormatter, accept_stdin +@docstring(update.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=update.__doc__, add_help=True, formatter_class=SmartFormatter, ) @@ -99,9 +99,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional nargs='*', type=str, default=None, - help='List only URLs matching these filter patterns.' + help='Update only URLs matching these filter patterns.' ) - command = parser.parse_args(args) + command = parser.parse_args(args or ()) filter_patterns_str = accept_stdin(stdin) update( diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py index d3707161..50b5e5c2 100755 --- a/archivebox/cli/archivebox_version.py +++ b/archivebox/cli/archivebox_version.py @@ -2,23 +2,24 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox version' -__description__ = 'Print the ArchiveBox version and dependency information' import sys import argparse from typing import Optional, List, IO -from ..main import version -from ..util import reject_stdin +from ..main import version, docstring from ..config import OUTPUT_DIR +from .logging import SmartFormatter, reject_stdin +@docstring(version.__doc__) def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: parser = argparse.ArgumentParser( prog=__command__, - description=__description__, + description=version.__doc__, add_help=True, + formatter_class=SmartFormatter, ) parser.add_argument( '--quiet', '-q', diff --git a/archivebox/main.py b/archivebox/main.py index 76c10cca..ebf51b50 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -118,7 +118,10 @@ ALLOWED_IN_OUTPUT_DIR = { FAVICON_FILENAME, } +@enforce_types def help(out_dir: str=OUTPUT_DIR) -> None: + """Print the ArchiveBox help message and usage""" + all_subcommands = list_subcommands() COMMANDS_HELP_TEXT = '\n '.join( f'{cmd.ljust(20)} {summary}' @@ -182,7 +185,11 @@ def help(out_dir: str=OUTPUT_DIR) -> None: print(' https://github.com/pirate/ArchiveBox/wiki') -def version(quiet: bool=False, out_dir: str=OUTPUT_DIR) -> None: +@enforce_types +def version(quiet: bool=False, + out_dir: str=OUTPUT_DIR) -> None: + """Print the ArchiveBox version and dependency information""" + if quiet: print(VERSION) else: @@ -191,37 +198,44 @@ def version(quiet: bool=False, out_dir: str=OUTPUT_DIR) -> None: print('{white}[i] Dependency versions:{reset}'.format(**ANSI)) for name, dependency in DEPENDENCIES.items(): - print_dependency_version(name, dependency) + print(printable_dependency_version(name, dependency)) print() print('{white}[i] Code locations:{reset}'.format(**ANSI)) for name, folder in CODE_LOCATIONS.items(): - print_folder_status(name, folder) + print(printable_folder_status(name, folder)) print() print('{white}[i] External locations:{reset}'.format(**ANSI)) for name, folder in EXTERNAL_LOCATIONS.items(): - print_folder_status(name, folder) + print(printable_folder_status(name, folder)) print() print('{white}[i] Data locations:{reset}'.format(**ANSI)) for name, folder in DATA_LOCATIONS.items(): - print_folder_status(name, folder) + print(printable_folder_status(name, folder)) print() check_dependencies() -def run(subcommand: str, subcommand_args: Optional[List[str]], stdin: Optional[IO]=None, out_dir: str=OUTPUT_DIR) -> None: +@enforce_types +def run(subcommand: str, + subcommand_args: Optional[List[str]], + stdin: Optional[IO]=None, + out_dir: str=OUTPUT_DIR) -> None: + """Run a given ArchiveBox subcommand with the given list of args""" run_subcommand( subcommand=subcommand, subcommand_args=subcommand_args, stdin=stdin, - out_dir=out_dir, + pwd=out_dir, ) +@enforce_types def init(out_dir: str=OUTPUT_DIR) -> None: + """Initialize a new ArchiveBox collection in the current directory""" os.makedirs(out_dir, exist_ok=True) is_empty = not len(set(os.listdir(out_dir)) - ALLOWED_IN_OUTPUT_DIR) @@ -364,7 +378,10 @@ def init(out_dir: str=OUTPUT_DIR) -> None: print(' archivebox help') +@enforce_types def info(out_dir: str=OUTPUT_DIR) -> None: + """Print out some info and statistics about the archive collection""" + check_data_folder(out_dir=out_dir) print('{green}[*] Scanning archive collection main index...{reset}'.format(**ANSI)) @@ -454,6 +471,7 @@ def add(import_str: Optional[str]=None, update_all: bool=not ONLY_NEW, index_only: bool=False, out_dir: str=OUTPUT_DIR) -> List[Link]: + """Add a new URL or list of URLs to your archive""" check_data_folder(out_dir=out_dir) @@ -518,6 +536,7 @@ def remove(filter_str: Optional[str]=None, yes: bool=False, delete: bool=False, out_dir: str=OUTPUT_DIR) -> List[Link]: + """Remove the specified URLs from the archive""" check_data_folder(out_dir=out_dir) @@ -586,7 +605,7 @@ def remove(filter_str: Optional[str]=None, @enforce_types def update(resume: Optional[float]=None, - only_new: bool=not ONLY_NEW, + only_new: bool=ONLY_NEW, index_only: bool=False, overwrite: bool=False, filter_patterns_str: Optional[str]=None, @@ -596,6 +615,7 @@ def update(resume: Optional[float]=None, after: Optional[str]=None, before: Optional[str]=None, out_dir: str=OUTPUT_DIR) -> List[Link]: + """Import any new links from subscriptions and retry any previously failed/skipped links""" check_dependencies() check_data_folder(out_dir=out_dir) @@ -659,8 +679,9 @@ def list_all(filter_patterns_str: Optional[str]=None, before: Optional[float]=None, sort: Optional[str]=None, csv: Optional[str]=None, - json: Optional[str]=None, + json: bool=False, out_dir: str=OUTPUT_DIR) -> Iterable[Link]: + """List, filter, and export information about archive entries""" check_data_folder(out_dir=out_dir) @@ -756,12 +777,14 @@ def list_folders(links: List[Link], raise ValueError('Status not recognized.') +@enforce_types def config(config_options_str: Optional[str]=None, config_options: Optional[List[str]]=None, get: bool=False, set: bool=False, reset: bool=False, out_dir: str=OUTPUT_DIR) -> None: + """Get and set your ArchiveBox project configuration values""" check_data_folder(out_dir=out_dir) @@ -863,6 +886,7 @@ def schedule(add: bool=False, every: Optional[str]=None, import_path: Optional[str]=None, out_dir: str=OUTPUT_DIR): + """Set ArchiveBox to regularly import URLs at specific times using cron""" check_data_folder(out_dir=out_dir) @@ -957,10 +981,13 @@ def schedule(add: bool=False, raise SystemExit(0) +@enforce_types +def server(runserver_args: Optional[List[str]]=None, + reload: bool=False, + debug: bool=False, + out_dir: str=OUTPUT_DIR) -> None: + """Run the ArchiveBox HTTP server""" - - -def server(runserver_args: Optional[List[str]]=None, reload: bool=False, out_dir: str=OUTPUT_DIR) -> None: runserver_args = runserver_args or [] check_data_folder(out_dir=out_dir) @@ -982,7 +1009,10 @@ def server(runserver_args: Optional[List[str]]=None, reload: bool=False, out_dir call_command("runserver", *runserver_args) +@enforce_types def manage(args: Optional[List[str]]=None, out_dir: str=OUTPUT_DIR) -> None: + """Run an ArchiveBox Django management command""" + check_data_folder(out_dir=out_dir) setup_django(out_dir)