From 6006b4f93b5fb51d7b2fa4074c6720d46ec65b91 Mon Sep 17 00:00:00 2001
From: Cristian <cristian@swapps.com>
Date: Fri, 24 Jul 2020 12:25:25 -0500
Subject: [PATCH] refactor: Organize code to remove flake8 issues
---
archivebox/cli/archivebox_add.py | 3 ++-
archivebox/cli/archivebox_config.py | 3 ++-
archivebox/cli/archivebox_help.py | 3 ++-
archivebox/cli/archivebox_init.py | 3 ++-
archivebox/cli/archivebox_list.py | 3 ++-
archivebox/cli/archivebox_manage.py | 3 ++-
archivebox/cli/archivebox_remove.py | 3 ++-
archivebox/cli/archivebox_schedule.py | 3 ++-
archivebox/cli/archivebox_server.py | 3 ++-
archivebox/cli/archivebox_shell.py | 3 ++-
archivebox/cli/archivebox_status.py | 3 ++-
archivebox/cli/archivebox_update.py | 3 ++-
archivebox/cli/archivebox_version.py | 3 ++-
archivebox/cli/tests.py | 2 +-
archivebox/config/__init__.py | 2 +-
archivebox/core/settings.py | 1 -
archivebox/core/tests.py | 2 +-
archivebox/core/views.py | 3 +--
archivebox/core/welcome_message.py | 1 -
archivebox/index/__init__.py | 5 ++---
archivebox/logging_util.py | 10 ++++------
archivebox/main.py | 7 +++----
archivebox/parsers/__init__.py | 4 +---
archivebox/util.py | 6 +++---
24 files changed, 43 insertions(+), 39 deletions(-)
diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py
index de6c3dbc..b9c06a55 100644
--- a/archivebox/cli/archivebox_add.py
+++ b/archivebox/cli/archivebox_add.py
@@ -8,7 +8,8 @@ import argparse
from typing import List, Optional, IO
-from ..main import add, docstring
+from ..main import add
+from ..util import docstring
from ..config import OUTPUT_DIR, ONLY_NEW
from ..logging_util import SmartFormatter, accept_stdin, stderr
diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py
index 7dffbfad..f81286c6 100644
--- a/archivebox/cli/archivebox_config.py
+++ b/archivebox/cli/archivebox_config.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import config, docstring
+from ..main import config
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, accept_stdin
diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py
index 0b245a03..46f17cbc 100755
--- a/archivebox/cli/archivebox_help.py
+++ b/archivebox/cli/archivebox_help.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import help, docstring
+from ..main import help
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py
index e27b26d9..6255ef26 100755
--- a/archivebox/cli/archivebox_init.py
+++ b/archivebox/cli/archivebox_init.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import init, docstring
+from ..main import init
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py
index 2f2180b0..95c5cc4e 100644
--- a/archivebox/cli/archivebox_list.py
+++ b/archivebox/cli/archivebox_list.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import list_all, docstring
+from ..main import list_all
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..index import (
get_indexed_folders,
diff --git a/archivebox/cli/archivebox_manage.py b/archivebox/cli/archivebox_manage.py
index cab5d565..f05604e1 100644
--- a/archivebox/cli/archivebox_manage.py
+++ b/archivebox/cli/archivebox_manage.py
@@ -7,7 +7,8 @@ import sys
from typing import Optional, List, IO
-from ..main import manage, docstring
+from ..main import manage
+from ..util import docstring
from ..config import OUTPUT_DIR
diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py
index 36779e87..8fe717fb 100644
--- a/archivebox/cli/archivebox_remove.py
+++ b/archivebox/cli/archivebox_remove.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import remove, docstring
+from ..main import remove
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, accept_stdin
diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py
index 4dfa2769..ffd09f22 100644
--- a/archivebox/cli/archivebox_schedule.py
+++ b/archivebox/cli/archivebox_schedule.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import schedule, docstring
+from ..main import schedule
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py
index 0d529a40..a5c168cc 100644
--- a/archivebox/cli/archivebox_server.py
+++ b/archivebox/cli/archivebox_server.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import server, docstring
+from ..main import server
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_shell.py b/archivebox/cli/archivebox_shell.py
index 5d5498cc..bcd5fdd6 100644
--- a/archivebox/cli/archivebox_shell.py
+++ b/archivebox/cli/archivebox_shell.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import shell, docstring
+from ..main import shell
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py
index c0bed3fe..2bef19c7 100644
--- a/archivebox/cli/archivebox_status.py
+++ b/archivebox/cli/archivebox_status.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import status, docstring
+from ..main import status
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py
index 663e03f3..9d483362 100644
--- a/archivebox/cli/archivebox_update.py
+++ b/archivebox/cli/archivebox_update.py
@@ -8,7 +8,8 @@ import argparse
from typing import List, Optional, IO
-from ..main import update, docstring
+from ..main import update
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..index import (
get_indexed_folders,
diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py
index c56ab43f..e7922f37 100755
--- a/archivebox/cli/archivebox_version.py
+++ b/archivebox/cli/archivebox_version.py
@@ -8,7 +8,8 @@ import argparse
from typing import Optional, List, IO
-from ..main import version, docstring
+from ..main import version
+from ..util import docstring
from ..config import OUTPUT_DIR
from ..logging_util import SmartFormatter, reject_stdin
diff --git a/archivebox/cli/tests.py b/archivebox/cli/tests.py
index 14d0e4c6..1f44784d 100755
--- a/archivebox/cli/tests.py
+++ b/archivebox/cli/tests.py
@@ -198,7 +198,7 @@ class TestRemove(unittest.TestCase):
def test_remove_regex(self):
with output_hidden():
- archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', 'http(s)?:\/\/(.+\.)?(example\d\.com)'])
+ archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])
all_links = load_main_index(out_dir=OUTPUT_DIR)
assert len(all_links) == 4
diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py
index 87bedbd6..12579dbe 100644
--- a/archivebox/config/__init__.py
+++ b/archivebox/config/__init__.py
@@ -868,4 +868,4 @@ def setup_django(out_dir: str=None, check_db=False, config: ConfigDict=CONFIG) -
except KeyboardInterrupt:
raise SystemExit(2)
-os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8))
+os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8)) # noqa: F821
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 9a28b4f6..e9cc0dc8 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -8,7 +8,6 @@ from django.utils.crypto import get_random_string
from ..config import (
OUTPUT_DIR,
SECRET_KEY,
- DEBUG,
ALLOWED_HOSTS,
PYTHON_DIR,
ACTIVE_THEME,
diff --git a/archivebox/core/tests.py b/archivebox/core/tests.py
index 7ce503c2..4d66077c 100644
--- a/archivebox/core/tests.py
+++ b/archivebox/core/tests.py
@@ -1,3 +1,3 @@
-from django.test import TestCase
+#from django.test import TestCase
# Create your tests here.
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index ce6e8f58..c411e98b 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -4,7 +4,6 @@ from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.views import View, static
-from django.conf import settings
from core.models import Snapshot
@@ -75,7 +74,7 @@ class AddLinks(View):
}
add_stdout = StringIO()
with redirect_stdout(add_stdout):
- extracted_links = add(**input_kwargs)
+ add(**input_kwargs)
print(add_stdout.getvalue())
context = {
diff --git a/archivebox/core/welcome_message.py b/archivebox/core/welcome_message.py
index a948c78e..ed5d2d77 100644
--- a/archivebox/core/welcome_message.py
+++ b/archivebox/core/welcome_message.py
@@ -2,5 +2,4 @@ from archivebox.logging_util import log_shell_welcome_msg
if __name__ == '__main__':
- from main import *
log_shell_welcome_msg()
diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py
index b4e918b8..09c4d8a3 100644
--- a/archivebox/index/__init__.py
+++ b/archivebox/index/__init__.py
@@ -294,7 +294,6 @@ def parse_links_from_source(source_path: str) -> Tuple[List[Link], List[Link]]:
def dedupe_links(existing_links: List[Link],
new_links: List[Link]) -> Tuple[List[Link], List[Link]]:
- from ..parsers import parse_links
# merge existing links in out_dir and new links
all_links = validate_links(existing_links + new_links)
all_link_urls = {link.url for link in existing_links}
@@ -537,7 +536,7 @@ def get_unrecognized_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Option
link_guessed = parse_json_link_details(entry.path, guess=True)
write_json_link_details(link_guessed, out_dir=entry.path)
link = parse_json_link_details(entry.path)
- except Exception as e:
+ except Exception:
pass
if index_exists and link is None:
@@ -565,7 +564,7 @@ def is_valid(link: Link) -> bool:
try:
parsed_link = parse_json_link_details(link.link_dir, guess=True)
return link.url == parsed_link.url
- except Exception as e:
+ except Exception:
pass
return False
diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py
index 361a1aec..d60cf5fe 100644
--- a/archivebox/logging_util.py
+++ b/archivebox/logging_util.py
@@ -5,8 +5,6 @@ import os
import sys
import time
import argparse
-import logging
-import signal
from multiprocessing import Process
from datetime import datetime
@@ -263,9 +261,9 @@ def log_archiving_finished(num_links: int):
assert _LAST_RUN_STATS.archiving_start_ts is not None
seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
if seconds > 60:
- duration = '{0:.2f} min'.format(seconds / 60, 2)
+ duration = '{0:.2f} min'.format(seconds / 60)
else:
- duration = '{0:.2f} sec'.format(seconds, 2)
+ duration = '{0:.2f} sec'.format(seconds)
print()
print('{}[√] [{}] Update of {} pages complete ({}){}'.format(
@@ -385,8 +383,8 @@ def log_removal_started(links: List["Link"], yes: bool, delete: bool):
)
else:
print(
- f' Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
- f' (Pass --delete if you also want to permanently delete the data folders)'
+ ' Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+ ' (Pass --delete if you also want to permanently delete the data folders)'
)
if not yes:
diff --git a/archivebox/main.py b/archivebox/main.py
index 1c7068b3..1cb34b30 100644
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -20,10 +20,9 @@ from .parsers import (
save_file_as_source,
)
from .index.schema import Link
-from .util import enforce_types, docstring # type: ignore
+from .util import enforce_types # type: ignore
from .system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
from .index import (
- links_after_timestamp,
load_main_index,
parse_links_from_source,
dedupe_links,
@@ -291,7 +290,6 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None:
print('\n{green}[+] Building main SQL index and running migrations...{reset}'.format(**ANSI))
setup_django(out_dir, check_db=False)
- from django.conf import settings
DATABASE_FILE = os.path.join(out_dir, SQL_INDEX_FILENAME)
print(f' √ {DATABASE_FILE}')
print()
@@ -469,7 +467,8 @@ def status(out_dir: str=OUTPUT_DIR) -> None:
users = get_admins().values_list('username', flat=True)
print(f' UI users {len(users)}: {", ".join(users)}')
last_login = User.objects.order_by('last_login').last()
- print(f' Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}')
+ if last_login:
+ print(f' Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}')
last_updated = Snapshot.objects.order_by('updated').last()
print(f' Last changes: {str(last_updated.updated)[:16]}')
diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py
index 40899b06..20c8ef52 100644
--- a/archivebox/parsers/__init__.py
+++ b/archivebox/parsers/__init__.py
@@ -19,17 +19,15 @@ from ..config import (
OUTPUT_DIR,
SOURCES_DIR_NAME,
TIMEOUT,
- check_data_folder,
)
from ..util import (
basename,
- domain,
download_url,
enforce_types,
URL_REGEX,
)
from ..index.schema import Link
-from ..logging_util import pretty_path, TimedProgress, log_source_saved
+from ..logging_util import TimedProgress, log_source_saved
from .pocket_html import parse_pocket_html_export
from .pinboard_rss import parse_pinboard_rss_export
from .shaarli_rss import parse_shaarli_rss_export
diff --git a/archivebox/util.py b/archivebox/util.py
index c43585c0..ca940e30 100644
--- a/archivebox/util.py
+++ b/archivebox/util.py
@@ -226,11 +226,11 @@ def ansi_to_html(text):
argsdict = match.groupdict()
if argsdict['arg_3'] is None:
if argsdict['arg_2'] is None:
- bold, color = 0, argsdict['arg_1']
+ _, color = 0, argsdict['arg_1']
else:
- bold, color = argsdict['arg_1'], argsdict['arg_2']
+ _, color = argsdict['arg_1'], argsdict['arg_2']
else:
- bold, color = argsdict['arg_3'], argsdict['arg_2']
+ _, color = argsdict['arg_3'], argsdict['arg_2']
return TEMPLATE.format(COLOR_DICT[color][0])