move logging_util into archivebox.misc subfolder

This commit is contained in:
Nick Sweeting 2024-11-18 19:08:49 -08:00
parent e469c5a344
commit 4a5d607296
No known key found for this signature in database
44 changed files with 89 additions and 113 deletions
archivebox
cli
config
core
extractors
index
main.py
misc
parsers
pkgs
abx-plugin-archivedotorg/abx_plugin_archivedotorg
abx-plugin-chrome/abx_plugin_chrome
abx-plugin-curl/abx_plugin_curl
abx-plugin-favicon/abx_plugin_favicon
abx-plugin-git/abx_plugin_git
abx-plugin-htmltotext/abx_plugin_htmltotext
abx-plugin-mercury/abx_plugin_mercury
abx-plugin-readability/abx_plugin_readability
abx-plugin-singlefile/abx_plugin_singlefile
abx-plugin-title/abx_plugin_title
abx-plugin-wget/abx_plugin_wget
abx-plugin-ytdlp/abx_plugin_ytdlp
workers

View file

@ -246,7 +246,7 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
command.subcommand = 'help'
if command.subcommand not in ('version',):
from ..logging_util import log_cli_command
from archivebox.misc.logging_util import log_cli_command
log_cli_command(
subcommand=command.subcommand,

View file

@ -18,8 +18,8 @@ from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.django import setup_django
from archivebox.config.permissions import USER, HOSTNAME
from archivebox.misc.checks import check_data_folder
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
from archivebox.parsers import PARSERS
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
if TYPE_CHECKING:

View file

@ -11,8 +11,8 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
from ..main import config
from ..logging_util import SmartFormatter, accept_stdin
@docstring(config.__doc__)

View file

@ -9,9 +9,9 @@ from pathlib import Path
from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from archivebox.config import DATA_DIR
from ..main import help
from ..logging_util import SmartFormatter, reject_stdin
@docstring(help.__doc__)

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import init

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import install

View file

@ -8,8 +8,9 @@ import argparse
from pathlib import Path
from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.misc.util import docstring
from archivebox.misc.logging_util import SmartFormatter, reject_stdin, stderr
from ..main import list_all
from ..index import (
LINK_FILTERS,
@ -24,7 +25,6 @@ from ..index import (
get_corrupted_folders,
get_unrecognized_folders,
)
from ..logging_util import SmartFormatter, reject_stdin, stderr
@docstring(list_all.__doc__)

View file

@ -11,7 +11,7 @@ from typing import List, Optional, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin, stderr
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
from ..main import oneshot

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
from ..main import remove

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import schedule

View file

@ -11,7 +11,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import server
@docstring(server.__doc__)

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import shell

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import status

View file

@ -21,7 +21,7 @@ from archivebox.index import (
get_corrupted_folders,
get_unrecognized_folders,
)
from archivebox.logging_util import SmartFormatter, accept_stdin
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
# from ..main import update
def update():

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
# from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR, VERSION
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
# @docstring(version.__doc__)

View file

@ -21,7 +21,7 @@ from collections.abc import Mapping
from benedict import benedict
from ..misc.logging import DEFAULT_CLI_COLORS
from archivebox.misc.logging import DEFAULT_CLI_COLORS
from .paths import (
PACKAGE_DIR,

View file

@ -10,10 +10,10 @@ from rich.console import Console
import django
from . import CONSTANTS
from archivebox.misc import logging
from . import CONSTANTS
from .common import SHELL_CONFIG
from ..misc import logging
if not SHELL_CONFIG.USE_COLOR:

View file

@ -114,7 +114,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
"""Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
from archivebox.logging_util import pretty_path
from archivebox.misc.logging_util import pretty_path
try:
socket_path = str(dir_path / '.test_socket.sock')

View file

@ -17,9 +17,8 @@ from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.misc.util import htmldecode, urldecode
from archivebox.misc.paginators import AccelleratedPaginator
from archivebox.misc.logging_util import printable_filesize
from archivebox.search.admin import SearchResultsAdminMixin
from archivebox.logging_util import printable_filesize
from archivebox.index.html import snapshot_icons
from archivebox.extractors import archive_links
from archivebox.main import remove

View file

@ -11,8 +11,7 @@ import pydantic
import django.template
from archivebox.config import CONSTANTS
from ..misc.logging import IS_TTY
from archivebox.misc.logging import IS_TTY
IGNORABLE_URL_PATTERNS = [

View file

@ -31,8 +31,7 @@ from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.logging_util import printable_filesize
from archivebox.misc.logging_util import printable_filesize
from archivebox.search import query_search_index

View file

@ -17,7 +17,7 @@ from ..index import (
write_link_details,
)
from archivebox.misc.util import enforce_types
from ..logging_util import (
from archivebox.misc.logging_util import (
log_archiving_started,
log_archiving_paused,
log_archiving_finished,

View file

@ -12,13 +12,12 @@ from urllib.parse import urlparse
from django.db.models import QuerySet, Q
from archivebox.misc.logging import stderr
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
from ..logging_util import (
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
from archivebox.misc.logging import stderr
from archivebox.misc.logging_util import (
TimedProgress,
log_indexing_process_started,
log_indexing_process_finished,

View file

@ -21,9 +21,9 @@ from archivebox.misc.util import (
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.version import get_COMMIT_HASH
from archivebox.misc.logging_util import printable_filesize
from .schema import Link
from ..logging_util import printable_filesize
MAIN_INDEX_TEMPLATE = 'static_index.html'
MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'

View file

@ -18,20 +18,30 @@ from abx_pkg import Binary
import abx
import archivebox
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
from archivebox.config.permissions import SudoPermission, IN_DOCKER
from archivebox.config.collection import write_config_file, load_all_config, get_real_name
from archivebox.misc.checks import check_data_folder
from archivebox.misc.util import enforce_types # type: ignore
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
from archivebox.misc.system import run as run_shell
from archivebox.misc.logging import stderr, hint
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
from archivebox.config.permissions import SudoPermission, IN_DOCKER
from archivebox.config.collection import (
write_config_file,
load_all_config,
get_real_name,
from archivebox.misc.logging_util import (
TimedProgress,
log_importing_started,
log_crawl_started,
log_removal_started,
log_removal_finished,
log_list_started,
log_list_finished,
printable_config,
printable_folders,
printable_filesize,
printable_folder_status,
)
from .cli import (
CLI_SUBCOMMANDS,
run_subcommand,
@ -78,19 +88,6 @@ from .index.sql import (
from .index.html import generate_index_from_links
from .index.csv import links_to_csv
from .extractors import archive_links, archive_link, ignore_methods
from .logging_util import (
TimedProgress,
log_importing_started,
log_crawl_started,
log_removal_started,
log_removal_finished,
log_list_started,
log_list_finished,
printable_config,
printable_folders,
printable_filesize,
printable_folder_status,
)
@enforce_types

View file

@ -157,9 +157,9 @@ def check_data_dir_permissions():
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.common import STORAGE_CONFIG
from archivebox.logging_util import pretty_path
tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
@ -204,9 +204,9 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
import archivebox
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
from archivebox.config.common import STORAGE_CONFIG
from archivebox.logging_util import pretty_path
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR

View file

@ -15,7 +15,7 @@ from dataclasses import dataclass
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
if TYPE_CHECKING:
from .index.schema import Link, ArchiveResult
from ..index.schema import Link, ArchiveResult
from rich import print
from rich.panel import Panel
@ -303,14 +303,14 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
print()
if resume:
print('[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]'.format(
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
resume,
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
resume,
))
else:
print('[green][▶] [{}] Starting archiving of {} snapshots in index...[/]'.format(
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
))
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
@ -400,8 +400,9 @@ def log_archive_method_started(method: str):
def log_archive_method_finished(result: "ArchiveResult"):
"""quote the argument with whitespace in a command so the user can
copy-paste the outputted string directly to run the cmd
"""
quote the argument with whitespace in a command so the user can
copy-paste the outputted string directly to run the cmd
"""
# Prettify CMD string and make it safe to copy-paste by quoting arguments
quoted_cmd = ' '.join(
@ -467,7 +468,7 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
print(' {}'.format(' '.join(filter_patterns or ())))
def log_list_finished(links):
from .index.csv import links_to_csv
from ..index.csv import links_to_csv
print()
print('---------------------------------------------------------------------------------------------------')
print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
@ -545,8 +546,7 @@ def printable_filesize(num_bytes: Union[int, float]) -> str:
@enforce_types
def printable_folders(folders: Dict[str, Optional["Link"]],
with_headers: bool=False) -> str:
def printable_folders(folders: Dict[str, Optional["Link"]], with_headers: bool=False) -> str:
return '\n'.join(
f'{folder} {link and link.url} "{link and link.title}"'
for folder, link in folders.items()

View file

@ -17,6 +17,7 @@ from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG
from archivebox.misc.system import atomic_write
from archivebox.misc.logging import stderr, hint
from archivebox.misc.logging_util import TimedProgress, log_source_saved
from archivebox.misc.util import (
basename,
htmldecode,
@ -25,7 +26,6 @@ from archivebox.misc.util import (
)
from ..index.schema import Link
from ..logging_util import TimedProgress, log_source_saved
from . import pocket_api
from . import readwise_reader_api

View file

@ -5,8 +5,8 @@ from pathlib import Path
from typing import Optional, List, Dict, Tuple
from collections import defaultdict
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe

View file

@ -10,9 +10,9 @@ from abx_plugin_default_binproviders import env
from archivebox.config import CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
from archivebox.misc.logging import STDERR
from archivebox.misc.util import dedupe
from archivebox.logging_util import pretty_path
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
CHROMIUM_BINARY_NAMES_LINUX = [

View file

@ -5,11 +5,8 @@ from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.system import run, chmod_file, atomic_write
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.logging_util import TimedProgress
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.misc.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -4,12 +4,9 @@ from pathlib import Path
from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -5,8 +5,8 @@ from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -4,14 +4,10 @@ from pathlib import Path
from typing import Optional
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
from archivebox.misc.system import atomic_write
from archivebox.misc.util import (
enforce_types,
get_headers,
dedupe,
)
from archivebox.misc.util import enforce_types, get_headers, dedupe
from archivebox.misc.logging_util import TimedProgress
from .binaries import CURL_BINARY
from .config import CURL_CONFIG

View file

@ -4,8 +4,8 @@ from pathlib import Path
from archivebox.misc.system import chmod_file, run
from archivebox.misc.util import enforce_types, domain, dedupe
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
from archivebox.logging_util import TimedProgress
from abx_plugin_curl.config import CURL_CONFIG
from abx_plugin_curl.binaries import CURL_BINARY

View file

@ -13,7 +13,7 @@ from archivebox.misc.util import (
without_query,
without_fragment,
)
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from abx_plugin_git.config import GIT_CONFIG

View file

@ -9,7 +9,7 @@ from archivebox.config import VERSION
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.misc.system import atomic_write
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from abx_plugin_title.extractor import get_html

View file

@ -6,13 +6,10 @@ from subprocess import CompletedProcess
from typing import Optional, List
import json
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.misc.system import run, atomic_write
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.misc.util import enforce_types, is_static_file
from .config import MERCURY_CONFIG
from .binaries import MERCURY_BINARY

View file

@ -1,15 +1,15 @@
__package__ = 'abx_plugin_readability'
import json
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional
import json
from archivebox.misc.system import run, atomic_write
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from abx_plugin_title.extractor import get_html
from .config import READABILITY_CONFIG

View file

@ -1,17 +1,17 @@
__package__ = 'abx_plugin_singlefile'
from pathlib import Path
from typing import Optional
import json
from pathlib import Path
from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from abx_plugin_chrome.config import CHROME_CONFIG
from abx_plugin_chrome.binaries import CHROME_BINARY
from .config import SINGLEFILE_CONFIG
from .binaries import SINGLEFILE_BINARY

View file

@ -6,13 +6,9 @@ from pathlib import Path
from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.util import (
enforce_types,
download_url,
htmldecode,
dedupe,
)
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.util import enforce_types, download_url, htmldecode, dedupe)
from abx_plugin_curl.config import CURL_CONFIG
from abx_plugin_curl.binaries import CURL_BINARY

View file

@ -2,13 +2,12 @@ __package__ = 'abx_plugin_wget'
import re
import os
from pathlib import Path
from typing import Optional
from pathlib import Path
from datetime import datetime, timezone
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import (
enforce_types,
@ -19,6 +18,7 @@ from archivebox.misc.util import (
urldecode,
dedupe,
)
from .config import WGET_CONFIG
from .binaries import WGET_BINARY

View file

@ -6,7 +6,7 @@ from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from .config import YTDLP_CONFIG
from .binaries import YTDLP_BINARY

View file

@ -19,7 +19,7 @@ from archivebox.config import CONSTANTS
from archivebox.config.paths import get_or_create_working_tmp_dir
from archivebox.config.permissions import ARCHIVEBOX_USER
from archivebox.misc.logging import STDERR
from archivebox.logging_util import pretty_path
from archivebox.misc.logging_util import pretty_path
LOG_FILE_NAME = "supervisord.log"
CONFIG_FILE_NAME = "supervisord.conf"