mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-28 05:34:14 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip refactoring

Browse source
This commit is contained in:
Nick Sweeting 2024-01-19 07:09:24 -08:00
parent ea2c5a21ff
commit 3234a366d5
2 changed files with 79 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

23
archivebox/plugins/gallerydl/models.py
View file

@ -5,6 +5,9 @@ class GalleryDLDependency(SingletonModel):
GALLERYDL_ENABLED = models.BooleanField(default=True)
GALLERYDL_BINARY = models.CharField(max_length=255, default='gallery-dl')
# GALLERYDL_WORKERS = models.IntegerField(default='{NUM_CORES}')
def __str__(self):
return "GalleryDL Dependency Configuration"
@ -28,6 +31,20 @@ class GalleryDLDependency(SingletonModel):
return self.GALLERYDL_ENABLED and self.is_valid
def run(args, pwd, timeout):
errors = None
timer = TimedProgress(timeout, prefix=' ')
try:
proc = run(cmd=[self.bin_path, *args]=True, pwd=pwd, timeout=timeout)run(cmd=[self.bin_path, *args]=True, pwd=pwd, timeout=timeout)
except Exception as err:
errors = err
finally:
timer.end()
return proc, timer, errors
def pretty_version(self):
if self.enabled:
if self.is_valid:
@ -142,9 +159,8 @@ class GalleryDLExtractor(SingletonModel):
]
status, stdout, stderr, output_path = 'failed', '', '', None
timer = TimedProgress(timeout, prefix=' ')
try:
proc = run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT, text=True)
proc, timer, errors = self.GALLERYDL_DEPENDENCY.run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT)
stdout, stderr = proc.stdout, proc.stderr
if 'ERROR: Unsupported URL' in stderr:
@ -154,11 +170,8 @@ class GalleryDLExtractor(SingletonModel):
if proc.returncode == 0 and 'finished' in stdout:
output_path = extractor_dir / 'index.html'
status = 'succeeded'
except Exception as err:
stderr += err
finally:
timer.end()
num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)