mirrors/streamrip
1
0
Fork 0
mirror of https://github.com/nathom/streamrip.git synced 2025-05-24 20:14:42 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use ThreadPoolExecutor instead of threading

Browse source 
Threading caused errors related to discarded connection pools and too many files open.
This commit is contained in:
nathom 2021-04-19 13:41:32 -07:00
parent 383159d5f4
commit 540d557d26
1 changed files with 18 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

37
streamrip/core.py
View file

@ -2,7 +2,7 @@ import logging
import os import os
import re import re
import sys import sys
import threading import concurrent.futures
from getpass import getpass from getpass import getpass
from hashlib import md5 from hashlib import md5
from string import Formatter from string import Formatter
@ -132,8 +132,8 @@ class MusicDL(list):
"new_tracknumbers": self.config.session["metadata"][ "new_tracknumbers": self.config.session["metadata"][
"new_playlist_tracknumbers" "new_playlist_tracknumbers"
], ],
"download_videos": self.config.session['tidal']['download_videos'], "download_videos": self.config.session["tidal"]["download_videos"],
'download_booklets': self.config.session['qobuz']['download_booklets'], "download_booklets": self.config.session["qobuz"]["download_booklets"],
} }
def download(self): def download(self):
@ -142,7 +142,8 @@ class MusicDL(list):
except KeyError: except KeyError:
click.secho( click.secho(
"Updating config file... Some settings may be lost. Please run the " "Updating config file... Some settings may be lost. Please run the "
"command again.", fg="magenta" "command again.",
fg="magenta",
) )
self.config.update() self.config.update()
exit() exit()
@ -186,8 +187,8 @@ class MusicDL(list):
item.download(**arguments) item.download(**arguments)
if isinstance(item, Track): if isinstance(item, Track):
item.tag() item.tag()
if arguments['conversion']['enabled']: if arguments["conversion"]["enabled"]:
item.convert(**arguments['conversion']) item.convert(**arguments["conversion"])
if self.db != [] and hasattr(item, "id"): if self.db != [] and hasattr(item, "id"):
self.db.add(item.id) self.db.add(item.id)
@ -280,7 +281,7 @@ class MusicDL(list):
global tracks_not_found global tracks_not_found
try: try:
track = next(self.search(lastfm_source, query, media_type="track")) track = next(self.search(lastfm_source, query, media_type="track"))
if self.config.session['metadata']['set_playlist_to_album']: if self.config.session["metadata"]["set_playlist_to_album"]:
track.version = track.work = None track.version = track.work = None
playlist.append(track) playlist.append(track)
except NoResultsFound: except NoResultsFound:
@ -294,18 +295,16 @@ class MusicDL(list):
pl = Playlist(client=self.get_client(lastfm_source), name=title) pl = Playlist(client=self.get_client(lastfm_source), name=title)
pl.creator = user_regex.search(purl).group(1) pl.creator = user_regex.search(purl).group(1)
processes = [] with concurrent.futures.ThreadPoolExecutor(max_workers=15) as executor:
futures = [
for title, artist in queries: executor.submit(search_query, f"{title} {artist}", pl)
query = f"{title} {artist}" for title, artist in queries
proc = threading.Thread( ]
target=search_query, args=(query, pl), daemon=True # only for the progress bar
) for f in tqdm(
proc.start() concurrent.futures.as_completed(futures), total=len(futures)
processes.append(proc) ):
pass
for proc in tqdm(processes, unit="tracks", desc="Searching"):
proc.join()
pl.loaded = True pl.loaded = True
click.secho(f"{tracks_not_found} tracks not found.", fg="yellow") click.secho(f"{tracks_not_found} tracks not found.", fg="yellow")