From 51da383919bf141e728cdde69221df14913cc73d Mon Sep 17 00:00:00 2001 From: nathom Date: Thu, 22 Apr 2021 16:54:07 -0700 Subject: [PATCH 1/9] Add help for macOS Music app --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6ffdd40..c77fd82 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,14 @@ qobuz: concurrent_downloads: 'Download (and convert) tracks all at once, instead of sequentially. If you are converting the tracks, and/or have fast internet, this will substantially improve processing speed.' ``` +## Integration with macOS Music app + +`streamrip` was designed to be used seamlessly with the macOS Music app. To set it up, you need to find the `Automatically Add to Music.localized` folder inside the file given at `Music.app -> Preferences -> Files -> Music Media folder location`. Set the downloads folder to the path in the config file. + +Next, enable `conversion` and set the `codec` to `alac`. If you want to save space, set `sampling_rate` to `48000`. Finally, set `keep_hires_cover` to `false`. + +Now, you can download anything and it will appear in your Library! + ## Troubleshooting From 10ceecb55c294917cff501cf8c5fea53f298206a Mon Sep 17 00:00:00 2001 From: nathom Date: Thu, 22 Apr 2021 16:54:44 -0700 Subject: [PATCH 2/9] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a6e8b77..1456aad 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ StreamripDownloads *.wav *.log *.mp4 +*.opus +*.mkv +*.aac +*.pyc From 7347330a42698b10e228509c658abe1e54bf1a9b Mon Sep 17 00:00:00 2001 From: nathom Date: Thu, 22 Apr 2021 17:22:33 -0700 Subject: [PATCH 3/9] Add support for Youtube urls --- .gitignore | 1 + streamrip/bases.py | 38 +++++++++++++++++++++++++++++++++++++- streamrip/config.py | 3 +++ streamrip/constants.py | 2 +- streamrip/core.py | 25 +++++++++++++++++++------ streamrip/tracklists.py | 4 ++-- 6 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 1456aad..fc71f6f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ StreamripDownloads *.mkv *.aac *.pyc +*test.py diff --git a/streamrip/bases.py b/streamrip/bases.py index a9141bd..3a03ef7 100644 --- a/streamrip/bases.py +++ b/streamrip/bases.py @@ -765,7 +765,7 @@ class Tracklist(list): else: for item in self: - if self.client.source != 'soundcloud': + if self.client.source != "soundcloud": # soundcloud only gets metadata after `target` is called # message will be printed in `target` click.secho(f'\nDownloading "{item!s}"', fg="blue") @@ -951,3 +951,39 @@ class Tracklist(list): if isinstance(key, int): super().__setitem__(key, val) + + +class YoutubeVideo: + """Dummy class implemented for consistency with the Media API.""" + + class DummyClient: + source = 'youtube' + + def __init__(self, url: str): + self.url = url + self.client = self.DummyClient() + + def download(self, parent_folder='StreamripDownloads', **kwargs): + filename_formatter = "%(track_number)s.%(track)s.%(container)s" + filename = os.path.join(parent_folder, filename_formatter) + + p = subprocess.Popen( + [ + "youtube-dl", + "-x", + "--add-metadata", + "--audio-format", + "mp3", + "--embed-thumbnail", + "-o", + filename, + self.url, + ] + ) + p.wait() + + def load_meta(self, *args, **kwargs): + pass + + def tag(self, *args, **kwargs): + pass diff --git a/streamrip/config.py b/streamrip/config.py index 094db84..d89a1a1 100644 --- a/streamrip/config.py +++ b/streamrip/config.py @@ -69,6 +69,9 @@ class Config: "soundcloud": { "quality": 0, }, + "youtube": { + "quality": 0, + }, "database": {"enabled": True, "path": None}, "conversion": { "enabled": False, diff --git a/streamrip/constants.py b/streamrip/constants.py index 0ccc05b..272cb7d 100644 --- a/streamrip/constants.py +++ b/streamrip/constants.py @@ -145,7 +145,7 @@ LASTFM_URL_REGEX = r"https://www.last.fm/user/\w+/playlists/\w+" QOBUZ_INTERPRETER_URL_REGEX = ( r"https?://www\.qobuz\.com/\w\w-\w\w/interpreter/[-\w]+/[-\w]+" ) - +YOUTUBE_URL_REGEX = r"https://www\.youtube\.com/watch\?v=\w+" TIDAL_MAX_Q = 7 diff --git a/streamrip/core.py b/streamrip/core.py index 00bac6b..4f30b5d 100644 --- a/streamrip/core.py +++ b/streamrip/core.py @@ -12,13 +12,14 @@ import click import requests from tqdm import tqdm -from .bases import Track, Video +from .bases import Track, Video, YoutubeVideo from .clients import DeezerClient, QobuzClient, SoundCloudClient, TidalClient from .config import Config from .constants import ( CONFIG_PATH, DB_PATH, LASTFM_URL_REGEX, + YOUTUBE_URL_REGEX, MEDIA_TYPES, QOBUZ_INTERPRETER_URL_REGEX, SOUNDCLOUD_URL_REGEX, @@ -58,6 +59,7 @@ class MusicDL(list): self.soundcloud_url_parse = re.compile(SOUNDCLOUD_URL_REGEX) self.lastfm_url_parse = re.compile(LASTFM_URL_REGEX) self.interpreter_url_parse = re.compile(QOBUZ_INTERPRETER_URL_REGEX) + self.youtube_url_parse = re.compile(YOUTUBE_URL_REGEX) self.config = config if self.config is None: @@ -89,7 +91,17 @@ class MusicDL(list): :raises ParsingError """ - for source, url_type, item_id in self.parse_urls(url): + # youtube is handled by youtube-dl, so much of the + # processing is not necessary + youtube_urls = self.youtube_url_parse.findall(url) + if youtube_urls != []: + self.extend(YoutubeVideo(u) for u in youtube_urls) + + parsed = self.parse_urls(url) + if not parsed and len(self) == 0: + raise ParsingError(url) + + for source, url_type, item_id in parsed: if item_id in self.db: logger.info( f"ID {item_id} already downloaded, use --no-db to override." @@ -170,6 +182,10 @@ class MusicDL(list): item.client.source ) + if item is YoutubeVideo: + item.download(**arguments) + continue + arguments["quality"] = self.config.session[item.client.source]["quality"] if isinstance(item, Artist): filters_ = tuple( @@ -266,10 +282,7 @@ class MusicDL(list): logger.debug(f"Parsed urls: {parsed}") - if parsed != []: - return parsed - - raise ParsingError(f"Error parsing URL: `{url}`") + return parsed def handle_lastfm_urls(self, urls): # https://www.last.fm/user/nathan3895/playlists/12058911 diff --git a/streamrip/tracklists.py b/streamrip/tracklists.py index 71efc58..913c025 100644 --- a/streamrip/tracklists.py +++ b/streamrip/tracklists.py @@ -415,10 +415,10 @@ class Playlist(Tracklist): self.download_message() def _download_item(self, item: Track, **kwargs): - kwargs['parent_folder'] = self.folder + kwargs["parent_folder"] = self.folder if self.client.source == "soundcloud": item.load_meta() - click.secho(f"Downloading {item!s}", fg='blue') + click.secho(f"Downloading {item!s}", fg="blue") if kwargs.get("set_playlist_to_album", False): item["album"] = self.name From 201065516d97ad8fadf3ee6c964c770e158f8d5d Mon Sep 17 00:00:00 2001 From: nathom Date: Thu, 22 Apr 2021 18:03:22 -0700 Subject: [PATCH 4/9] Add support for Youtube video downloads --- streamrip/bases.py | 24 ++++++++++++++++++++++-- streamrip/config.py | 2 ++ streamrip/constants.py | 5 +++-- streamrip/core.py | 8 +++++++- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/streamrip/bases.py b/streamrip/bases.py index 3a03ef7..e707e3f 100644 --- a/streamrip/bases.py +++ b/streamrip/bases.py @@ -957,13 +957,19 @@ class YoutubeVideo: """Dummy class implemented for consistency with the Media API.""" class DummyClient: - source = 'youtube' + source = "youtube" def __init__(self, url: str): self.url = url self.client = self.DummyClient() - def download(self, parent_folder='StreamripDownloads', **kwargs): + def download( + self, + parent_folder="StreamripDownloads", + download_youtube_videos=False, + youtube_video_downloads_folder="StreamripDownloads", + **kwargs, + ): filename_formatter = "%(track_number)s.%(track)s.%(container)s" filename = os.path.join(parent_folder, filename_formatter) @@ -980,6 +986,20 @@ class YoutubeVideo: self.url, ] ) + + print(f"{download_youtube_videos=}") + if download_youtube_videos: + pv = subprocess.Popen( + [ + "youtube-dl", + "-o", + os.path.join( + youtube_video_downloads_folder, "%(title)s.%(container)s" + ), + self.url, + ] + ) + pv.wait() p.wait() def load_meta(self, *args, **kwargs): diff --git a/streamrip/config.py b/streamrip/config.py index d89a1a1..b8ecb2b 100644 --- a/streamrip/config.py +++ b/streamrip/config.py @@ -71,6 +71,8 @@ class Config: }, "youtube": { "quality": 0, + "download_videos": False, + "video_downloads_folder": DOWNLOADS_DIR, }, "database": {"enabled": True, "path": None}, "conversion": { diff --git a/streamrip/constants.py b/streamrip/constants.py index 272cb7d..4fc6d6b 100644 --- a/streamrip/constants.py +++ b/streamrip/constants.py @@ -12,7 +12,8 @@ CONFIG_PATH = os.path.join(CONFIG_DIR, "config.yaml") LOG_DIR = click.get_app_dir(APPNAME) DB_PATH = os.path.join(LOG_DIR, "downloads.db") -DOWNLOADS_DIR = os.path.join(Path.home(), "StreamripDownloads") +HOME = Path.home() +DOWNLOADS_DIR = os.path.join(HOME, "StreamripDownloads") AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0" @@ -145,7 +146,7 @@ LASTFM_URL_REGEX = r"https://www.last.fm/user/\w+/playlists/\w+" QOBUZ_INTERPRETER_URL_REGEX = ( r"https?://www\.qobuz\.com/\w\w-\w\w/interpreter/[-\w]+/[-\w]+" ) -YOUTUBE_URL_REGEX = r"https://www\.youtube\.com/watch\?v=\w+" +YOUTUBE_URL_REGEX = r"https://www\.youtube\.com/watch\?v=[-\w]+" TIDAL_MAX_Q = 7 diff --git a/streamrip/core.py b/streamrip/core.py index 4f30b5d..8075bed 100644 --- a/streamrip/core.py +++ b/streamrip/core.py @@ -147,6 +147,12 @@ class MusicDL(list): ], "download_videos": self.config.session["tidal"]["download_videos"], "download_booklets": self.config.session["qobuz"]["download_booklets"], + "download_youtube_videos": self.config.session["youtube"][ + "download_videos" + ], + "youtube_video_downloads_folder": self.config.session["youtube"][ + "video_downloads_folder" + ], } def download(self): @@ -169,7 +175,7 @@ class MusicDL(list): ) click.secho("rip config --reset ", fg="yellow", nl=False) click.secho("to reset it. You will need to log in again.", fg="red") - logger.debug(err) + click.secho(err, fg='red') exit() logger.debug("Arguments from config: %s", arguments) From d54e66af734067a5d308ec7231923af3bde09a68 Mon Sep 17 00:00:00 2001 From: nathom Date: Thu, 22 Apr 2021 18:10:16 -0700 Subject: [PATCH 5/9] Make youtube-dl downloads quiet --- streamrip/bases.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/streamrip/bases.py b/streamrip/bases.py index e707e3f..8f191da 100644 --- a/streamrip/bases.py +++ b/streamrip/bases.py @@ -970,13 +970,15 @@ class YoutubeVideo: youtube_video_downloads_folder="StreamripDownloads", **kwargs, ): + click.secho(f"Downloading url {self.url}", fg="blue") filename_formatter = "%(track_number)s.%(track)s.%(container)s" filename = os.path.join(parent_folder, filename_formatter) p = subprocess.Popen( [ "youtube-dl", - "-x", + "-x", # audio only + "-q", # quiet mode "--add-metadata", "--audio-format", "mp3", @@ -987,11 +989,12 @@ class YoutubeVideo: ] ) - print(f"{download_youtube_videos=}") if download_youtube_videos: + click.secho("Downloading video stream", fg='blue') pv = subprocess.Popen( [ "youtube-dl", + "-q", "-o", os.path.join( youtube_video_downloads_folder, "%(title)s.%(container)s" From 905a65d10d3a14d8b58a579da2fdc02d2f6b4e7b Mon Sep 17 00:00:00 2001 From: nathom Date: Fri, 23 Apr 2021 20:44:05 -0700 Subject: [PATCH 6/9] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fc71f6f..43f2052 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ StreamripDownloads *.aac *.pyc *test.py +/.mypy_cache From c5d139d4bd60abc10eced2ed10d703e320c71dc8 Mon Sep 17 00:00:00 2001 From: nathom Date: Sun, 25 Apr 2021 18:52:44 -0700 Subject: [PATCH 7/9] Fix #62 --- streamrip/tracklists.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/streamrip/tracklists.py b/streamrip/tracklists.py index 913c025..452e38a 100644 --- a/streamrip/tracklists.py +++ b/streamrip/tracklists.py @@ -7,7 +7,7 @@ import logging import os import re from tempfile import gettempdir -from typing import Generator, Iterable, Union +from typing import Dict, Generator, Iterable, Union import click from pathvalidate import sanitize_filename @@ -28,11 +28,6 @@ from .utils import ( logger = logging.getLogger(__name__) -TYPE_REGEXES = { - "remaster": re.compile(r"(?i)(re)?master(ed)?"), - "extra": re.compile(r"(?i)(anniversary|deluxe|live|collector|demo|expanded|remix)"), -} - class Album(Tracklist): """Represents a downloadable album. @@ -649,6 +644,13 @@ class Artist(Tracklist): # ----------- Filters -------------- + TYPE_REGEXES = { + "remaster": re.compile(r"(?i)(re)?master(ed)?"), + "extra": re.compile( + r"(?i)(anniversary|deluxe|live|collector|demo|expanded|remix)" + ), + } + def _remove_repeats(self, bit_depth=max, sampling_rate=max) -> Generator: """Remove the repeated albums from self. May remove different versions of the same album. @@ -656,7 +658,7 @@ class Artist(Tracklist): :param bit_depth: either max or min functions :param sampling_rate: either max or min functions """ - groups = dict() + groups: Dict[str, list] = {} for album in self: if (t := self.essence(album.title)) not in groups: groups[t] = [] @@ -683,7 +685,7 @@ class Artist(Tracklist): """ return ( album["albumartist"] != "Various Artists" - and TYPE_REGEXES["extra"].search(album.title) is None + and self.TYPE_REGEXES["extra"].search(album.title) is None ) def _features(self, album: Album) -> bool: @@ -709,7 +711,7 @@ class Artist(Tracklist): :type album: Album :rtype: bool """ - return TYPE_REGEXES["extra"].search(album.title) is None + return self.TYPE_REGEXES["extra"].search(album.title) is None def _non_remasters(self, album: Album) -> bool: """Passed as a parameter by the user. @@ -721,7 +723,7 @@ class Artist(Tracklist): :type album: Album :rtype: bool """ - return TYPE_REGEXES["remaster"].search(album.title) is not None + return self.TYPE_REGEXES["remaster"].search(album.title) is not None def _non_albums(self, album: Album) -> bool: """This will ignore non-album releases. @@ -731,8 +733,7 @@ class Artist(Tracklist): :type album: Album :rtype: bool """ - # Doesn't work yet - return album["release_type"] == "album" + return len(album) > 1 # --------- Magic Methods -------- @@ -751,7 +752,7 @@ class Artist(Tracklist): """ return self.name - def __hash__(self) -> int: + def __hash__(self): return hash(self.id) From 5cb287cdbe6c6817e8bd775aa2c914bef0d27aa7 Mon Sep 17 00:00:00 2001 From: nathom Date: Mon, 26 Apr 2021 16:20:14 -0700 Subject: [PATCH 8/9] Bump version --- setup.py | 2 +- streamrip/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f6e253b..d6688d9 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ requirements = read_file("requirements.txt").strip().split() # https://github.com/pypa/sampleproject/blob/main/setup.py setup( name=pkg_name, - version="0.4.3", + version="0.5", author="Nathan", author_email="nathanthomas707@gmail.com", keywords="lossless, hi-res, qobuz, tidal, deezer, audio, convert, soundcloud, mp3", diff --git a/streamrip/__init__.py b/streamrip/__init__.py index bcb32fa..4b49527 100644 --- a/streamrip/__init__.py +++ b/streamrip/__init__.py @@ -1,4 +1,4 @@ """streamrip: the all in one music downloader. """ -__version__ = "0.4.3" +__version__ = "0.5" From dad58d8d22704760c8c89c0b00641e584de340bd Mon Sep 17 00:00:00 2001 From: nathom Date: Mon, 26 Apr 2021 16:30:38 -0700 Subject: [PATCH 9/9] Add help for youtube config section --- streamrip/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/streamrip/config.py b/streamrip/config.py index b8ecb2b..86fe08e 100644 --- a/streamrip/config.py +++ b/streamrip/config.py @@ -231,6 +231,10 @@ class ConfigDocumentation: quality: 0, 1, or 2 soundcloud: quality: Only 0 is available + youtube: + quality: Only 0 is available for now + download_videos: Download the video along with the audio + video_downloads_folder: The path to download the videos to database: This stores a list of item IDs so that repeats are not downloaded. filters: Filter a Qobuz artist's discography. Set to 'true' to turn on a filter. extras: Remove Collectors Editions, live recordings, etc.