diff --git a/src/client.py b/src/client.py index 33195bb..896ac69 100644 --- a/src/client.py +++ b/src/client.py @@ -2,11 +2,19 @@ import logging from abc import ABC, abstractmethod +from typing import Optional, Union + +import aiohttp +import aiolimiter from .downloadable import Downloadable logger = logging.getLogger("streamrip") +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0" +) + class Client(ABC): source: str @@ -17,7 +25,7 @@ class Client(ABC): raise NotImplemented @abstractmethod - async def get_metadata(self, item_id, media_type): + async def get_metadata(self, item: dict[str, Union[str, int, float]], media_type): raise NotImplemented @abstractmethod @@ -27,3 +35,25 @@ class Client(ABC): @abstractmethod async def get_downloadable(self, item_id: str, quality: int) -> Downloadable: raise NotImplemented + + @staticmethod + def get_rate_limiter( + requests_per_min: int, + ) -> Optional[aiolimiter.AsyncLimiter]: + return ( + aiolimiter.AsyncLimiter(requests_per_min, 60) + if requests_per_min > 0 + else None + ) + + @staticmethod + def get_session(headers: Optional[dict] = None) -> aiohttp.ClientSession: + if headers is None: + headers = {} + return aiohttp.ClientSession( + headers={"User-Agent": DEFAULT_USER_AGENT}, **headers + ) + + +class NonStreamable(Exception): + pass diff --git a/src/config.py b/src/config.py index f0690e2..64867b6 100644 --- a/src/config.py +++ b/src/config.py @@ -3,9 +3,7 @@ import copy import logging import os -from collections import defaultdict from dataclasses import dataclass -from typing import Any from tomlkit.api import dumps, parse from tomlkit.toml_document import TOMLDocument @@ -272,6 +270,7 @@ class ConfigData: def set_modified(self): self._modified = True + @property def modified(self): return self._modified @@ -289,7 +288,7 @@ class Config: self.session: ConfigData = copy.deepcopy(self.file) def save_file(self): - if not self.file.modified(): + if not self.file.modified: return with open(self._path, "w") as toml_file: diff --git a/src/core.py b/src/core.py index 2c28c01..b4a25a6 100644 --- a/src/core.py +++ b/src/core.py @@ -1,11 +1,12 @@ """The stuff that ties everything together for the CLI to use.""" +import asyncio import concurrent.futures import html import logging import os import re -import threading +from abc import ABC, abstractmethod from getpass import getpass from hashlib import md5 from string import Formatter @@ -84,6 +85,101 @@ DB_PATH_MAP = {"downloads": DB_PATH, "failed_downloads": FAILED_DB_PATH} # ---------------------------------------------- # +class CredentialPrompter(ABC): + def __init__(self, config: Config): + self.config = config + + @abstractmethod + def has_creds(self) -> bool: + raise NotImplemented + + @abstractmethod + def prompt(self): + """Prompt for credentials in the appropriate way, + and save them to the configuration.""" + raise NotImplemented + + @abstractmethod + def save(self): + """Save current config to file""" + raise NotImplemented + + +class QobuzPrompter(CredentialPrompter): + def has_creds(self) -> bool: + c = self.config.session.qobuz + return c.email_or_userid != "" and c.password_or_token != "" + + def prompt(self): + secho("Enter Qobuz email:", fg="green") + email = input() + secho( + "Enter Qobuz password (will not show on screen):", + fg="green", + ) + pwd = md5(getpass(prompt="").encode("utf-8")).hexdigest() + secho( + f'Credentials saved to config file at "{self.config._path}"', + fg="green", + ) + c = self.config.session.qobuz + c.use_auth_token = False + c.email_or_userid = email + c.password_or_token = pwd + + def save(self): + c = self.config.session.qobuz + cf = self.config.file.qobuz + cf.use_auth_token = False + cf.email_or_userid = c.email_or_userid + cf.password_or_token = c.password_or_token + self.config.file.set_modified() + + +class TidalPrompter(CredentialPrompter): + def prompt(self): + # TODO: needs to be moved from TidalClient to here + raise NotImplemented + + +class DeezerPrompter(CredentialPrompter): + def has_creds(self): + c = self.config.session.deezer + return c.arl != "" + + def prompt(self): + secho( + "If you're not sure how to find the ARL cookie, see the instructions at ", + nl=False, + dim=True, + ) + secho( + "https://github.com/nathom/streamrip/wiki/Finding-your-Deezer-ARL-Cookie", + underline=True, + fg="blue", + ) + + c = self.config.session.deezer + c.arl = input(style("ARL: ", fg="green")) + + def save(self): + c = self.config.session.deezer + cf = self.config.file.deezer + cf.arl = c.arl + self.config.file.set_modified() + secho( + f'Credentials saved to config file at "{self.config._path}"', + fg="green", + ) + + +PROMPTERS = { + "qobuz": QobuzPrompter, + "deezer": DeezerPrompter, + "tidal": TidalPrompter, +} + + class RipCore(list): def __init__(self, config: Config): """Create a RipCore object. @@ -166,7 +262,7 @@ class RipCore(list): :param item_id: :type item_id: str """ - client = self.get_client(source) + client = self.get_client_and_log_in(source) if media_type not in MEDIA_TYPES: if "playlist" in media_type: # for SoundCloud @@ -320,7 +416,7 @@ class RipCore(list): """ self.extend(self.search("qobuz", featured_list, "featured", limit=max_items)) - def get_client(self, source: str) -> Client: + def get_client_and_log_in(self, source: str) -> Client: """Get a client given the source and log in. :param source: @@ -336,14 +432,14 @@ class RipCore(list): return client - def login(self, client): + async def login(self, client): """Log into a client, if applicable. :param client: """ - creds = self.config.creds(client.source) - if client.source == "deezer" and creds["arl"] == "": - if self.config.session["deezer"]["deezloader_warnings"]: + c = self.config.session + if client.source == "deezer" and c.deezer.arl == "": + if c.deezer.deezloader_warnings: secho( "Falling back to Deezloader (unstable). If you have a subscription, run ", nl=False, @@ -355,23 +451,18 @@ class RipCore(list): while True: try: - client.login(**creds) + await client.login() break except AuthenticationError: secho("Invalid credentials, try again.", fg="yellow") - self.prompt_creds(client.source) - creds = self.config.creds(client.source) + self.prompt_and_set_credentials(client.source) except MissingCredentials: - logger.debug("Credentials are missing. Prompting..") - get_tokens = threading.Thread( - target=client._get_app_id_and_secrets, daemon=True - ) - get_tokens.start() - - self.prompt_creds(client.source) - creds = self.config.creds(client.source) - - get_tokens.join() + if client.source == "qobuz": + get_tokens = asyncio.create_task(client._get_app_id_and_secrets()) + self.prompt_and_set_credentials(client.source) + await get_tokens + else: + self.prompt_and_set_credentials(client.source) if ( client.source == "qobuz" @@ -442,7 +533,7 @@ class RipCore(list): soundcloud_urls = SOUNDCLOUD_URL_REGEX.findall(url) if soundcloud_urls: - soundcloud_client = self.get_client("soundcloud") + soundcloud_client = self.get_client_and_log_in("soundcloud") assert isinstance(soundcloud_client, SoundcloudClient) # for typing # TODO: Make this async @@ -550,7 +641,7 @@ class RipCore(list): secho(f"Fetching playlist at {purl}", fg="blue") title, queries = self.get_lastfm_playlist(purl) - pl = Playlist(client=self.get_client(lastfm_source), name=title) + pl = Playlist(client=self.get_client_and_log_in(lastfm_source), name=title) creator_match = user_regex.search(purl) if creator_match is not None: pl.creator = creator_match.group(1) @@ -614,7 +705,7 @@ class RipCore(list): """ logger.debug("searching for %s", query) - client = self.get_client(source) + client = self.get_client_and_log_in(source) if isinstance(client, DeezloaderClient) and media_type == "featured": raise IneligibleError( @@ -845,12 +936,25 @@ class RipCore(list): path = self.config.session["downloads"]["folder"] return os.path.join(path, source.capitalize()) - def prompt_creds(self, source: str): + async def prompt_and_set_credentials(self, source: str): """Prompt the user for credentials. :param source: :type source: str """ + prompter = PROMPTERS[source] + client = self.clients[source] + while True: + prompter.prompt() + try: + await client.login() + break + except AuthenticationError: + secho("Invalid credentials, try again.", fg="yellow") + except MissingCredentials: + secho("Credentials not found, try again.", fg="yellow") + self.prompt_and_set_credentials(client.source) + if source == "qobuz": secho("Enter Qobuz email:", fg="green") self.config.file[source]["email"] = input() diff --git a/src/downloadable.py b/src/downloadable.py index c098db1..ed88675 100644 --- a/src/downloadable.py +++ b/src/downloadable.py @@ -41,7 +41,7 @@ class Downloadable(ABC): class BasicDownloadable(Downloadable): """Just downloads a URL.""" - def __init__(self, session, url: str): + def __init__(self, session: aiohttp.ClientSession, url: str): self.session = session self.url = url @@ -59,7 +59,7 @@ class DeezerDownloadable(Downloadable): def __init__(self, resp: dict): self.resp = resp - async def _download(self, path: str) -> bool: + async def _download(self, path: str): raise NotImplemented @@ -67,7 +67,7 @@ class TidalDownloadable(Downloadable): def __init__(self, info: dict): self.info = info - async def _download(self, path: str) -> bool: + async def _download(self, path: str): raise NotImplemented @@ -75,5 +75,5 @@ class SoundcloudDownloadable(Downloadable): def __init__(self, info: dict): self.info = info - async def _download(self, path: str) -> bool: + async def _download(self, path: str): raise NotImplemented diff --git a/src/qobuz_client.py b/src/qobuz_client.py index cfdabc3..9ad27e2 100644 --- a/src/qobuz_client.py +++ b/src/qobuz_client.py @@ -8,7 +8,7 @@ from typing import AsyncGenerator, Optional import aiohttp from aiolimiter import AsyncLimiter -from .client import Client +from .client import DEFAULT_USER_AGENT, Client from .config import Config from .downloadable import BasicDownloadable, Downloadable from .exceptions import ( @@ -23,9 +23,6 @@ from .qobuz_spoofer import QobuzSpoofer logger = logging.getLogger("streamrip") -DEFAULT_USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0" -) QOBUZ_BASE_URL = "https://www.qobuz.com/api.json/0.2" QOBUZ_FEATURED_KEYS = { @@ -54,9 +51,10 @@ class QobuzClient(Client): def __init__(self, config: Config): self.logged_in = False self.config = config - self.session = aiohttp.ClientSession(headers={"User-Agent": DEFAULT_USER_AGENT}) - rate_limit = config.session.downloads.requests_per_minute - self.rate_limiter = AsyncLimiter(rate_limit, 60) if rate_limit > 0 else None + self.session = self.get_session() + self.rate_limiter = self.get_rate_limiter( + config.session.downloads.requests_per_minute + ) self.secret: Optional[str] = None async def login(self): @@ -185,7 +183,7 @@ class QobuzClient(Client): ) raise NonStreamable - return BasicDownloadable(stream_url) + return BasicDownloadable(self.session, stream_url) async def _paginate(self, epoint: str, params: dict) -> AsyncGenerator[dict, None]: response = await self._api_request(epoint, params) diff --git a/src/soundcloud_client.py b/src/soundcloud_client.py index 2729fae..7486c85 100644 --- a/src/soundcloud_client.py +++ b/src/soundcloud_client.py @@ -1,6 +1,11 @@ -from .client import Client +import re + +from .client import Client, NonStreamable from .config import Config -from .downloadable import Downloadable +from .downloadable import SoundcloudDownloadable + +BASE = "https://api-v2.soundcloud.com" +SOUNDCLOUD_USER_ID = "672320-86895-162383-801513" class SoundcloudClient(Client): @@ -8,16 +13,115 @@ class SoundcloudClient(Client): logged_in = False def __init__(self, config: Config): - self.config = config.soundcloud + self.global_config = config + self.config = config.session.soundcloud + self.session = self.get_session() + self.rate_limiter = self.get_rate_limiter( + config.session.downloads.requests_per_minute + ) async def login(self): client_id, app_version = self.config.client_id, self.config.app_version - pass + if not client_id or not app_version or not self._announce(): + client_id, app_version = await self._refresh_tokens() - async def get_downloadable(self, track: dict, _) -> Downloadable: - pass + # update file and session configs and save to disk + c = self.global_config.file.soundcloud + self.config.client_id = c.client_id = client_id + self.config.client_id = c.app_version = app_version + self.global_config.file.set_modified() + + async def _announce(self): + resp = await self._api_request("announcements") + return resp.status == 200 + + async def _refresh_tokens(self) -> tuple[str, str]: + """Return a valid client_id, app_version pair.""" + STOCK_URL = "https://soundcloud.com/" + async with self.session.get(STOCK_URL) as resp: + page_text = await resp.text(encoding="utf-8") + + *_, client_id_url_match = re.finditer( + r"window\.__sc_version="(\d+)"', page_text + ) + if app_version_match is None: + raise Exception("Could not find app version in %s" % client_id_url_match) + app_version = app_version_match.group(1) + + async with self.session.get(client_id_url) as resp: + page_text2 = await resp.text(encoding="utf-8") + + client_id_match = re.search(r'client_id:\s*"(\w+)"', page_text2) + assert client_id_match is not None + client_id = client_id_match.group(1) + + return client_id, app_version + + async def get_downloadable(self, item: dict, _) -> SoundcloudDownloadable: + if not item["streamable"] or item["policy"] == "BLOCK": + raise NonStreamable(item) + + if item["downloadable"] and item["has_downloads_left"]: + resp = await self._api_request(f"tracks/{item['id']}/download") + resp_json = await resp.json() + return SoundcloudDownloadable( + {"url": resp_json["redirectUri"], "type": "original"} + ) + + else: + url = None + for tc in item["media"]["transcodings"]: + fmt = tc["format"] + if fmt["protocol"] == "hls" and fmt["mime_type"] == "audio/mpeg": + url = tc["url"] + break + + assert url is not None + + resp = await self._request(url) + resp_json = await resp.json() + return SoundcloudDownloadable({"url": resp_json["url"], "type": "mp3"}) async def search( self, query: str, media_type: str, limit: int = 50, offset: int = 0 ): - pass + params = { + "q": query, + "facet": "genre", + "user_id": SOUNDCLOUD_USER_ID, + "limit": limit, + "offset": offset, + "linked_partitioning": "1", + } + resp = await self._api_request(f"search/{media_type}s", params=params) + return await resp.json() + + async def _api_request(self, path, params=None, headers=None): + url = f"{BASE}/{path}" + return await self._request(url, params=params, headers=headers) + + async def _request(self, url, params=None, headers=None): + c = self.config + _params = { + "client_id": c.client_id, + "app_version": c.app_version, + "app_locale": "en", + } + if params is not None: + _params.update(params) + + async with self.session.get(url, params=_params, headers=headers) as resp: + return resp + + async def _resolve_url(self, url: str) -> dict: + resp = await self._api_request(f"resolve?url={url}") + return await resp.json()