diff --git a/.gitignore b/.gitignore index 39b1e22..a6b3982 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ tidal-scraper/__pycache__ + +dist +tidal_scraper/__pycache__ \ No newline at end of file diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 1e75b99..0000000 --- a/TODO.md +++ /dev/null @@ -1,7 +0,0 @@ -- [ ] installer or pip package -- [ ] installer should create state and config homes if not existing -- [ ] proper SIGTERM handling -- [ ] decrypt and write in chunks -- [ ] test error logger - -- [ ] Switch to tomllib once ubuntu updates their python package to 3.11 (I haven't switched yet to avoid github issues being made) diff --git a/conf.toml b/conf.toml index 5815b6a..a27ab37 100644 --- a/conf.toml +++ b/conf.toml @@ -8,6 +8,13 @@ user_id = dest_dir = "./downloads/" +# Leave empty for default state location +# Default location is the first valid path in this order: +# XDG_STATE_HOME/tidal-scraper +# XDG_CACHE_HOME/tidal-scraper +# $HOME/.cache/tidal-scraper +state_dir = + # These templates are passed their respective tidalapi objects # Possible attributes can be found here: https://tidalapi.netlify.app/api.html album_dir = "{album.artist.name}/{album.name}/" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e9d914c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +[project] +name = 'tidal-scraper' +version = '1.0' +description = 'A library to download music from tidal' +authors = [ + { name = "Luca Bilke", email="luca@snaile.de" } +] +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: POSIX", + "Topic :: Multimedia :: Sound/Audio", + "Natural Language :: English", +] +dependencies = [ + "pycrypto", + "tqdm", + "mutagen", + "tidalapi @ git+https://github.com/tamland/python-tidal@5207a3cff2af437a2d0d67b743c875a67f8d1d08", +] + +[tool.hatch.metadata] +allow-direct-references = true + +[project.scripts] +tidal-scraper = "tidal_scraper.cli:run" diff --git a/tidal-scraper/helper.py b/tidal-scraper/helper.py deleted file mode 100644 index eac4ba9..0000000 --- a/tidal-scraper/helper.py +++ /dev/null @@ -1,42 +0,0 @@ -import re -import os -import toml -# TODO: wait for python to update to 3.11 for ubuntu users -# import tomllib -import sys -import traceback - -extensions = [".flac", ".mp4", ".m4a", ""] - -home = os.getenv("HOME") -state_dir = os.getenv("XDG_STATE_HOME") or os.getenv("XDG_CACHE_HOME") -conf_dir = os.getenv("XDG_CONFIG_HOME") -if not state_dir: - assert home - state_dir = home + "/.cache" - -if not conf_dir: - assert home - conf_dir = home + "/.config" -conf_dir += "/tidal-scraper" -state_dir += "/tidal-scraper" - -with open(conf_dir + "/conf.toml", "r") as f: - conf = toml.load(f) -# with open(conf_dir + "/conf.toml", "rb") as f: - # conf = tomllib.load(f) - - -def clean_template(path: str, **kwargs) -> str: - path = os.path.expanduser(path) - split = path.split("/") - cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split] - return "/".join(cleaned_split) - - -def log_error(template: str, **kwargs): - with open(conf["error_log"], "a") as f: - msg = template.format(**kwargs) - f.write(msg + "\n") - traceback.format_exception(*sys.exc_info()) - f.write("\n\n") diff --git a/tidal-scraper/run.py b/tidal-scraper/run.py deleted file mode 100755 index cb0fffc..0000000 --- a/tidal-scraper/run.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/env python3 -from download import download_album -from state import State -from helper import conf - -s = State(conf['user_id'], conf['quality']) -s.login() -albums = s.favorites.albums() - -try: - s.load_dl_state -except: - pass - -download_album(albums[0]) -s.set_dl_state(albums[0], True) -s.write_dl_state() diff --git a/tidal_scraper/cli.py b/tidal_scraper/cli.py new file mode 100644 index 0000000..dde954a --- /dev/null +++ b/tidal_scraper/cli.py @@ -0,0 +1,117 @@ +from tidalapi import Album, Track, Artist, Playlist +from tidal_scraper.download import ( + download_album, + download_playlist, + download_track, + download_artist, +) +from tidal_scraper.state import State +from tidal_scraper.helper import get_conf, human_sleep +from argparse import ArgumentParser, Namespace +from pathlib import Path + + +def handle_favorites(state: State, conf: dict, args: Namespace) -> None: + match args.obj: + case "album": + for album in state.favorites.albums(): + download_album(album, conf) + human_sleep() + case "track": + for track in state.favorites.tracks(): + download_track(track, conf) + human_sleep() + case "artist": + for artist in state.favorites.artists(): + download_artist(artist, conf) + human_sleep() + case "playlist": + for playlist in state.favorites.playlists(): + download_playlist(playlist, conf) + human_sleep() + + +def handle_id(state: State, conf: dict, args: Namespace) -> None: + match args.obj: + case "album": + album = Album(state.session, args.id) + download_album(album, conf) + case "track": + track = Track(state.session, args.id) + download_track(track, conf) + case "artist": + artist = Artist(state.session, args.id) + download_artist(artist, conf) + case "playlist": + playlist = Playlist(state.session, args.id) + download_playlist(playlist, conf) + + +def run(): + parser = ArgumentParser(prog="tidal-scraper", description="Tidal music downloader") + parser.add_argument( + "-c", "--config", help="Configuration file to use", type=Path, dest="conf_file" + ) + parser.add_argument( + "-s", "--state", help="Directory to keep state in", type=Path, dest="state_dir" + ) + obj = parser.add_mutually_exclusive_group(required=True) + obj.add_argument( + "-a", + "--album", + action="store_const", + const="album", + help="Download an album", + dest="obj", + ) + obj.add_argument( + "-A", + "--artist", + action="store_const", + const="artist", + help="Download an artist's albums", + dest="obj", + ) + obj.add_argument( + "-p", + "--playlist", + action="store_const", + const="playlist", + help="Download a playlist", + dest="obj", + ) + obj.add_argument( + "-t", + "--track", + action="store_const", + const="track", + help="Download a single track", + dest="obj", + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "-f", + "--favorite", + action="store_true", + help="Download all favorited", + dest="favorite", + ) + source.add_argument( + "-i", + "--id", + type=int, + help="Download by ID", + dest="id", + ) + args = parser.parse_args() + + conf = get_conf(args.state_dir, args.conf_file) + state = State(conf["user_id"], conf["quality"], conf["state_dir"]) + state.login(conf["state_dir"] + "auth.json") + + if args.favorite: + handle_favorites(state, conf, args) + elif args.id is not None: + handle_id(state, conf, args) + + state.write_dl_state(conf["state_dir"] + "state.json") diff --git a/tidal-scraper/download.py b/tidal_scraper/download.py similarity index 64% rename from tidal-scraper/download.py rename to tidal_scraper/download.py index 51eedb8..0c65bbf 100644 --- a/tidal-scraper/download.py +++ b/tidal_scraper/download.py @@ -1,5 +1,5 @@ -import metadata -from helper import conf, extensions, clean_template, log_error +import tidal_scraper.metadata as metadata +from tidal_scraper.helper import extensions, clean_template, log_error, human_sleep import tidalapi import os @@ -44,8 +44,6 @@ def __decrypt_file(fp: BinaryIO, key: bytes, nonce: bytes) -> None: def __download_file(url: str, fp: BinaryIO) -> str: with requests.get(url, stream=True) as r: - if conf["debug"]: - print(r.headers) r.raise_for_status() mime = r.headers.get("Content-Type", "") total_bytes = int(r.headers.get("Content-Length", 0)) @@ -56,22 +54,35 @@ def __download_file(url: str, fp: BinaryIO) -> str: return mime -def download_track(track: tidalapi.Track, dest: str) -> None: +def download_track( + track: tidalapi.Track, + conf: dict | None = None, + name_template: str | None = None, + dest_dir: str | None = None, + skip_dl: bool | None = None, + errorfile: str | None = None, +) -> None: album = track.album assert album + if conf is None: + assert skip_dl is not None + assert errorfile is not None + assert name_template is not None + assert dest_dir is not None + else: + skip_dl = skip_dl or conf["skip_downloaded"] + errorfile = errorfile or conf["error_log"] + dest_dir = dest_dir or conf["dest_dir"] + name_template = name_template or conf["track_name"] + + dest = dest_dir + clean_template(name_template, track=track) print(f"Starting {album.artist.name} - {track.name}") - dest += clean_template( - conf["track_name"], - track=track, - ) http_failures = 0 while http_failures <= 3: try: print("running") stream = track.stream() manifest = json.loads(b64decode(stream.manifest)) - if conf["debug"]: - print(manifest) url = manifest["urls"][0] codec = manifest["codecs"] if ".mp4" in url: @@ -84,7 +95,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None: if ext in url: dest += ext break - if os.path.exists(dest) and conf["skip_downloaded"]: + if os.path.exists(dest) and skip_dl: print(f"Skipping track") return @@ -115,6 +126,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None: raise e except Exception as e: log_error( + errorfile or "error.log", "Failure while downloading {artist} - {track}", artist=album.artist.name, track=track.name, @@ -123,9 +135,35 @@ def download_track(track: tidalapi.Track, dest: str) -> None: def download_cover( - obj: tidalapi.Album | tidalapi.Playlist, dest: str, size: int + obj: tidalapi.Album | tidalapi.Playlist, + conf: dict | None = None, + dest: str | None = None, + size: int | None = None, + skip_dl: bool | None = None, ) -> None: - if os.path.exists(dest) and conf["skip_downloaded"]: + if conf is None: + assert dest is not None + assert size is not None + assert skip_dl is not None + else: + if type(obj) is tidalapi.Album: + dest = clean_template( + conf["dest_dir"] + "/" + conf["album_dir"], + album=obj, + ) + size = conf["album_image_size"] + elif type(obj) is tidalapi.Playlist: + dest = clean_template( + conf["dest_dir"] + "/" + conf["playlist_dir"], + playlist=obj, + ) + size = conf["playlist_image_size"] + + skip_dl = conf["skip_downloaded"] + assert dest + assert size + + if os.path.exists(dest) and skip_dl: return url = obj.image(size) @@ -133,31 +171,34 @@ def download_cover( __download_file(url, f) -def download_album(album: tidalapi.Album) -> None: +def download_album(album: tidalapi.Album, conf: dict) -> None: dest = clean_template( conf["dest_dir"] + "/" + conf["album_dir"], album=album, ) os.makedirs(os.path.dirname(dest), exist_ok=True) - download_cover(album, dest, conf["album_image_size"]) + download_cover(album, conf) tracks = album.tracks() for track in tracks: - download_track(track, dest) + download_track(track, conf, dest_dir=dest) + human_sleep() -def download_playlist(playlist: tidalapi.Playlist) -> None: +def download_playlist(playlist: tidalapi.Playlist, conf: dict) -> None: dest = clean_template( conf["dest_dir"] + "/" + conf["playlist_dir"], playlist=playlist, ) os.makedirs(os.path.dirname(dest), exist_ok=True) - download_cover(playlist, dest, conf["playlist_image_size"]) + download_cover(playlist, conf) tracks = playlist.tracks() for track in tracks: - download_track(track, dest) + download_track(track, conf, dest_dir=dest) + human_sleep() -def download_artist(artist: tidalapi.Artist) -> None: +def download_artist(artist: tidalapi.Artist, conf: dict) -> None: albums = artist.get_albums() for album in albums: - download_album(album) + download_album(album, conf) + human_sleep() diff --git a/tidal_scraper/helper.py b/tidal_scraper/helper.py new file mode 100644 index 0000000..372be59 --- /dev/null +++ b/tidal_scraper/helper.py @@ -0,0 +1,50 @@ +import re +import os +import time +import random + +import tomllib +import sys +import traceback + +extensions = [".flac", ".mp4", ".m4a", ""] + + +def get_conf(state_dir: str | None = None, conf_file: str | None = None) -> dict: + home = os.getenv("HOME") + assert home + + conf_file = (os.getenv("XDG_CONFIG_HOME") or home + "/.config") + "/tidal-scraper/conf.toml" + with open(conf_file, "rb") as f: + conf = tomllib.load(f) + + if not state_dir: + state_dir = ( + os.getenv("XDG_STATE_HOME") + or os.getenv("XDG_CACHE_HOME") + or home + "/.cache" + ) + state_dir += "/tidal-scraper" + conf["state_dir"] = state_dir + + return conf + + +def clean_template(path: str, **kwargs) -> str: + path = os.path.expanduser(path) + split = path.split("/") + cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split] + return "/".join(cleaned_split) + + +def log_error(logfile: str, template: str, **kwargs): + with open(logfile, "a") as f: + msg = template.format(**kwargs) + f.write(msg + "\n") + traceback.format_exception(*sys.exc_info()) + f.write("\n\n") + + +def human_sleep() -> None: + t = random.randrange(10, 50) / 10 + time.sleep(t) diff --git a/tidal-scraper/metadata.py b/tidal_scraper/metadata.py similarity index 56% rename from tidal-scraper/metadata.py rename to tidal_scraper/metadata.py index bc18a80..52ce7be 100644 --- a/tidal-scraper/metadata.py +++ b/tidal_scraper/metadata.py @@ -1,36 +1,17 @@ -from mutagen import flac, mp4 # , mp3 +from mutagen import flac, mp4 from mutagen.mp4 import MP4Tags from mutagen._vorbis import VCommentDict from typing import BinaryIO -# from mutagen.id3._tags import ID3Tags - -# from mutagen.id3._frames import ( -# APIC, -# TALB, -# TCOP, -# TDRC, -# TIT2, -# TPE1, -# TRCK, -# TOPE, -# TCON, -# TCOM, -# TSRC, -# USLT, -# ) - def __write_flac(file: flac.FLAC, **kwargs) -> None: tags = VCommentDict() tags["title"] = kwargs["title"] tags["album"] = kwargs["album"] - # There doesn't seem to be a standard way of listing multiple artists in an ID3 tag - # This method seems to be the most widely recognized - tags["albumartist"] = "; ".join(kwargs["albumartist"]) - tags["artist"] = "; ".join(kwargs["artist"]) + tags["albumartist"] = kwargs["albumartist"] + tags["artist"] = kwargs["artist"] tags["copyright"] = kwargs["copyright"] tags["tracknumber"] = kwargs["tracknumber"] tags["tracktotal"] = kwargs["tracktotal"] @@ -70,37 +51,6 @@ def __write_mp4(file: mp4.MP4, **kwargs) -> None: file.save -# def __write_mp3(file: mp3.MP3, **kwargs) -> None: -# tags = ID3Tags() -# tags.add(TIT2(encoding=3, text=kwargs["title"])) -# tags.add(TALB(encoding=3, text=kwargs["album"])) -# tags.add(TOPE(encoding=3, text=kwargs["albumartist"])) -# tags.add(TPE1(encoding=3, text=kwargs["artist"])) -# tags.add(TCOP(encoding=3, text=kwargs["copyright"])) -# tags.add(TRCK(encoding=3, text=kwargs["tracknumber"])) -# tags.add(TCON(encoding=3, text=kwargs["genre"])) -# tags.add(TDRC(encoding=3, text=kwargs["date"])) -# tags.add(TCOM(encoding=3, text=kwargs["composer"])) -# tags.add(TSRC(encoding=3, text=kwargs["isrc"])) -# tags.add(USLT(encoding=3, text=kwargs["lyrics"])) -# tags.add(APIC(encoding=3, data=kwargs["cover"], mime=kwargs["cover_mime"])) -# -# match kwargs['cover_mime']: -# case 'image/jpeg': -# fmt = mp4.AtomDataType(13) -# case 'image/png': -# fmt = mp4.AtomDataType(14) -# case _: -# fmt = None -# -# if fmt is not None: -# pic = mp4.MP4Cover(kwargs['cover']) -# pic.imageformat = fmt -# -# file.tags = tags -# file.save() - - def write( fp: BinaryIO, mime: str, diff --git a/tidal-scraper/state.py b/tidal_scraper/state.py similarity index 78% rename from tidal-scraper/state.py rename to tidal_scraper/state.py index ff33dfc..a728330 100644 --- a/tidal-scraper/state.py +++ b/tidal_scraper/state.py @@ -1,13 +1,10 @@ import json from datetime import datetime from tidalapi import session, user, playlist, media, album, artist -from helper import conf, state_dir class State: - def __init__( - self, user_id: int, quality: str, dl_state_path: str = state_dir + "/state.json" - ): + def __init__(self, user_id: int, quality: str, dl_state_path: str): match quality: case "master": q = session.Quality.master @@ -18,11 +15,11 @@ class State: case "low": q = session.Quality.low case _: - raise Exception("Quality misconfigured in conf.toml") + raise Exception("Bad Quality String") config = session.Config(quality=q) self.user_id = user_id self.session = session.Session(config) - self.favorites = user.Favorites(self.session, conf["user_id"]) + self.favorites = user.Favorites(self.session, user_id) try: self.load_dl_state(dl_state_path) except: @@ -33,7 +30,7 @@ class State: "tracks": {}, } - def login(self, auth_file: str | None = state_dir + "/auth.json") -> None: + def login(self, auth_file: str | None = None) -> None: s = self.session try: assert auth_file @@ -84,16 +81,12 @@ class State: self._state[t][obj.id] = downloaded - def write_dl_state(self, dl_state_path: str | None = None) -> None: - if dl_state_path is None: - dl_state_path = state_dir + "/state.json" - with open(dl_state_path, "w") as f: + def write_dl_state(self, statefile: str) -> None: + with open(statefile, "w") as f: json.dump(self._state, f) - def load_dl_state(self, dl_state_path: str | None = None) -> None: - if dl_state_path is None: - dl_state_path = state_dir + "/state.json" - with open(dl_state_path, "r") as f: + def load_dl_state(self, statefile: str) -> None: + with open(statefile, "r") as f: self._state = json.load(f) assert type(self._state["albums"]) is dict[int, bool]