fix bugs, add cli tool, create package

.gitignore

@@ -1 +1,4 @@
 tidal-scraper/__pycache__
+
+dist
+tidal_scraper/__pycache__

TODO.md

@@ -1,7 +0,0 @@
-- [ ] installer or pip package
-- [ ] installer should create state and config homes if not existing
-- [ ] proper SIGTERM handling
-- [ ] decrypt and write in chunks
-- [ ] test error logger
-
-- [ ] Switch to tomllib once ubuntu updates their python package to 3.11 (I haven't switched yet to avoid github issues being made)

conf.toml

@@ -8,6 +8,13 @@ user_id =
 
 dest_dir = "./downloads/"
 
+# Leave empty for default state location 
+# Default location is the first valid path in this order:
+# XDG_STATE_HOME/tidal-scraper
+# XDG_CACHE_HOME/tidal-scraper
+# $HOME/.cache/tidal-scraper
+state_dir =
+
 # These templates are passed their respective tidalapi objects
 # Possible attributes can be found here: https://tidalapi.netlify.app/api.html
 album_dir = "{album.artist.name}/{album.name}/"

pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+[project]
+name = 'tidal-scraper'
+version = '1.0'
+description = 'A library to download music from tidal'
+authors = [
+    { name = "Luca Bilke", email="luca@snaile.de" }
+]
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3.11",
+    "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+    "Operating System :: POSIX",
+    "Topic :: Multimedia :: Sound/Audio",
+    "Natural Language :: English",
+]
+dependencies = [
+    "pycrypto",
+    "tqdm",
+    "mutagen",
+    "tidalapi @ git+https://github.com/tamland/python-tidal@5207a3cff2af437a2d0d67b743c875a67f8d1d08",
+]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[project.scripts]
+tidal-scraper = "tidal_scraper.cli:run"

tidal-scraper/helper.py

@@ -1,42 +0,0 @@
-import re
-import os
-import toml
-# TODO: wait for python to update to 3.11 for ubuntu users
-# import tomllib
-import sys
-import traceback
-
-extensions = [".flac", ".mp4", ".m4a", ""]
-
-home = os.getenv("HOME")
-state_dir = os.getenv("XDG_STATE_HOME") or os.getenv("XDG_CACHE_HOME")
-conf_dir = os.getenv("XDG_CONFIG_HOME")
-if not state_dir:
-    assert home
-    state_dir = home + "/.cache"
-
-if not conf_dir:
-    assert home
-    conf_dir = home + "/.config"
-conf_dir += "/tidal-scraper"
-state_dir += "/tidal-scraper"
-
-with open(conf_dir + "/conf.toml", "r") as f:
-    conf = toml.load(f)
-# with open(conf_dir + "/conf.toml", "rb") as f:
-    # conf = tomllib.load(f)
-
-
-def clean_template(path: str, **kwargs) -> str:
-    path = os.path.expanduser(path)
-    split = path.split("/")
-    cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split]
-    return "/".join(cleaned_split)
-
-
-def log_error(template: str, **kwargs):
-    with open(conf["error_log"], "a") as f:
-        msg = template.format(**kwargs)
-        f.write(msg + "\n")
-        traceback.format_exception(*sys.exc_info())
-        f.write("\n\n")

tidal-scraper/run.py

@@ -1,17 +0,0 @@
-#!/bin/env python3
-from download import download_album
-from state import State
-from helper import conf
-
-s = State(conf['user_id'], conf['quality'])
-s.login()
-albums = s.favorites.albums()
-
-try:
-    s.load_dl_state
-except:
-    pass
-
-download_album(albums[0])
-s.set_dl_state(albums[0], True)
-s.write_dl_state()

tidal_scraper/cli.py

@@ -0,0 +1,117 @@
+from tidalapi import Album, Track, Artist, Playlist
+from tidal_scraper.download import (
+    download_album,
+    download_playlist,
+    download_track,
+    download_artist,
+)
+from tidal_scraper.state import State
+from tidal_scraper.helper import get_conf, human_sleep
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+
+
+def handle_favorites(state: State, conf: dict, args: Namespace) -> None:
+    match args.obj:
+        case "album":
+            for album in state.favorites.albums():
+                download_album(album, conf)
+                human_sleep()
+        case "track":
+            for track in state.favorites.tracks():
+                download_track(track, conf)
+                human_sleep()
+        case "artist":
+            for artist in state.favorites.artists():
+                download_artist(artist, conf)
+                human_sleep()
+        case "playlist":
+            for playlist in state.favorites.playlists():
+                download_playlist(playlist, conf)
+                human_sleep()
+
+
+def handle_id(state: State, conf: dict, args: Namespace) -> None:
+    match args.obj:
+        case "album":
+            album = Album(state.session, args.id)
+            download_album(album, conf)
+        case "track":
+            track = Track(state.session, args.id)
+            download_track(track, conf)
+        case "artist":
+            artist = Artist(state.session, args.id)
+            download_artist(artist, conf)
+        case "playlist":
+            playlist = Playlist(state.session, args.id)
+            download_playlist(playlist, conf)
+
+
+def run():
+    parser = ArgumentParser(prog="tidal-scraper", description="Tidal music downloader")
+    parser.add_argument(
+        "-c", "--config", help="Configuration file to use", type=Path, dest="conf_file"
+    )
+    parser.add_argument(
+        "-s", "--state", help="Directory to keep state in", type=Path, dest="state_dir"
+    )
+    obj = parser.add_mutually_exclusive_group(required=True)
+    obj.add_argument(
+        "-a",
+        "--album",
+        action="store_const",
+        const="album",
+        help="Download an album",
+        dest="obj",
+    )
+    obj.add_argument(
+        "-A",
+        "--artist",
+        action="store_const",
+        const="artist",
+        help="Download an artist's albums",
+        dest="obj",
+    )
+    obj.add_argument(
+        "-p",
+        "--playlist",
+        action="store_const",
+        const="playlist",
+        help="Download a playlist",
+        dest="obj",
+    )
+    obj.add_argument(
+        "-t",
+        "--track",
+        action="store_const",
+        const="track",
+        help="Download a single track",
+        dest="obj",
+    )
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument(
+        "-f",
+        "--favorite",
+        action="store_true",
+        help="Download all favorited",
+        dest="favorite",
+    )
+    source.add_argument(
+        "-i",
+        "--id",
+        type=int,
+        help="Download by ID",
+        dest="id",
+    )
+    args = parser.parse_args()
+
+    conf = get_conf(args.state_dir, args.conf_file)
+    state = State(conf["user_id"], conf["quality"], conf["state_dir"])
+    state.login(conf["state_dir"] + "auth.json")
+
+    if args.favorite:
+        handle_favorites(state, conf, args)
+    elif args.id is not None:
+        handle_id(state, conf, args)
+
+    state.write_dl_state(conf["state_dir"] + "state.json")

tidal_scraper/download.py

@@ -1,5 +1,5 @@
-import metadata
+import tidal_scraper.metadata as metadata
-from helper import conf, extensions, clean_template, log_error
+from tidal_scraper.helper import extensions, clean_template, log_error, human_sleep
 
 import tidalapi
 import os
@@ -44,8 +44,6 @@ def __decrypt_file(fp: BinaryIO, key: bytes, nonce: bytes) -> None:
 
 def __download_file(url: str, fp: BinaryIO) -> str:
     with requests.get(url, stream=True) as r:
-        if conf["debug"]:
-            print(r.headers)
         r.raise_for_status()
         mime = r.headers.get("Content-Type", "")
         total_bytes = int(r.headers.get("Content-Length", 0))
@@ -56,22 +54,35 @@ def __download_file(url: str, fp: BinaryIO) -> str:
     return mime
 
 
-def download_track(track: tidalapi.Track, dest: str) -> None:
+def download_track(
+    track: tidalapi.Track,
+    conf: dict | None = None,
+    name_template: str | None = None,
+    dest_dir: str | None = None,
+    skip_dl: bool | None = None,
+    errorfile: str | None = None,
+) -> None:
     album = track.album
     assert album
+    if conf is None:
+        assert skip_dl is not None
+        assert errorfile is not None
+        assert name_template is not None
+        assert dest_dir is not None
+    else:
+        skip_dl = skip_dl or conf["skip_downloaded"]
+        errorfile = errorfile or conf["error_log"]
+        dest_dir = dest_dir or conf["dest_dir"]
+        name_template = name_template or conf["track_name"]
+
+    dest = dest_dir + clean_template(name_template, track=track)
     print(f"Starting {album.artist.name} - {track.name}")
-    dest += clean_template(
-        conf["track_name"],
-        track=track,
-    )
     http_failures = 0
     while http_failures <= 3:
         try:
             print("running")
             stream = track.stream()
             manifest = json.loads(b64decode(stream.manifest))
-            if conf["debug"]:
-                print(manifest)
             url = manifest["urls"][0]
             codec = manifest["codecs"]
             if ".mp4" in url:
@@ -84,7 +95,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
                     if ext in url:
                         dest += ext
                         break
-            if os.path.exists(dest) and conf["skip_downloaded"]:
+            if os.path.exists(dest) and skip_dl:
                 print(f"Skipping track")
                 return
 
@@ -115,6 +126,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
             raise e
         except Exception as e:
             log_error(
+                errorfile or "error.log",
                 "Failure while downloading {artist} - {track}",
                 artist=album.artist.name,
                 track=track.name,
@@ -123,9 +135,35 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
 
 
 def download_cover(
-    obj: tidalapi.Album | tidalapi.Playlist, dest: str, size: int
+    obj: tidalapi.Album | tidalapi.Playlist,
+    conf: dict | None = None,
+    dest: str | None = None,
+    size: int | None = None,
+    skip_dl: bool | None = None,
 ) -> None:
-    if os.path.exists(dest) and conf["skip_downloaded"]:
+    if conf is None:
+        assert dest is not None
+        assert size is not None
+        assert skip_dl is not None
+    else:
+        if type(obj) is tidalapi.Album:
+            dest = clean_template(
+                conf["dest_dir"] + "/" + conf["album_dir"],
+                album=obj,
+            )
+            size = conf["album_image_size"]
+        elif type(obj) is tidalapi.Playlist:
+            dest = clean_template(
+                conf["dest_dir"] + "/" + conf["playlist_dir"],
+                playlist=obj,
+            )
+            size = conf["playlist_image_size"]
+
+        skip_dl = conf["skip_downloaded"]
+        assert dest
+        assert size
+
+    if os.path.exists(dest) and skip_dl:
         return
 
     url = obj.image(size)
@@ -133,31 +171,34 @@ def download_cover(
         __download_file(url, f)
 
 
-def download_album(album: tidalapi.Album) -> None:
+def download_album(album: tidalapi.Album, conf: dict) -> None:
     dest = clean_template(
         conf["dest_dir"] + "/" + conf["album_dir"],
         album=album,
     )
     os.makedirs(os.path.dirname(dest), exist_ok=True)
-    download_cover(album, dest, conf["album_image_size"])
+    download_cover(album, conf)
     tracks = album.tracks()
     for track in tracks:
-        download_track(track, dest)
+        download_track(track, conf, dest_dir=dest)
+        human_sleep()
 
 
-def download_playlist(playlist: tidalapi.Playlist) -> None:
+def download_playlist(playlist: tidalapi.Playlist, conf: dict) -> None:
     dest = clean_template(
         conf["dest_dir"] + "/" + conf["playlist_dir"],
         playlist=playlist,
     )
     os.makedirs(os.path.dirname(dest), exist_ok=True)
-    download_cover(playlist, dest, conf["playlist_image_size"])
+    download_cover(playlist, conf)
     tracks = playlist.tracks()
     for track in tracks:
-        download_track(track, dest)
+        download_track(track, conf, dest_dir=dest)
+        human_sleep()
 
 
-def download_artist(artist: tidalapi.Artist) -> None:
+def download_artist(artist: tidalapi.Artist, conf: dict) -> None:
     albums = artist.get_albums()
     for album in albums:
-        download_album(album)
+        download_album(album, conf)
+        human_sleep()

tidal_scraper/helper.py

@@ -0,0 +1,50 @@
+import re
+import os
+import time
+import random
+
+import tomllib
+import sys
+import traceback
+
+extensions = [".flac", ".mp4", ".m4a", ""]
+
+
+def get_conf(state_dir: str | None = None, conf_file: str | None = None) -> dict:
+    home = os.getenv("HOME")
+    assert home
+
+    conf_file = (os.getenv("XDG_CONFIG_HOME") or home + "/.config") + "/tidal-scraper/conf.toml"
+    with open(conf_file, "rb") as f:
+        conf = tomllib.load(f)
+
+    if not state_dir:
+        state_dir = (
+            os.getenv("XDG_STATE_HOME")
+            or os.getenv("XDG_CACHE_HOME")
+            or home + "/.cache"
+        )
+        state_dir += "/tidal-scraper"
+        conf["state_dir"] = state_dir
+
+    return conf
+
+
+def clean_template(path: str, **kwargs) -> str:
+    path = os.path.expanduser(path)
+    split = path.split("/")
+    cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split]
+    return "/".join(cleaned_split)
+
+
+def log_error(logfile: str, template: str, **kwargs):
+    with open(logfile, "a") as f:
+        msg = template.format(**kwargs)
+        f.write(msg + "\n")
+        traceback.format_exception(*sys.exc_info())
+        f.write("\n\n")
+
+
+def human_sleep() -> None:
+    t = random.randrange(10, 50) / 10
+    time.sleep(t)

tidal_scraper/metadata.py

@@ -1,36 +1,17 @@
-from mutagen import flac, mp4  # , mp3
+from mutagen import flac, mp4
 
 from mutagen.mp4 import MP4Tags
 from mutagen._vorbis import VCommentDict
 
 from typing import BinaryIO
 
-# from mutagen.id3._tags import ID3Tags
-
-# from mutagen.id3._frames import (
-#     APIC,
-#     TALB,
-#     TCOP,
-#     TDRC,
-#     TIT2,
-#     TPE1,
-#     TRCK,
-#     TOPE,
-#     TCON,
-#     TCOM,
-#     TSRC,
-#     USLT,
-# )
-
 
 def __write_flac(file: flac.FLAC, **kwargs) -> None:
     tags = VCommentDict()
     tags["title"] = kwargs["title"]
     tags["album"] = kwargs["album"]
-    # There doesn't seem to be a standard way of listing multiple artists in an ID3 tag
+    tags["albumartist"] = kwargs["albumartist"]
-    # This method seems to be the most widely recognized
+    tags["artist"] = kwargs["artist"]
-    tags["albumartist"] = "; ".join(kwargs["albumartist"])
-    tags["artist"] = "; ".join(kwargs["artist"])
     tags["copyright"] = kwargs["copyright"]
     tags["tracknumber"] = kwargs["tracknumber"]
     tags["tracktotal"] = kwargs["tracktotal"]
@@ -70,37 +51,6 @@ def __write_mp4(file: mp4.MP4, **kwargs) -> None:
     file.save
 
 
-# def __write_mp3(file: mp3.MP3, **kwargs) -> None:
-#     tags = ID3Tags()
-#     tags.add(TIT2(encoding=3, text=kwargs["title"]))
-#     tags.add(TALB(encoding=3, text=kwargs["album"]))
-#     tags.add(TOPE(encoding=3, text=kwargs["albumartist"]))
-#     tags.add(TPE1(encoding=3, text=kwargs["artist"]))
-#     tags.add(TCOP(encoding=3, text=kwargs["copyright"]))
-#     tags.add(TRCK(encoding=3, text=kwargs["tracknumber"]))
-#     tags.add(TCON(encoding=3, text=kwargs["genre"]))
-#     tags.add(TDRC(encoding=3, text=kwargs["date"]))
-#     tags.add(TCOM(encoding=3, text=kwargs["composer"]))
-#     tags.add(TSRC(encoding=3, text=kwargs["isrc"]))
-#     tags.add(USLT(encoding=3, text=kwargs["lyrics"]))
-#     tags.add(APIC(encoding=3, data=kwargs["cover"], mime=kwargs["cover_mime"]))
-#
-#     match kwargs['cover_mime']:
-#         case 'image/jpeg':
-#             fmt = mp4.AtomDataType(13)
-#         case 'image/png':
-#             fmt = mp4.AtomDataType(14)
-#         case _:
-#             fmt = None
-#
-#     if fmt is not None:
-#         pic = mp4.MP4Cover(kwargs['cover'])
-#         pic.imageformat = fmt
-#
-#     file.tags = tags
-#     file.save()
-
-
 def write(
     fp: BinaryIO,
     mime: str,

tidal_scraper/state.py

@@ -1,13 +1,10 @@
 import json
 from datetime import datetime
 from tidalapi import session, user, playlist, media, album, artist
-from helper import conf, state_dir
 
 
 class State:
-    def __init__(
+    def __init__(self, user_id: int, quality: str, dl_state_path: str):
-        self, user_id: int, quality: str, dl_state_path: str = state_dir + "/state.json"
-    ):
         match quality:
             case "master":
                 q = session.Quality.master
@@ -18,11 +15,11 @@ class State:
             case "low":
                 q = session.Quality.low
             case _:
-                raise Exception("Quality misconfigured in conf.toml")
+                raise Exception("Bad Quality String")
         config = session.Config(quality=q)
         self.user_id = user_id
         self.session = session.Session(config)
-        self.favorites = user.Favorites(self.session, conf["user_id"])
+        self.favorites = user.Favorites(self.session, user_id)
         try:
             self.load_dl_state(dl_state_path)
         except:
@@ -33,7 +30,7 @@ class State:
                 "tracks": {},
             }
 
-    def login(self, auth_file: str | None = state_dir + "/auth.json") -> None:
+    def login(self, auth_file: str | None = None) -> None:
         s = self.session
         try:
             assert auth_file
@@ -84,16 +81,12 @@ class State:
 
         self._state[t][obj.id] = downloaded
 
-    def write_dl_state(self, dl_state_path: str | None = None) -> None:
+    def write_dl_state(self, statefile: str) -> None:
-        if dl_state_path is None:
+        with open(statefile, "w") as f:
-            dl_state_path = state_dir + "/state.json"
-        with open(dl_state_path, "w") as f:
             json.dump(self._state, f)
 
-    def load_dl_state(self, dl_state_path: str | None = None) -> None:
+    def load_dl_state(self, statefile: str) -> None:
-        if dl_state_path is None:
+        with open(statefile, "r") as f:
-            dl_state_path = state_dir + "/state.json"
-        with open(dl_state_path, "r") as f:
             self._state = json.load(f)
 
         assert type(self._state["albums"]) is dict[int, bool]