snailed
/
tidal-scraper
2
0
Fork 0
Code

fix bugs, add cli tool, create package

This commit is contained in:
Luca Bilke 2023-07-05 11:50:05 +02:00
parent 036e981fe2
commit 2ac8a4e1bc
No known key found for this signature in database
GPG Key ID: 7B77C51E8C779E75
11 changed files with 281 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1 +1,4 @@
tidal-scraper/__pycache__
dist
tidal_scraper/__pycache__

7
TODO.md
View File

@ -1,7 +0,0 @@
- [ ] installer or pip package
- [ ] installer should create state and config homes if not existing
- [ ] proper SIGTERM handling
- [ ] decrypt and write in chunks
- [ ] test error logger
- [ ] Switch to tomllib once ubuntu updates their python package to 3.11 (I haven't switched yet to avoid github issues being made)

7
conf.toml
View File

@ -8,6 +8,13 @@ user_id =
dest_dir = "./downloads/"
# Leave empty for default state location
# Default location is the first valid path in this order:
# XDG_STATE_HOME/tidal-scraper
# XDG_CACHE_HOME/tidal-scraper
# $HOME/.cache/tidal-scraper
state_dir =
# These templates are passed their respective tidalapi objects
# Possible attributes can be found here: https://tidalapi.netlify.app/api.html
album_dir = "{album.artist.name}/{album.name}/"

30
pyproject.toml Normal file
View File

@ -0,0 +1,30 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = 'tidal-scraper'
version = '1.0'
description = 'A library to download music from tidal'
authors = [
{ name = "Luca Bilke", email="luca@snaile.de" }
]
requires-python = ">=3.11"
classifiers = [
"Programming Language :: Python :: 3.11",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Operating System :: POSIX",
"Topic :: Multimedia :: Sound/Audio",
"Natural Language :: English",
]
dependencies = [
"pycrypto",
"tqdm",
"mutagen",
"tidalapi @ git+https://github.com/tamland/python-tidal@5207a3cff2af437a2d0d67b743c875a67f8d1d08",
]
[tool.hatch.metadata]
allow-direct-references = true
[project.scripts]
tidal-scraper = "tidal_scraper.cli:run"

42
tidal-scraper/helper.py
View File

@ -1,42 +0,0 @@
import re
import os
import toml
# TODO: wait for python to update to 3.11 for ubuntu users
# import tomllib
import sys
import traceback
extensions = [".flac", ".mp4", ".m4a", ""]
home = os.getenv("HOME")
state_dir = os.getenv("XDG_STATE_HOME") or os.getenv("XDG_CACHE_HOME")
conf_dir = os.getenv("XDG_CONFIG_HOME")
if not state_dir:
assert home
state_dir = home + "/.cache"
if not conf_dir:
assert home
conf_dir = home + "/.config"
conf_dir += "/tidal-scraper"
state_dir += "/tidal-scraper"
with open(conf_dir + "/conf.toml", "r") as f:
conf = toml.load(f)
# with open(conf_dir + "/conf.toml", "rb") as f:
# conf = tomllib.load(f)
def clean_template(path: str, **kwargs) -> str:
path = os.path.expanduser(path)
split = path.split("/")
cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split]
return "/".join(cleaned_split)
def log_error(template: str, **kwargs):
with open(conf["error_log"], "a") as f:
msg = template.format(**kwargs)
f.write(msg + "\n")
traceback.format_exception(*sys.exc_info())
f.write("\n\n")

17
tidal-scraper/run.py
View File

@ -1,17 +0,0 @@
#!/bin/env python3
from download import download_album
from state import State
from helper import conf
s = State(conf['user_id'], conf['quality'])
s.login()
albums = s.favorites.albums()
try:
s.load_dl_state
except:
pass
download_album(albums[0])
s.set_dl_state(albums[0], True)
s.write_dl_state()

117
tidal_scraper/cli.py Normal file
View File

@ -0,0 +1,117 @@
from tidalapi import Album, Track, Artist, Playlist
from tidal_scraper.download import (
download_album,
download_playlist,
download_track,
download_artist,
)
from tidal_scraper.state import State
from tidal_scraper.helper import get_conf, human_sleep
from argparse import ArgumentParser, Namespace
from pathlib import Path
def handle_favorites(state: State, conf: dict, args: Namespace) -> None:
match args.obj:
case "album":
for album in state.favorites.albums():
download_album(album, conf)
human_sleep()
case "track":
for track in state.favorites.tracks():
download_track(track, conf)
human_sleep()
case "artist":
for artist in state.favorites.artists():
download_artist(artist, conf)
human_sleep()
case "playlist":
for playlist in state.favorites.playlists():
download_playlist(playlist, conf)
human_sleep()
def handle_id(state: State, conf: dict, args: Namespace) -> None:
match args.obj:
case "album":
album = Album(state.session, args.id)
download_album(album, conf)
case "track":
track = Track(state.session, args.id)
download_track(track, conf)
case "artist":
artist = Artist(state.session, args.id)
download_artist(artist, conf)
case "playlist":
playlist = Playlist(state.session, args.id)
download_playlist(playlist, conf)
def run():
parser = ArgumentParser(prog="tidal-scraper", description="Tidal music downloader")
parser.add_argument(
"-c", "--config", help="Configuration file to use", type=Path, dest="conf_file"
)
parser.add_argument(
"-s", "--state", help="Directory to keep state in", type=Path, dest="state_dir"
)
obj = parser.add_mutually_exclusive_group(required=True)
obj.add_argument(
"-a",
"--album",
action="store_const",
const="album",
help="Download an album",
dest="obj",
)
obj.add_argument(
"-A",
"--artist",
action="store_const",
const="artist",
help="Download an artist's albums",
dest="obj",
)
obj.add_argument(
"-p",
"--playlist",
action="store_const",
const="playlist",
help="Download a playlist",
dest="obj",
)
obj.add_argument(
"-t",
"--track",
action="store_const",
const="track",
help="Download a single track",
dest="obj",
)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument(
"-f",
"--favorite",
action="store_true",
help="Download all favorited",
dest="favorite",
)
source.add_argument(
"-i",
"--id",
type=int,
help="Download by ID",
dest="id",
)
args = parser.parse_args()
conf = get_conf(args.state_dir, args.conf_file)
state = State(conf["user_id"], conf["quality"], conf["state_dir"])
state.login(conf["state_dir"] + "auth.json")
if args.favorite:
handle_favorites(state, conf, args)
elif args.id is not None:
handle_id(state, conf, args)
state.write_dl_state(conf["state_dir"] + "state.json")

85
tidal-scraper/download.py → tidal_scraper/download.py
View File

@ -1,5 +1,5 @@
import metadata
from helper import conf, extensions, clean_template, log_error
import tidal_scraper.metadata as metadata
from tidal_scraper.helper import extensions, clean_template, log_error, human_sleep
import tidalapi
import os
@ -44,8 +44,6 @@ def __decrypt_file(fp: BinaryIO, key: bytes, nonce: bytes) -> None:
def __download_file(url: str, fp: BinaryIO) -> str:
with requests.get(url, stream=True) as r:
if conf["debug"]:
print(r.headers)
r.raise_for_status()
mime = r.headers.get("Content-Type", "")
total_bytes = int(r.headers.get("Content-Length", 0))
@ -56,22 +54,35 @@ def __download_file(url: str, fp: BinaryIO) -> str:
return mime
def download_track(track: tidalapi.Track, dest: str) -> None:
def download_track(
track: tidalapi.Track,
conf: dict | None = None,
name_template: str | None = None,
dest_dir: str | None = None,
skip_dl: bool | None = None,
errorfile: str | None = None,
) -> None:
album = track.album
assert album
if conf is None:
assert skip_dl is not None
assert errorfile is not None
assert name_template is not None
assert dest_dir is not None
else:
skip_dl = skip_dl or conf["skip_downloaded"]
errorfile = errorfile or conf["error_log"]
dest_dir = dest_dir or conf["dest_dir"]
name_template = name_template or conf["track_name"]
dest = dest_dir + clean_template(name_template, track=track)
print(f"Starting {album.artist.name} - {track.name}")
dest += clean_template(
conf["track_name"],
track=track,
)
http_failures = 0
while http_failures <= 3:
try:
print("running")
stream = track.stream()
manifest = json.loads(b64decode(stream.manifest))
if conf["debug"]:
print(manifest)
url = manifest["urls"][0]
codec = manifest["codecs"]
if ".mp4" in url:
@ -84,7 +95,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
if ext in url:
dest += ext
break
if os.path.exists(dest) and conf["skip_downloaded"]:
if os.path.exists(dest) and skip_dl:
print(f"Skipping track")
return
@ -115,6 +126,7 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
raise e
except Exception as e:
log_error(
errorfile or "error.log",
"Failure while downloading {artist} - {track}",
artist=album.artist.name,
track=track.name,
@ -123,9 +135,35 @@ def download_track(track: tidalapi.Track, dest: str) -> None:
def download_cover(
obj: tidalapi.Album | tidalapi.Playlist, dest: str, size: int
obj: tidalapi.Album | tidalapi.Playlist,
conf: dict | None = None,
dest: str | None = None,
size: int | None = None,
skip_dl: bool | None = None,
) -> None:
if os.path.exists(dest) and conf["skip_downloaded"]:
if conf is None:
assert dest is not None
assert size is not None
assert skip_dl is not None
else:
if type(obj) is tidalapi.Album:
dest = clean_template(
conf["dest_dir"] + "/" + conf["album_dir"],
album=obj,
)
size = conf["album_image_size"]
elif type(obj) is tidalapi.Playlist:
dest = clean_template(
conf["dest_dir"] + "/" + conf["playlist_dir"],
playlist=obj,
)
size = conf["playlist_image_size"]
skip_dl = conf["skip_downloaded"]
assert dest
assert size
if os.path.exists(dest) and skip_dl:
return
url = obj.image(size)
@ -133,31 +171,34 @@ def download_cover(
__download_file(url, f)
def download_album(album: tidalapi.Album) -> None:
def download_album(album: tidalapi.Album, conf: dict) -> None:
dest = clean_template(
conf["dest_dir"] + "/" + conf["album_dir"],
album=album,
)
os.makedirs(os.path.dirname(dest), exist_ok=True)
download_cover(album, dest, conf["album_image_size"])
download_cover(album, conf)
tracks = album.tracks()
for track in tracks:
download_track(track, dest)
download_track(track, conf, dest_dir=dest)
human_sleep()
def download_playlist(playlist: tidalapi.Playlist) -> None:
def download_playlist(playlist: tidalapi.Playlist, conf: dict) -> None:
dest = clean_template(
conf["dest_dir"] + "/" + conf["playlist_dir"],
playlist=playlist,
)
os.makedirs(os.path.dirname(dest), exist_ok=True)
download_cover(playlist, dest, conf["playlist_image_size"])
download_cover(playlist, conf)
tracks = playlist.tracks()
for track in tracks:
download_track(track, dest)
download_track(track, conf, dest_dir=dest)
human_sleep()
def download_artist(artist: tidalapi.Artist) -> None:
def download_artist(artist: tidalapi.Artist, conf: dict) -> None:
albums = artist.get_albums()
for album in albums:
download_album(album)
download_album(album, conf)
human_sleep()

50
tidal_scraper/helper.py Normal file
View File

@ -0,0 +1,50 @@
import re
import os
import time
import random
import tomllib
import sys
import traceback
extensions = [".flac", ".mp4", ".m4a", ""]
def get_conf(state_dir: str | None = None, conf_file: str | None = None) -> dict:
home = os.getenv("HOME")
assert home
conf_file = (os.getenv("XDG_CONFIG_HOME") or home + "/.config") + "/tidal-scraper/conf.toml"
with open(conf_file, "rb") as f:
conf = tomllib.load(f)
if not state_dir:
state_dir = (
os.getenv("XDG_STATE_HOME")
or os.getenv("XDG_CACHE_HOME")
or home + "/.cache"
)
state_dir += "/tidal-scraper"
conf["state_dir"] = state_dir
return conf
def clean_template(path: str, **kwargs) -> str:
path = os.path.expanduser(path)
split = path.split("/")
cleaned_split = [re.sub("/", " ", s.format(**kwargs)) for s in split]
return "/".join(cleaned_split)
def log_error(logfile: str, template: str, **kwargs):
with open(logfile, "a") as f:
msg = template.format(**kwargs)
f.write(msg + "\n")
traceback.format_exception(*sys.exc_info())
f.write("\n\n")
def human_sleep() -> None:
t = random.randrange(10, 50) / 10
time.sleep(t)

56
tidal-scraper/metadata.py → tidal_scraper/metadata.py
View File

@ -1,36 +1,17 @@
from mutagen import flac, mp4 # , mp3
from mutagen import flac, mp4
from mutagen.mp4 import MP4Tags
from mutagen._vorbis import VCommentDict
from typing import BinaryIO
# from mutagen.id3._tags import ID3Tags
# from mutagen.id3._frames import (
# APIC,
# TALB,
# TCOP,
# TDRC,
# TIT2,
# TPE1,
# TRCK,
# TOPE,
# TCON,
# TCOM,
# TSRC,
# USLT,
# )
def __write_flac(file: flac.FLAC, **kwargs) -> None:
tags = VCommentDict()
tags["title"] = kwargs["title"]
tags["album"] = kwargs["album"]
# There doesn't seem to be a standard way of listing multiple artists in an ID3 tag
# This method seems to be the most widely recognized
tags["albumartist"] = "; ".join(kwargs["albumartist"])
tags["artist"] = "; ".join(kwargs["artist"])
tags["albumartist"] = kwargs["albumartist"]
tags["artist"] = kwargs["artist"]
tags["copyright"] = kwargs["copyright"]
tags["tracknumber"] = kwargs["tracknumber"]
tags["tracktotal"] = kwargs["tracktotal"]
@ -70,37 +51,6 @@ def __write_mp4(file: mp4.MP4, **kwargs) -> None:
file.save
# def __write_mp3(file: mp3.MP3, **kwargs) -> None:
# tags = ID3Tags()
# tags.add(TIT2(encoding=3, text=kwargs["title"]))
# tags.add(TALB(encoding=3, text=kwargs["album"]))
# tags.add(TOPE(encoding=3, text=kwargs["albumartist"]))
# tags.add(TPE1(encoding=3, text=kwargs["artist"]))
# tags.add(TCOP(encoding=3, text=kwargs["copyright"]))
# tags.add(TRCK(encoding=3, text=kwargs["tracknumber"]))
# tags.add(TCON(encoding=3, text=kwargs["genre"]))
# tags.add(TDRC(encoding=3, text=kwargs["date"]))
# tags.add(TCOM(encoding=3, text=kwargs["composer"]))
# tags.add(TSRC(encoding=3, text=kwargs["isrc"]))
# tags.add(USLT(encoding=3, text=kwargs["lyrics"]))
# tags.add(APIC(encoding=3, data=kwargs["cover"], mime=kwargs["cover_mime"]))
#
# match kwargs['cover_mime']:
# case 'image/jpeg':
# fmt = mp4.AtomDataType(13)
# case 'image/png':
# fmt = mp4.AtomDataType(14)
# case _:
# fmt = None
#
# if fmt is not None:
# pic = mp4.MP4Cover(kwargs['cover'])
# pic.imageformat = fmt
#
# file.tags = tags
# file.save()
def write(
fp: BinaryIO,
mime: str,

23
tidal-scraper/state.py → tidal_scraper/state.py
View File

@ -1,13 +1,10 @@
import json
from datetime import datetime
from tidalapi import session, user, playlist, media, album, artist
from helper import conf, state_dir
class State:
def __init__(
self, user_id: int, quality: str, dl_state_path: str = state_dir + "/state.json"
):
def __init__(self, user_id: int, quality: str, dl_state_path: str):
match quality:
case "master":
q = session.Quality.master
@ -18,11 +15,11 @@ class State:
case "low":
q = session.Quality.low
case _:
raise Exception("Quality misconfigured in conf.toml")
raise Exception("Bad Quality String")
config = session.Config(quality=q)
self.user_id = user_id
self.session = session.Session(config)
self.favorites = user.Favorites(self.session, conf["user_id"])
self.favorites = user.Favorites(self.session, user_id)
try:
self.load_dl_state(dl_state_path)
except:
@ -33,7 +30,7 @@ class State:
"tracks": {},
}
def login(self, auth_file: str | None = state_dir + "/auth.json") -> None:
def login(self, auth_file: str | None = None) -> None:
s = self.session
try:
assert auth_file
@ -84,16 +81,12 @@ class State:
self._state[t][obj.id] = downloaded
def write_dl_state(self, dl_state_path: str | None = None) -> None:
if dl_state_path is None:
dl_state_path = state_dir + "/state.json"
with open(dl_state_path, "w") as f:
def write_dl_state(self, statefile: str) -> None:
with open(statefile, "w") as f:
json.dump(self._state, f)
def load_dl_state(self, dl_state_path: str | None = None) -> None:
if dl_state_path is None:
dl_state_path = state_dir + "/state.json"
with open(dl_state_path, "r") as f:
def load_dl_state(self, statefile: str) -> None:
with open(statefile, "r") as f:
self._state = json.load(f)
assert type(self._state["albums"]) is dict[int, bool]