From 6bda8eeca4bef2c05804f0c17278bc3547689069 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Sat, 18 Apr 2020 20:39:03 -0600 Subject: [PATCH] Working on moving playlist details over to AdapterManager --- Pipfile | 1 + Pipfile.lock | 16 ++- docs/index.rst | 2 +- setup.py | 1 + sublime/adapters/adapter_base.py | 3 +- sublime/adapters/adapter_manager.py | 96 ++++++++++++++++-- sublime/adapters/api_objects.py | 14 +-- sublime/adapters/filesystem/adapter.py | 86 +++++----------- sublime/adapters/filesystem/database.py | 51 ++++++++++ sublime/adapters/subsonic/adapter.py | 53 +++++++--- sublime/cache_manager.py | 2 +- sublime/database/tables.py | 127 ------------------------ sublime/ui/playlists.py | 2 +- 13 files changed, 230 insertions(+), 224 deletions(-) create mode 100644 sublime/adapters/filesystem/database.py delete mode 100644 sublime/database/tables.py diff --git a/Pipfile b/Pipfile index 0783502..fa95a1c 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ sphinx = "*" sphinx-rtd-theme = "*" termcolor = "*" yapf = "*" +flake8-import-order = "*" [packages] sublime-music = {editable = true,extras = ["keyring"],path = "."} diff --git a/Pipfile.lock b/Pipfile.lock index a65bc09..9ff7f73 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ec62e729c70d9ce38576ae97cfdd88b0a006886cbca1fa4ff0160383e96453a6" + "sha256": "ada2f525e7d89500110d3063b4c271d60dd9e8a026ced396d5f3058b3e29e386" }, "pipfile-spec": 6, "requires": { @@ -156,6 +156,12 @@ ], "version": "==4.0.1" }, + "peewee": { + "hashes": [ + "sha256:1269a9736865512bd4056298003aab190957afe07d2616cf22eaf56cb6398369" + ], + "version": "==3.13.3" + }, "protobuf": { "hashes": [ "sha256:0bae429443cc4748be2aadfdaf9633297cfaeb24a9a02d0ab15849175ce90fab", @@ -401,6 +407,14 @@ "index": "pypi", "version": "==3.2.2" }, + "flake8-import-order": { + "hashes": [ + "sha256:90a80e46886259b9c396b578d75c749801a41ee969a235e163cfe1be7afd2543", + "sha256:a28dc39545ea4606c1ac3c24e9d05c849c6e5444a50fb7e9cdd430fc94de6e92" + ], + "index": "pypi", + "version": "==0.18.1" + }, "flake8-pep3101": { "hashes": [ "sha256:86e3eb4e42de8326dcd98ebdeaf9a3c6854203a48f34aeb3e7e8ed948107f512", diff --git a/docs/index.rst b/docs/index.rst index 2947250..377a310 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ Linux Desktop. .. _Navidrome: https://www.navidrome.org/ .. figure:: ./_static/screenshots/play-queue.png - :width: 80 % + :width: 80% :align: center :target: ./_static/screenshots/play-queue.png diff --git a/setup.py b/setup.py index dbbab2e..0feb0c7 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( 'Deprecated', 'fuzzywuzzy', 'osxmmkeys ; sys_platform=="darwin"', + 'peewee', 'pychromecast', 'PyGObject', 'python-dateutil', diff --git a/sublime/adapters/adapter_base.py b/sublime/adapters/adapter_base.py index 62b7247..ce73450 100644 --- a/sublime/adapters/adapter_base.py +++ b/sublime/adapters/adapter_base.py @@ -160,7 +160,8 @@ class Adapter(abc.ABC): def can_service_requests(self) -> bool: """ Specifies whether or not the adapter can currently service requests. If - this is ``False``, none of the other functions are expected to work. + this is ``False``, none of the other data retrieval functions are + expected to work. For example, if your adapter requires access to an external service, use this function to determine if it is currently possible to connect diff --git a/sublime/adapters/adapter_manager.py b/sublime/adapters/adapter_manager.py index 9a5b24d..d0c812a 100644 --- a/sublime/adapters/adapter_manager.py +++ b/sublime/adapters/adapter_manager.py @@ -73,6 +73,7 @@ class Result(Generic[T]): class AdapterManager: available_adapters: Set[Any] = {FilesystemAdapter, SubsonicAdapter} executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=50) + is_shutting_down: bool = False @dataclass class _AdapterManagerInternal: @@ -104,8 +105,13 @@ class AdapterManager: @staticmethod def shutdown(): - assert AdapterManager._instance - AdapterManager._instance.shutdown() + logging.info('AdapterManager shutdown start') + AdapterManager.is_shutting_down = True + AdapterManager.executor.shutdown() + if AdapterManager._instance: + AdapterManager._instance.shutdown() + + logging.info('CacheManager shutdown complete') @staticmethod def reset(config: AppConfiguration): @@ -131,7 +137,7 @@ class AdapterManager: caching_adapter_type = FilesystemAdapter caching_adapter = None - if caching_adapter_type: + if caching_adapter_type and ground_truth_adapter_type.can_be_cached: caching_adapter = caching_adapter_type( { key: getattr(config.server, key) @@ -146,13 +152,24 @@ class AdapterManager: caching_adapter=caching_adapter, ) + @staticmethod + def can_get_playlists() -> bool: + # It only matters that the ground truth one can service the request. + return ( + AdapterManager._instance.ground_truth_adapter.can_service_requests + and + AdapterManager._instance.ground_truth_adapter.can_get_playlists) + @staticmethod def get_playlists( before_download: Callable[[], None] = lambda: None, force: bool = False, # TODO: rename to use_ground_truth_adapter? ) -> Result[List[Playlist]]: assert AdapterManager._instance - if not force and AdapterManager._instance.caching_adapter: + if (not force and AdapterManager._instance.caching_adapter and + AdapterManager._instance.caching_adapter.can_service_requests + and + AdapterManager._instance.caching_adapter.can_get_playlists): try: return Result( AdapterManager._instance.caching_adapter.get_playlists()) @@ -162,6 +179,13 @@ class AdapterManager: logging.exception( f'Error on {"get_playlists"} retrieving from cache.') + if (AdapterManager._instance.ground_truth_adapter + and not AdapterManager._instance.ground_truth_adapter + .can_service_requests and not AdapterManager._instance + .ground_truth_adapter.can_get_playlists): + raise Exception( + f'No adapters can service {"get_playlists"} at the moment.') + def future_fn() -> List[Playlist]: assert AdapterManager._instance if before_download: @@ -176,7 +200,6 @@ class AdapterManager: def future_finished(f: Future): assert AdapterManager._instance assert AdapterManager._instance.caching_adapter - print(' ohea', f) AdapterManager._instance.caching_adapter.ingest_new_data( 'get_playlists', (), @@ -188,5 +211,64 @@ class AdapterManager: return future @staticmethod - def get_playlist_details(playlist_id: str) -> Result[PlaylistDetails]: - raise NotImplementedError() + def can_get_playlist_details() -> bool: + # It only matters that the ground truth one can service the request. + return ( + AdapterManager._instance.ground_truth_adapter.can_service_requests + and AdapterManager._instance.ground_truth_adapter + .can_get_playlist_details) + + @staticmethod + def get_playlist_details( + playlist_id: str, + before_download: Callable[[], None] = lambda: None, + force: bool = False, # TODO: rename to use_ground_truth_adapter? + ) -> Result[PlaylistDetails]: + assert AdapterManager._instance + if (not force and AdapterManager._instance.caching_adapter and + AdapterManager._instance.caching_adapter.can_service_requests + and AdapterManager._instance.caching_adapter + .can_get_playlist_details): + try: + return Result( + AdapterManager._instance.caching_adapter + .get_playlist_details(playlist_id)) + except CacheMissError: + logging.debug(f'Cache Miss on {"get_playlist_details"}.') + except Exception: + logging.exception( + f'Error on {"get_playlist_details"} retrieving from cache.' + ) + + if (AdapterManager._instance.ground_truth_adapter + and not AdapterManager._instance.ground_truth_adapter + .can_service_requests and not AdapterManager._instance + .ground_truth_adapter.can_get_playlist_details): + raise Exception( + f'No adapters can service {"get_playlist_details"} at the moment.' + ) + + def future_fn() -> PlaylistDetails: + assert AdapterManager._instance + if before_download: + before_download() + return ( + AdapterManager._instance.ground_truth_adapter + .get_playlist_details(playlist_id)) + + future: Result[PlaylistDetails] = Result(future_fn) + + if AdapterManager._instance.caching_adapter: + + def future_finished(f: Future): + assert AdapterManager._instance + assert AdapterManager._instance.caching_adapter + AdapterManager._instance.caching_adapter.ingest_new_data( + 'get_playlist_details', + (playlist_id, ), + f.result(), + ) + + future.add_done_callback(future_finished) + + return future diff --git a/sublime/adapters/api_objects.py b/sublime/adapters/api_objects.py index a3097d6..6c19e90 100644 --- a/sublime/adapters/api_objects.py +++ b/sublime/adapters/api_objects.py @@ -1,7 +1,7 @@ """ Defines the objects that are returned by adapter methods. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime, timedelta from typing import List, Optional @@ -15,26 +15,26 @@ class Song: class Playlist: id: str name: str - songCount: Optional[int] = None # TODO rename + song_count: Optional[int] = None duration: Optional[timedelta] = None created: Optional[datetime] = None changed: Optional[datetime] = None comment: Optional[str] = None owner: Optional[str] = None public: Optional[bool] = None - coverArt: Optional[str] = None # TODO rename + cover_art: Optional[str] = None @dataclass(frozen=True) -class PlaylistDetails(): +class PlaylistDetails: id: str name: str - songCount: int # TODO rename + song_count: int duration: timedelta + songs: List[Song] created: Optional[datetime] = None changed: Optional[datetime] = None comment: Optional[str] = None owner: Optional[str] = None public: Optional[bool] = None - coverArt: Optional[str] = None # TODO rename - songs: List[Song] = field(default_factory=list) + cover_art: Optional[str] = None diff --git a/sublime/adapters/filesystem/adapter.py b/sublime/adapters/filesystem/adapter.py index 794b310..77bf439 100644 --- a/sublime/adapters/filesystem/adapter.py +++ b/sublime/adapters/filesystem/adapter.py @@ -1,10 +1,14 @@ import logging -import sqlite3 -from typing import Any, Dict, List, Optional, Tuple +from dataclasses import asdict from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from playhouse.sqliteq import SqliteQueueDatabase from sublime.adapters.api_objects import (Playlist, PlaylistDetails) -from .. import CachingAdapter, ConfigParamDescriptor, CacheMissError + +from . import database +from .. import CacheMissError, CachingAdapter, ConfigParamDescriptor class FilesystemAdapter(CachingAdapter): @@ -31,35 +35,14 @@ class FilesystemAdapter(CachingAdapter): ): self.data_directory = data_directory logging.info('Opening connection to the database.') - self.database_filename = data_directory.joinpath('.cache_meta.db') - database_connection = sqlite3.connect( - self.database_filename, - detect_types=sqlite3.PARSE_DECLTYPES, + database_filename = data_directory.joinpath('cache.db') + self.database = SqliteQueueDatabase( + database_filename, + autorollback=True, ) - - # TODO extract this out eventually - c = database_connection.cursor() - c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='playlists';" - ) - if not c.fetchone(): - c.execute( - """ - CREATE TABLE playlists ( - id TEXT NOT NULL UNIQUE PRIMARY KEY, - name TEXT NOT NULL, - song_count INTEGER, - duration INTEGER, - created INTEGER, - changed INTEGER, - comment TEXT, - owner TEXT, -- TODO convert to a a FK - public INT, - cover_art TEXT -- TODO convert to a FK - ) - """) - - c.close() + database.proxy.initialize(self.database) + self.database.connect() + self.database.create_tables(database.ALL_TABLES) def shutdown(self): logging.info('Shutdown complete') @@ -78,18 +61,10 @@ class FilesystemAdapter(CachingAdapter): can_get_playlists: bool = True def get_playlists(self) -> List[Playlist]: - database_connection = sqlite3.connect( - self.database_filename, - detect_types=sqlite3.PARSE_DECLTYPES, - ) - with database_connection: - playlists = database_connection.execute( - """ - SELECT * from playlists - """).fetchall() - return [Playlist(*p) for p in playlists] - - raise CacheMissError() + playlists = list(database.Playlist.select()) + if len(playlists) == 0: # TODO not necessarily a cache miss + raise CacheMissError() + return playlists can_get_playlist_details: bool = True @@ -107,23 +82,8 @@ class FilesystemAdapter(CachingAdapter): params: Tuple[Any, ...], data: Any, ): - database_connection = sqlite3.connect( - self.database_filename, - detect_types=sqlite3.PARSE_DECLTYPES, - ) - with database_connection: - if function_name == 'get_playlists': - database_connection.executemany( - """ - INSERT OR IGNORE INTO playlists (id, name, song_count, duration, created, comment, owner, public, cover_art) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (id) DO - UPDATE SET id=?, name=?, song_count=?, duration=?, created=?, comment=?, owner=?, public=?, cover_art=?; - """, [ - ( - p.id, p.name, p.songCount, p.duration, p.created, - p.comment, p.owner, p.public, p.coverArt, p.id, - p.name, p.songCount, p.duration, p.created, - p.comment, p.owner, p.public, p.coverArt) - for p in data - ]) + if function_name == 'get_playlists': + ( + database.Playlist.insert_many( + map(lambda p: database.Playlist(**asdict(p)), + data)).on_conflict_replace()) diff --git a/sublime/adapters/filesystem/database.py b/sublime/adapters/filesystem/database.py new file mode 100644 index 0000000..0f97ae1 --- /dev/null +++ b/sublime/adapters/filesystem/database.py @@ -0,0 +1,51 @@ +from datetime import timedelta +from typing import Any, Optional + +from peewee import ( + BooleanField, + DatabaseProxy, + DateTimeField, + Field, + ForeignKeyField, + IntegerField, + Model, + TextField, +) + +proxy = DatabaseProxy() + + +class BaseModel(Model): + class Meta: + database = proxy + + +class DurationField(IntegerField): + def db_value(self, value: timedelta) -> Optional[int]: + return value.microseconds if value else None + + def python_value(self, value: Optional[int]) -> Optional[timedelta]: + return timedelta(microseconds=value) if value else None + + +class CoverArt(BaseModel): + id = TextField(unique=True, primary_key=True) + url = TextField() + filename = TextField(null=True) + + +class Playlist(BaseModel): + id = TextField(unique=True, primary_key=True) + name = TextField() + song_count = IntegerField() + duration = DurationField() + created = DateTimeField() + changed = DateTimeField() + public = BooleanField() + cover_art = TextField() + + +ALL_TABLES = ( + CoverArt, + Playlist, +) diff --git a/sublime/adapters/subsonic/adapter.py b/sublime/adapters/subsonic/adapter.py index 50f94ad..c1c1283 100644 --- a/sublime/adapters/subsonic/adapter.py +++ b/sublime/adapters/subsonic/adapter.py @@ -1,12 +1,18 @@ import logging import os -import requests -from datetime import datetime +import re +from datetime import datetime, timedelta +from pathlib import Path from time import sleep from typing import Any, Dict, List, Optional, Union -from pathlib import Path -from sublime.adapters.api_objects import (Playlist, PlaylistDetails) +import requests + +from sublime.adapters.api_objects import ( + Playlist, + PlaylistDetails, + Song, +) from .. import Adapter, ConfigParamDescriptor @@ -35,7 +41,6 @@ class SubsonicAdapter(Adapter): errors: Dict[str, Optional[str]] = {} # TODO: verify the URL - return errors def __init__(self, config: dict, data_directory: Path): @@ -125,22 +130,30 @@ class SubsonicAdapter(Adapter): raise Exception(f'Subsonic API Error #{code}: {message}') return subsonic_response - response = Response.from_json(subsonic_response) - # Check for an error and if it exists, raise it. - if response.error: - raise Server.SubsonicServerError(response.error) + _snake_case_re = re.compile(r'(? Dict[str, Any]: + return { + self._snake_case_re.sub('_', k).lower(): v + for k, v in obj.items() + } # Data Retrieval Methods # ========================================================================= can_get_playlists = True def get_playlists(self) -> List[Playlist]: - result = self._get_json(self._make_url('getPlaylists')).get( - 'playlists', {}).get('playlist') - return [Playlist(**p) for p in result] + result = [ + { + **p, + 'duration': + timedelta( + seconds=p['duration']) if p.get('duration') else None, + } for p in self._get_json(self._make_url('getPlaylists')).get( + 'playlists', {}).get('playlist') + ] + return [Playlist(**self._to_snake_case(p)) for p in result] can_get_playlist_details = True @@ -148,5 +161,15 @@ class SubsonicAdapter(Adapter): self, playlist_id: str, ) -> PlaylistDetails: - raise NotImplementedError() - + result = self._get_json( + self._make_url('getPlaylist'), + id=playlist_id, + ).get('playlist') + print(result) + assert result + result['duration'] = result.get('duration') or sum( + s.get('duration') or 0 for s in result['entry']) + result['songCount'] = result.get('songCount') or len(result['entry']) + songs = [Song(id=s['id']) for s in result['entry']] + del result['entry'] + return PlaylistDetails(**self._to_snake_case(result), songs=songs) diff --git a/sublime/cache_manager.py b/sublime/cache_manager.py index fcf0f04..b1297ec 100644 --- a/sublime/cache_manager.py +++ b/sublime/cache_manager.py @@ -268,8 +268,8 @@ class CacheManager(metaclass=Singleton): @staticmethod def shutdown(): - CacheManager.should_exit = True logging.info('CacheManager shutdown start') + CacheManager.should_exit = True CacheManager.executor.shutdown() CacheManager._instance.save_cache_info() logging.info('CacheManager shutdown complete') diff --git a/sublime/database/tables.py b/sublime/database/tables.py deleted file mode 100644 index fe70afe..0000000 --- a/sublime/database/tables.py +++ /dev/null @@ -1,127 +0,0 @@ -import sqlite3 -import typing -from datetime import datetime -from typing import Optional - -song_table = ''' -CREATE TABLE songs -( - id TEXT NOT NULL UNIQUE PRIMARY KEY, - parent TEXT, - title TEXT, - track INTEGER, - year INTEGER, - genre TEXT, - cover_art TEXT, - size INTEGER, - duration INTEGER, - path TEXT -) -''' - -conn = sqlite3.connect(':memory:') - -c = conn.cursor() -c.execute(song_table) -c.execute('''INSERT INTO songs (id, parent, title, year, genre) - VALUES ('1', 'ohea', 'Awake My Soul', 2019, 'Contemporary Christian')''') -c.execute('''INSERT INTO songs (id, parent, title, year, genre) - VALUES ('2', 'ohea', 'Way Maker', 2019, 'Contemporary Christian')''') -conn.commit() -c.execute('''SELECT * FROM songs''') -print(c.fetchall()) -conn.close() - -table_definitions = ( - ( - 'playlist', - { - 'id': int, - 'name': str, - 'comment': Optional[str], - 'owner': Optional[str], - 'public': Optional[bool], - 'song_count': int, - 'duration': int, - 'created': datetime, - 'changed': datetime, - 'cover_art': Optional[str], - }, - ), - ( - 'song', - { - 'id': int, - 'parent': Optional[str], - 'title': str, - }, - ), - ( - 'playlist_song_xref', - { - 'playlist_id': int, - 'song_id': int, - 'position': int, - }, - ), -) - -python_to_sql_type = { - int: 'INTEGER', - float: 'REAL', - bool: 'INTEGER', - datetime: 'TIMESTAMP', - str: 'TEXT', - bytes: 'BLOB', -} - -for name, fields in table_definitions: - field_defs = [] - for field, field_type in fields.items(): - if type(field_type) == tuple: - print(field_type) - elif type(field_type) is typing._GenericAlias: # type: ignore - sql_type = python_to_sql_type.get(field_type.__args__[0]) - constraints = '' - else: - sql_type = python_to_sql_type.get(field_type) - constraints = ' NOT NULL' - - field_defs.append(f'{field} {sql_type}{constraints}') - - print(f''' - CREATE TABLE {name} - ({','.join(field_defs)}) - ''') - -''' - - id: str - value: str - parent: Optional[str] - isDir: bool - title: str - album: Optional[str] - artist: Optional[str] - track: Optional[int] - year: Optional[int] - genre: Optional[str] - coverArt: Optional[str] - size: Optional[int] - contentType: Optional[str] - suffix: Optional[str] - transcodedContentType: Optional[str] - transcodedSuffix: Optional[str] - duration: Optional[int] - bitRate: Optional[int] - path: Optional[str] - userRating: Optional[UserRating] - averageRating: Optional[AverageRating] - playCount: Optional[int] - discNumber: Optional[int] - created: Optional[datetime] - starred: Optional[datetime] - albumId: Optional[str] - artistId: Optional[str] - type: Optional[MediaType] -''' diff --git a/sublime/ui/playlists.py b/sublime/ui/playlists.py index d9ebc9b..986b7c5 100644 --- a/sublime/ui/playlists.py +++ b/sublime/ui/playlists.py @@ -462,7 +462,7 @@ class PlaylistDetailPanel(Gtk.Overlay): ) @util.async_callback( - lambda *a, **k: CacheManager.get_playlist(*a, **k), + lambda *a, **k: AdapterManager.get_playlist_details(*a, **k), before_download=lambda self: self.show_loading_all(), on_failure=lambda self, e: self.playlist_view_loading_box.hide(), )