From 7e86106562e23f45532e2968725a3a30afc155a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Arnauts?= Date: Sun, 22 Mar 2020 15:37:15 +0100 Subject: [PATCH] Implement program caching for more metadata in listings (#8) --- Makefile | 2 +- addon.xml | 1 + .../resource.language.en_gb/strings.po | 33 ++++ .../resource.language.nl_nl/strings.po | 33 ++++ resources/lib/addon.py | 27 ++- resources/lib/kodiutils.py | 34 +++- resources/lib/modules/catalog.py | 30 ++-- resources/lib/modules/menu.py | 11 +- resources/lib/modules/metadata.py | 60 +++++++ resources/lib/modules/player.py | 17 +- resources/lib/modules/tvguide.py | 2 +- resources/lib/service.py | 83 +++++++++ resources/lib/viervijfzes/auth.py | 63 +++---- resources/lib/viervijfzes/auth_awsidp.py | 8 +- resources/lib/viervijfzes/content.py | 167 ++++++++++++++---- resources/lib/viervijfzes/search.py | 2 +- resources/settings.xml | 7 + service_entry.py | 8 + test/test_api.py | 9 +- test/test_auth.py | 9 +- test/test_epg.py | 8 +- 21 files changed, 482 insertions(+), 132 deletions(-) create mode 100644 resources/lib/modules/metadata.py create mode 100644 resources/lib/service.py create mode 100644 service_entry.py diff --git a/Makefile b/Makefile index 796cb29..3f2bb3d 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ version = $(shell xmllint --xpath 'string(/addon/@version)' addon.xml) git_branch = $(shell git rev-parse --abbrev-ref HEAD) git_hash = $(shell git rev-parse --short HEAD) zip_name = $(name)-$(version)-$(git_branch)-$(git_hash).zip -include_files = addon_entry.py addon.xml CHANGELOG.md LICENSE README.md resources/ +include_files = addon_entry.py addon.xml CHANGELOG.md LICENSE README.md service_entry.py resources/ include_paths = $(patsubst %,$(name)/%,$(include_files)) exclude_files = \*.new \*.orig \*.pyc \*.pyo diff --git a/addon.xml b/addon.xml index c11a84b..18b0fad 100644 --- a/addon.xml +++ b/addon.xml @@ -10,6 +10,7 @@ video + Watch content from VIER, VIJF and ZES. all diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 803dfa1..a558f5a 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -120,6 +120,18 @@ msgctxt "#30713" msgid "The requested video was not found in the guide." msgstr "" +msgctxt "#30714" +msgid "Local metadata is cleared." +msgstr "" + +msgctxt "#30715" +msgid "Updating metadata" +msgstr "" + +msgctxt "#30716" +msgid "Updating metadata ({index}/{total})..." +msgstr "" + msgctxt "#30717" msgid "This program is not available in the catalogue." msgstr "" @@ -141,3 +153,24 @@ msgstr "" msgctxt "#30805" msgid "Password" msgstr "" + +msgctxt "#30820" +msgid "Interface" +msgstr "" + +msgctxt "#30827" +msgid "Metadata" +msgstr "" + +msgctxt "#30829" +msgid "Periodically refresh metadata in the background" +msgstr "" + +msgctxt "#30831" +msgid "Update local metadata now" +msgstr "" + +msgctxt "#30833" +msgid "Clear local metadata" +msgstr "" + diff --git a/resources/language/resource.language.nl_nl/strings.po b/resources/language/resource.language.nl_nl/strings.po index ebd4766..f98a5c5 100644 --- a/resources/language/resource.language.nl_nl/strings.po +++ b/resources/language/resource.language.nl_nl/strings.po @@ -121,6 +121,18 @@ msgctxt "#30713" msgid "The requested video was not found in the guide." msgstr "De gevraagde video werd niet gevonden in de tv-gids." +msgctxt "#30714" +msgid "Local metadata is cleared." +msgstr "De locale metadata is verwijderd." + +msgctxt "#30715" +msgid "Updating metadata" +msgstr "Vernieuwen metadata" + +msgctxt "#30716" +msgid "Updating metadata ({index}/{total})..." +msgstr "Vernieuwen metadata ({index}/{total})..." + msgctxt "#30717" msgid "This program is not available in the catalogue." msgstr "Dit programma is niet beschikbaar in de catalogus." @@ -142,3 +154,24 @@ msgstr "E-mailadres" msgctxt "#30805" msgid "Password" msgstr "Wachtwoord" + +msgctxt "#30820" +msgid "Interface" +msgstr "Interface" + +msgctxt "#30827" +msgid "Metadata" +msgstr "Metadata" + +msgctxt "#30829" +msgid "Periodically refresh metadata in the background" +msgstr "Vernieuw de metdata automatisch in de achtergrond" + +msgctxt "#30831" +msgid "Update local metadata now" +msgstr "De locale metadata nu vernieuwen" + +msgctxt "#30833" +msgid "Clear local metadata" +msgstr "De locale metadata verwijderen" + diff --git a/resources/lib/addon.py b/resources/lib/addon.py index a8fab98..65f287a 100644 --- a/resources/lib/addon.py +++ b/resources/lib/addon.py @@ -3,12 +3,15 @@ from __future__ import absolute_import, division, unicode_literals +import logging + from routing import Plugin from resources.lib import kodilogging kodilogging.config() routing = Plugin() +_LOGGER = logging.getLogger('addon') @routing.route('/') @@ -67,11 +70,11 @@ def show_catalog_program(channel, program): Catalog().show_program(channel, program) -@routing.route('/program/program///') +@routing.route('/catalog/program///') def show_catalog_program_season(channel, program, season): """ Show a program from the catalog """ from resources.lib.modules.catalog import Catalog - Catalog().show_program_season(channel, program, int(season)) + Catalog().show_program_season(channel, program, season) @routing.route('/search') @@ -82,11 +85,11 @@ def show_search(query=None): Search().show_search(query) -@routing.route('/play/catalog//') -def play(channel, uuid): +@routing.route('/play/catalog/') +def play(uuid): """ Play the requested item """ from resources.lib.modules.player import Player - Player().play(channel, uuid) + Player().play(uuid) @routing.route('/play/page//') @@ -101,6 +104,20 @@ def play_from_page(channel, page): Player().play_from_page(channel, unquote(page)) +@routing.route('/metadata/update') +def metadata_update(): + """ Update the metadata for the listings (called from settings) """ + from resources.lib.modules.metadata import Metadata + Metadata().update() + + +@routing.route('/metadata/clean') +def metadata_clean(): + """ Clear metadata (called from settings) """ + from resources.lib.modules.metadata import Metadata + Metadata().clean() + + def run(params): """ Run the routing plugin """ routing.run(params) diff --git a/resources/lib/kodiutils.py b/resources/lib/kodiutils.py index 60e7ca2..4c4d67f 100644 --- a/resources/lib/kodiutils.py +++ b/resources/lib/kodiutils.py @@ -16,13 +16,14 @@ ADDON = xbmcaddon.Addon() SORT_METHODS = dict( unsorted=xbmcplugin.SORT_METHOD_UNSORTED, label=xbmcplugin.SORT_METHOD_LABEL_IGNORE_FOLDERS, + title=xbmcplugin.SORT_METHOD_TITLE, episode=xbmcplugin.SORT_METHOD_EPISODE, duration=xbmcplugin.SORT_METHOD_DURATION, year=xbmcplugin.SORT_METHOD_VIDEO_YEAR, date=xbmcplugin.SORT_METHOD_DATE, ) DEFAULT_SORT_METHODS = [ - 'unsorted', 'label' + 'unsorted', 'title' ] _LOGGER = logging.getLogger('kodiutils') @@ -269,7 +270,7 @@ def set_locale(): setlocale(LC_ALL, locale_lang) except (Error, ValueError) as exc: if locale_lang != 'en_GB': - _LOGGER.debug("Your system does not support locale '{locale}': {error}", locale=locale_lang, error=exc) + _LOGGER.debug("Your system does not support locale '%s': %s", locale_lang, exc) set_locale.cached = False return False set_locale.cached = True @@ -423,14 +424,14 @@ def listdir(path): def mkdir(path): """Create a directory (using xbmcvfs)""" from xbmcvfs import mkdir as vfsmkdir - _LOGGER.debug("Create directory '{path}'.", path=path) + _LOGGER.debug("Create directory '%s'.", path) return vfsmkdir(path) def mkdirs(path): """Create directory including parents (using xbmcvfs)""" from xbmcvfs import mkdirs as vfsmkdirs - _LOGGER.debug("Recursively create directory '{path}'.", path=path) + _LOGGER.debug("Recursively create directory '%s'.", path) return vfsmkdirs(path) @@ -458,14 +459,14 @@ def stat_file(path): def delete(path): """Remove a file (using xbmcvfs)""" from xbmcvfs import delete as vfsdelete - _LOGGER.debug("Delete file '{path}'.", path=path) + _LOGGER.debug("Delete file '%s'.", path) return vfsdelete(path) def container_refresh(url=None): """Refresh the current container or (re)load a container by URL""" if url: - _LOGGER.debug('Execute: Container.Refresh({url})', url=url) + _LOGGER.debug('Execute: Container.Refresh(%s)', url) xbmc.executebuiltin('Container.Refresh({url})'.format(url=url)) else: _LOGGER.debug('Execute: Container.Refresh') @@ -475,7 +476,7 @@ def container_refresh(url=None): def container_update(url): """Update the current container while respecting the path history.""" if url: - _LOGGER.debug('Execute: Container.Update({url})', url=url) + _LOGGER.debug('Execute: Container.Update(%s)', url) xbmc.executebuiltin('Container.Update({url})'.format(url=url)) else: # URL is a mandatory argument for Container.Update, use Container.Refresh instead @@ -529,7 +530,7 @@ def get_cache(key, ttl=None): with open_file(fullpath, 'r') as fdesc: try: - _LOGGER.info('Fetching {file} from cache', file=filename) + _LOGGER.debug('Fetching %s from cache', filename) import json value = json.load(fdesc) return value @@ -547,6 +548,21 @@ def set_cache(key, data): mkdirs(path) with open_file(fullpath, 'w') as fdesc: - _LOGGER.info('Storing to cache as {file}', file=filename) + _LOGGER.debug('Storing to cache as %s', filename) import json json.dump(data, fdesc) + + +def invalidate_cache(ttl=None): + """ Clear the cache """ + path = get_cache_path() + if not exists(path): + return + _, files = listdir(path) + import time + now = time.mktime(time.localtime()) + for filename in files: + fullpath = path + filename + if ttl and now - stat_file(fullpath).st_mtime() < ttl: + continue + delete(fullpath) diff --git a/resources/lib/modules/catalog.py b/resources/lib/modules/catalog.py index 1ec1835..13eb0d6 100644 --- a/resources/lib/modules/catalog.py +++ b/resources/lib/modules/catalog.py @@ -9,6 +9,7 @@ from resources.lib import kodiutils from resources.lib.kodiutils import TitleItem from resources.lib.modules.menu import Menu from resources.lib.viervijfzes import CHANNELS +from resources.lib.viervijfzes.auth import AuthApi from resources.lib.viervijfzes.content import ContentApi, UnavailableException _LOGGER = logging.getLogger('catalog') @@ -19,7 +20,8 @@ class Catalog: def __init__(self): """ Initialise object """ - self._api = ContentApi() + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + self._api = ContentApi(auth) self._menu = Menu() def show_catalog(self): @@ -34,9 +36,9 @@ class Catalog: listing = [self._menu.generate_titleitem(item) for item in items] - # Sort items by label, but don't put folders at the top. + # Sort items by title # Used for A-Z listing or when movies and episodes are mixed. - kodiutils.show_listing(listing, 30003, content='tvshows', sort='label') + kodiutils.show_listing(listing, 30003, content='tvshows', sort='title') def show_catalog_channel(self, channel): """ Show the programs of a specific channel @@ -52,9 +54,9 @@ class Catalog: for item in items: listing.append(self._menu.generate_titleitem(item)) - # Sort items by label, but don't put folders at the top. + # Sort items by title # Used for A-Z listing or when movies and episodes are mixed. - kodiutils.show_listing(listing, 30003, content='tvshows', sort='label') + kodiutils.show_listing(listing, 30003, content='tvshows', sort='title') def show_program(self, channel, program_id): """ Show a program from the catalog @@ -75,7 +77,7 @@ class Catalog: # Go directly to the season when we have only one season if len(program.seasons) == 1: - self.show_program_season(channel, program_id, program.seasons.values()[0].number) + self.show_program_season(channel, program_id, program.seasons.values()[0].uuid) return studio = CHANNELS.get(program.channel, {}).get('studio_icon') @@ -87,9 +89,8 @@ class Catalog: listing.append( TitleItem( title='* %s' % kodiutils.localize(30204), # * All seasons - path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=-1), + path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season='-1'), art_dict={ - 'thumb': program.cover, 'fanart': program.background, }, info_dict={ @@ -107,9 +108,8 @@ class Catalog: listing.append( TitleItem( title=s.title, # kodiutils.localize(30205, season=s.number), # Season {season} - path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=s.number), + path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=s.uuid), art_dict={ - 'thumb': s.cover, 'fanart': program.background, }, info_dict={ @@ -123,13 +123,13 @@ class Catalog: ) # Sort by label. Some programs return seasons unordered. - kodiutils.show_listing(listing, 30003, content='tvshows', sort=['label']) + kodiutils.show_listing(listing, 30003, content='tvshows') - def show_program_season(self, channel, program_id, season): + def show_program_season(self, channel, program_id, season_uuid): """ Show the episodes of a program from the catalog :type channel: str :type program_id: str - :type season: int + :type season_uuid: str """ try: program = self._api.get_program(channel, program_id) @@ -138,12 +138,12 @@ class Catalog: kodiutils.end_of_directory() return - if season == -1: + if season_uuid == "-1": # Show all episodes episodes = program.episodes else: # Show the episodes of the season that was selected - episodes = [e for e in program.episodes if e.season == season] + episodes = [e for e in program.episodes if e.season_uuid == season_uuid] listing = [self._menu.generate_titleitem(episode) for episode in episodes] diff --git a/resources/lib/modules/menu.py b/resources/lib/modules/menu.py index 10d7807..bb5bfac 100644 --- a/resources/lib/modules/menu.py +++ b/resources/lib/modules/menu.py @@ -86,7 +86,14 @@ class Menu: 'season': len(item.seasons) if item.seasons else None, }) - return TitleItem(title=item.title, + if isinstance(item.episodes, list) and not item.episodes: + # We know that we don't have episodes + title = '[COLOR gray]' + item.title + '[/COLOR]' + else: + # We have episodes, or we don't know it + title = item.title + + return TitleItem(title=title, path=kodiutils.url_for('show_catalog_program', channel=item.channel, program=item.path), art_dict=art_dict, info_dict=info_dict) @@ -113,7 +120,7 @@ class Menu: }) return TitleItem(title=info_dict['title'], - path=kodiutils.url_for('play', channel=item.channel, uuid=item.uuid), + path=kodiutils.url_for('play', uuid=item.uuid), art_dict=art_dict, info_dict=info_dict, stream_dict=stream_dict, diff --git a/resources/lib/modules/metadata.py b/resources/lib/modules/metadata.py new file mode 100644 index 0000000..5a938f9 --- /dev/null +++ b/resources/lib/modules/metadata.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" Metadata module """ + +from __future__ import absolute_import, division, unicode_literals + +from resources.lib import kodiutils +from resources.lib.viervijfzes import CHANNELS +from resources.lib.viervijfzes.content import ContentApi, Program + + +class Metadata: + """ Code responsible for the management of the local cached metadata """ + + def __init__(self): + """ Initialise object """ + self._api = ContentApi() + + def update(self): + """ Update the metadata with a foreground progress indicator """ + # Create progress indicator + progress = kodiutils.progress(message=kodiutils.localize(30715)) # Updating metadata + + def update_status(i, total): + """ Update the progress indicator """ + progress.update(int(((i + 1) / total) * 100), kodiutils.localize(30716, index=i + 1, total=total)) # Updating metadata ({index}/{total}) + return progress.iscanceled() + + self.fetch_metadata(callback=update_status) + + # Close progress indicator + progress.close() + + def fetch_metadata(self, callback=None): + """ Fetch the metadata for all the items in the catalog + :type callback: callable + """ + # Fetch all items from the catalog + items = [] + for channel in list(CHANNELS): + items.extend(self._api.get_programs(channel)) + count = len(items) + + # Loop over all of them and download the metadata + for index, item in enumerate(items): + if isinstance(item, Program): + self._api.get_program(item.channel, item.path) + + # Run callback after every item + if callback and callback(index, count): + # Stop when callback returns False + return False + + return True + + @staticmethod + def clean(): + """ Clear metadata (called from settings) """ + kodiutils.invalidate_cache() + kodiutils.set_setting('metadata_last_updated', '0') + kodiutils.ok_dialog(message=kodiutils.localize(30714)) # Local metadata is cleared diff --git a/resources/lib/modules/player.py b/resources/lib/modules/player.py index b36a463..9fd92f8 100644 --- a/resources/lib/modules/player.py +++ b/resources/lib/modules/player.py @@ -28,12 +28,11 @@ class Player: episode = ContentApi().get_episode(channel, path) # Play this now we have the uuid - self.play(channel, episode.uuid) + self.play(episode.uuid) @staticmethod - def play(channel, item): + def play(item): """ Play the requested item. - :type channel: string :type item: string """ try: @@ -48,17 +47,17 @@ class Player: # Fetch an auth token now try: - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) - token = auth.get_token() + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + + # Get stream information + resolved_stream = ContentApi(auth).get_stream_by_uuid(item) + except (InvalidLoginException, AuthenticationException) as ex: _LOGGER.error(ex) - kodiutils.ok_dialog(message=kodiutils.localize(30702, error=ex.message)) + kodiutils.ok_dialog(message=kodiutils.localize(30702, error=str(ex))) kodiutils.end_of_directory() return - # Get stream information - resolved_stream = ContentApi(token).get_stream(channel, item) - except GeoblockedException: kodiutils.ok_dialog(heading=kodiutils.localize(30709), message=kodiutils.localize(30710)) # This video is geo-blocked... return diff --git a/resources/lib/modules/tvguide.py b/resources/lib/modules/tvguide.py index f907c52..0d5a9e2 100644 --- a/resources/lib/modules/tvguide.py +++ b/resources/lib/modules/tvguide.py @@ -170,4 +170,4 @@ class TvGuide: return kodiutils.container_update( - kodiutils.url_for('play', channel=channel, uuid=broadcast.video_url)) + kodiutils.url_for('play', uuid=broadcast.video_url)) diff --git a/resources/lib/service.py b/resources/lib/service.py new file mode 100644 index 0000000..be37472 --- /dev/null +++ b/resources/lib/service.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" Background service code """ + +from __future__ import absolute_import, division, unicode_literals + +import hashlib +import logging +from time import time + +from xbmc import Monitor + +from resources.lib import kodilogging, kodiutils +from resources.lib.viervijfzes.auth import AuthApi + +kodilogging.config() +_LOGGER = logging.getLogger('service') + + +class BackgroundService(Monitor): + """ Background service code """ + + def __init__(self): + Monitor.__init__(self) + self.update_interval = 24 * 3600 # Every 24 hours + self.cache_expiry = 30 * 24 * 3600 # One month + + def run(self): + """ Background loop for maintenance tasks """ + _LOGGER.info('Service started') + + while not self.abortRequested(): + # Update every `update_interval` after the last update + if kodiutils.get_setting_bool('metadata_update') and int(kodiutils.get_setting('metadata_last_updated', 0)) + self.update_interval < time(): + self._update_metadata() + + # Stop when abort requested + if self.waitForAbort(10): + break + + _LOGGER.info('Service stopped') + + def onSettingsChanged(self): + """ Callback when a setting has changed """ + if self._has_credentials_changed(): + _LOGGER.info('Clearing auth tokens due to changed credentials') + AuthApi.clear_tokens() + + # Refresh container + kodiutils.container_refresh() + + @staticmethod + def _has_credentials_changed(): + """ Check if credentials have changed """ + old_hash = kodiutils.get_setting('credentials_hash') + new_hash = '' + if kodiutils.get_setting('username') or kodiutils.get_setting('password'): + new_hash = hashlib.md5((kodiutils.get_setting('username') + kodiutils.get_setting('password')).encode('utf-8')).hexdigest() + if new_hash != old_hash: + kodiutils.set_setting('credentials_hash', new_hash) + return True + return False + + def _update_metadata(self): + """ Update the metadata for the listings """ + from resources.lib.modules.metadata import Metadata + + # Clear outdated metadata + kodiutils.invalidate_cache(self.cache_expiry) + + def update_status(_i, _total): + """ Allow to cancel the background job """ + return self.abortRequested() or not kodiutils.get_setting_bool('metadata_update') + + success = Metadata().fetch_metadata(callback=update_status) + + # Update metadata_last_updated + if success: + kodiutils.set_setting('metadata_last_updated', str(int(time()))) + + +def run(): + """ Run the BackgroundService """ + BackgroundService().run() diff --git a/resources/lib/viervijfzes/auth.py b/resources/lib/viervijfzes/auth.py index bffa426..6526719 100644 --- a/resources/lib/viervijfzes/auth.py +++ b/resources/lib/viervijfzes/auth.py @@ -5,9 +5,9 @@ from __future__ import absolute_import, division, unicode_literals import json import logging -import os import time +from resources.lib import kodiutils from resources.lib.viervijfzes.auth_awsidp import AwsIdp, InvalidLoginException, AuthenticationException _LOGGER = logging.getLogger('auth-api') @@ -21,25 +21,24 @@ class AuthApi: TOKEN_FILE = 'auth-tokens.json' - def __init__(self, username, password, cache_dir=None): + def __init__(self, username, password): """ Initialise object """ self._username = username self._password = password - self._cache_dir = cache_dir + self._cache_dir = kodiutils.get_tokens_path() self._id_token = None self._expiry = 0 self._refresh_token = None - if self._cache_dir: - # Load tokens from cache - try: - with open(self._cache_dir + self.TOKEN_FILE, 'rb') as f: - data_json = json.loads(f.read()) - self._id_token = data_json.get('id_token') - self._refresh_token = data_json.get('refresh_token') - self._expiry = int(data_json.get('expiry', 0)) - except (IOError, TypeError, ValueError): - _LOGGER.info('We could not use the cache since it is invalid or non-existant.') + # Load tokens from cache + try: + with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'rb') as f: + data_json = json.loads(f.read()) + self._id_token = data_json.get('id_token') + self._refresh_token = data_json.get('refresh_token') + self._expiry = int(data_json.get('expiry', 0)) + except (IOError, TypeError, ValueError): + _LOGGER.info('We could not use the cache since it is invalid or non-existant.') def get_token(self): """ Get a valid token """ @@ -59,7 +58,7 @@ class AuthApi: self._expiry = now + 3600 _LOGGER.debug('Got an id token by refreshing: %s', self._id_token) except (InvalidLoginException, AuthenticationException) as e: - _LOGGER.error('Error logging in: %s', e.message) + _LOGGER.error('Error logging in: %s', str(e)) self._id_token = None self._refresh_token = None self._expiry = 0 @@ -74,33 +73,23 @@ class AuthApi: self._expiry = now + 3600 _LOGGER.debug('Got an id token by logging in: %s', self._id_token) - if self._cache_dir: - if not os.path.isdir(self._cache_dir): - os.mkdir(self._cache_dir) - - # Store new tokens in cache - with open(self._cache_dir + self.TOKEN_FILE, 'wb') as f: - data = json.dumps(dict( - id_token=self._id_token, - refresh_token=self._refresh_token, - expiry=self._expiry, - )) - f.write(data.encode('utf8')) + # Store new tokens in cache + if not kodiutils.exists(self._cache_dir): + kodiutils.mkdirs(self._cache_dir) + with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'wb') as f: + data = json.dumps(dict( + id_token=self._id_token, + refresh_token=self._refresh_token, + expiry=self._expiry, + )) + f.write(data.encode('utf8')) return self._id_token - def clear_cache(self): + @staticmethod + def clear_tokens(): """ Remove the cached tokens. """ - if not self._cache_dir: - return - - # Remove cache - os.remove(self._cache_dir + self.TOKEN_FILE) - - # Clear tokens in memory - self._id_token = None - self._refresh_token = None - self._expiry = 0 + kodiutils.delete(kodiutils.get_tokens_path() + AuthApi.TOKEN_FILE) @staticmethod def _authenticate(username, password): diff --git a/resources/lib/viervijfzes/auth_awsidp.py b/resources/lib/viervijfzes/auth_awsidp.py index 2208658..4883071 100644 --- a/resources/lib/viervijfzes/auth_awsidp.py +++ b/resources/lib/viervijfzes/auth_awsidp.py @@ -22,12 +22,10 @@ _LOGGER = logging.getLogger('auth-awsidp') class InvalidLoginException(Exception): """ The login credentials are invalid """ - pass class AuthenticationException(Exception): """ Something went wrong while logging in """ - pass class AwsIdp: @@ -90,7 +88,7 @@ class AwsIdp: "Content-Type": "application/x-amz-json-1.1" } auth_response = self._session.post(self.url, auth_data, headers=auth_headers) - auth_response_json = json.loads(auth_response.content) + auth_response_json = json.loads(auth_response.text) challenge_parameters = auth_response_json.get("ChallengeParameters") _LOGGER.debug(challenge_parameters) @@ -106,7 +104,7 @@ class AwsIdp: "Content-Type": "application/x-amz-json-1.1" } auth_response = self._session.post(self.url, challenge_data, headers=challenge_headers) - auth_response_json = json.loads(auth_response.content) + auth_response_json = json.loads(auth_response.text) _LOGGER.debug("Got response: %s", auth_response_json) if "message" in auth_response_json: @@ -138,7 +136,7 @@ class AwsIdp: } refresh_request_data = json.dumps(refresh_request) refresh_response = self._session.post(self.url, refresh_request_data, headers=refresh_headers) - refresh_json = json.loads(refresh_response.content) + refresh_json = json.loads(refresh_response.text) if "message" in refresh_json: raise AuthenticationException(refresh_json.get("message")) diff --git a/resources/lib/viervijfzes/content.py b/resources/lib/viervijfzes/content.py index 6a155aa..2ac2325 100644 --- a/resources/lib/viervijfzes/content.py +++ b/resources/lib/viervijfzes/content.py @@ -11,10 +11,15 @@ from datetime import datetime from six.moves.html_parser import HTMLParser import requests +from resources.lib import kodiutils from resources.lib.viervijfzes import CHANNELS _LOGGER = logging.getLogger('content-api') +CACHE_AUTO = 1 # Allow to use the cache, and query the API if no cache is available +CACHE_ONLY = 2 # Only use the cache, don't use the API +CACHE_PREVENT = 3 # Don't use the cache + class UnavailableException(Exception): """ Is thrown when an item is unavailable. """ @@ -89,8 +94,7 @@ class Episode: """ Defines an Episode. """ def __init__(self, uuid=None, nodeid=None, path=None, channel=None, program_title=None, title=None, description=None, cover=None, duration=None, - season=None, number=None, - rating=None, aired=None, expiry=None): + season=None, season_uuid=None, number=None, rating=None, aired=None, expiry=None): """ :type uuid: str :type nodeid: str @@ -102,6 +106,7 @@ class Episode: :type cover: str :type duration: int :type season: int + :type season_uuid: str :type number: int :type rating: str :type aired: datetime @@ -117,6 +122,7 @@ class Episode: self.cover = cover self.duration = duration self.season = season + self.season_uuid = season_uuid self.number = number self.rating = rating self.aired = aired @@ -129,30 +135,69 @@ class Episode: class ContentApi: """ VIER/VIJF/ZES Content API""" API_ENDPOINT = 'https://api.viervijfzes.be' + SITE_APIS = { + 'vier': 'https://www.vier.be/api', + 'vijf': 'https://www.vijf.be/api', + 'zes': 'https://www.zestv.be/api', + } - def __init__(self, token=None): + def __init__(self, auth=None): """ Initialise object """ self._session = requests.session() - self._token = token + self._auth = auth def get_notifications(self): """ Get a list of notifications for your account. :rtype list[dict] """ - response = self._get_url(self.API_ENDPOINT + '/notifications') + response = self._get_url(self.API_ENDPOINT + '/notifications', authentication=True) data = json.loads(response) return data - def get_stream(self, _channel, uuid): + def get_content_tree(self, channel): + """ Get a list of all the content. + :type channel: str + :rtype list[dict] + """ + if channel not in self.SITE_APIS: + raise Exception('Unknown channel %s' % channel) + + response = self._get_url(self.SITE_APIS[channel] + '/content_tree', authentication=True) + data = json.loads(response) + return data + + def get_stream_by_uuid(self, uuid): """ Get the stream URL to use for this video. - :type _channel: str :type uuid: str :rtype str """ - response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid) + response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True) data = json.loads(response) return data['video']['S'] + def get_programs_new(self, channel): + """ Get a list of all programs of the specified channel. + :type channel: str + :rtype list[Program] + """ + if channel not in CHANNELS: + raise Exception('Unknown channel %s' % channel) + + # Request all content from this channel + content_tree = self.get_content_tree(channel) + + programs = [] + for p in content_tree['programs']: + try: + program = self.get_program_by_uuid(p) + program.channel = channel + programs.append(program) + except UnavailableException: + # Some programs are not available, but do occur in the content tree + pass + + return programs + def get_programs(self, channel): """ Get a list of all programs of the specified channel. :type channel: str @@ -171,36 +216,90 @@ class ContentApi: r'\s+(?P[^<]+)</span>.*?' r'</a>', re.DOTALL) - programs = [ - Program(channel=channel, - path=program.group('path').lstrip('/'), - title=h.unescape(program.group('title').strip())) - for program in regex_programs.finditer(data) - ] + programs = [] + for item in regex_programs.finditer(data): + path = item.group('path').lstrip('/') + + program = self.get_program(channel, path, CACHE_ONLY) # Get program details, but from cache only + if program: + # Use program with metadata from cache + programs.append(program) + else: + # Use program with the values that we've parsed from the page + programs.append(Program(channel=channel, + path=path, + title=h.unescape(item.group('title').strip()))) return programs - def get_program(self, channel, path): + def get_program(self, channel, path, cache=CACHE_AUTO): """ Get a Program object from the specified page. :type channel: str :type path: str + :type cache: int :rtype Program NOTE: This function doesn't use an API. """ if channel not in CHANNELS: raise Exception('Unknown channel %s' % channel) - # Load webpage - page = self._get_url(CHANNELS[channel]['url'] + '/' + path) + if cache in [CACHE_AUTO, CACHE_ONLY]: + # Try to fetch from cache + data = kodiutils.get_cache(['program', channel, path]) + if data is None and cache == CACHE_ONLY: + return None + else: + data = None + + if data is None: + # Fetch webpage + page = self._get_url(CHANNELS[channel]['url'] + '/' + path) + + # Extract JSON + regex_program = re.compile(r'data-hero="([^"]+)', re.DOTALL) + json_data = HTMLParser().unescape(regex_program.search(page).group(1)) + data = json.loads(json_data)['data'] + + # Store response in cache + kodiutils.set_cache(['program', channel, path], data) - # Extract JSON - regex_program = re.compile(r'data-hero="([^"]+)', re.DOTALL) - json_data = HTMLParser().unescape(regex_program.search(page).group(1)) - data = json.loads(json_data)['data'] program = self._parse_program_data(data) return program + def get_program_by_uuid(self, uuid, cache=CACHE_AUTO): + """ Get a Program object. + :type uuid: str + :type cache: int + :rtype Program + """ + if cache in [CACHE_AUTO, CACHE_ONLY]: + # Try to fetch from cache + data = kodiutils.get_cache(['program', uuid]) + if data is None and cache == CACHE_ONLY: + return None + else: + data = None + + if data is None: + # Fetch from API + response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True) + data = json.loads(response) + + if not data: + raise UnavailableException() + + # Store response in cache + kodiutils.set_cache(['program', uuid], data) + + return Program( + uuid=uuid, + path=data['url']['S'].strip('/'), + title=data['label']['S'], + description=data['description']['S'], + cover=data['image']['S'], + ) + def get_episode(self, channel, path): """ Get a Episode object from the specified page. :type channel: str @@ -254,7 +353,7 @@ class ContentApi: # Create Season info program.seasons = { - playlist['episodes'][0]['seasonNumber']: Season( + key: Season( uuid=playlist['id'], path=playlist['link'].lstrip('/'), channel=playlist['pageInfo']['site'], @@ -262,12 +361,12 @@ class ContentApi: description=playlist['pageInfo']['description'], number=playlist['episodes'][0]['seasonNumber'], # You did not see this ) - for playlist in data['playlists'] + for key, playlist in enumerate(data['playlists']) } # Create Episodes info program.episodes = [ - ContentApi._parse_episode_data(episode) + ContentApi._parse_episode_data(episode, playlist['id']) for playlist in data['playlists'] for episode in playlist['episodes'] ] @@ -275,9 +374,10 @@ class ContentApi: return program @staticmethod - def _parse_episode_data(data): + def _parse_episode_data(data, season_uuid): """ Parse the Episode JSON. :type data: dict + :type season_uuid: str :rtype Episode """ @@ -291,22 +391,18 @@ class ContentApi: else: episode_number = None - if data.get('episodeTitle'): - episode_title = data.get('episodeTitle') - else: - episode_title = data.get('title') - episode = Episode( uuid=data.get('videoUuid'), nodeid=data.get('pageInfo', {}).get('nodeId'), path=data.get('link').lstrip('/'), channel=data.get('pageInfo', {}).get('site'), program_title=data.get('program', {}).get('title'), - title=episode_title, + title=data.get('title'), description=data.get('pageInfo', {}).get('description'), cover=data.get('image'), duration=data.get('duration'), season=data.get('seasonNumber'), + season_uuid=season_uuid, number=episode_number, aired=datetime.fromtimestamp(data.get('createdDate')), expiry=datetime.fromtimestamp(data.get('unpublishDate')) if data.get('unpublishDate') else None, @@ -314,19 +410,22 @@ class ContentApi: ) return episode - def _get_url(self, url, params=None): + def _get_url(self, url, params=None, authentication=False): """ Makes a GET request for the specified URL. :type url: str :rtype str """ - if self._token: + if authentication: + if not self._auth: + raise Exception('Requested to authenticate, but not auth object passed') response = self._session.get(url, params=params, headers={ - 'authorization': self._token, + 'authorization': self._auth.get_token(), }) else: response = self._session.get(url, params=params) if response.status_code != 200: + _LOGGER.error(response.text) raise Exception('Could not fetch data') return response.text diff --git a/resources/lib/viervijfzes/search.py b/resources/lib/viervijfzes/search.py index 4886be1..93e8a31 100644 --- a/resources/lib/viervijfzes/search.py +++ b/resources/lib/viervijfzes/search.py @@ -42,7 +42,7 @@ class SearchApi: if response.status_code != 200: raise Exception('Could not search') - data = json.loads(response.content) + data = json.loads(response.text) results = [] for hit in data['hits']['hits']: diff --git a/resources/settings.xml b/resources/settings.xml index ed67830..3fbdb78 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -1,8 +1,15 @@ <?xml version="1.0" encoding="utf-8" standalone="yes"?> <settings> + <setting id="metadata_last_updated" visible="false"/> <category label="30800"> <!-- Credentials --> <setting label="30801" type="lsep"/> <!-- Credentials --> <setting label="30803" type="text" id="username"/> <setting label="30805" type="text" id="password" option="hidden"/> </category> + <category label="30820"> <!-- Interface --> + <setting label="30827" type="lsep"/> <!-- Metadata --> + <setting label="30829" type="bool" id="metadata_update" default="true" subsetting="true"/> + <setting label="30831" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/update)"/> + <setting label="30833" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/clean)"/> + </category> </settings> diff --git a/service_entry.py b/service_entry.py new file mode 100644 index 0000000..3fc2869 --- /dev/null +++ b/service_entry.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" Service entry point """ + +from __future__ import absolute_import, division, unicode_literals + +from resources.lib import service + +service.run() diff --git a/test/test_api.py b/test/test_api.py index 40f8e9c..355c3ac 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -9,8 +9,8 @@ import logging import unittest import resources.lib.kodiutils as kodiutils -from resources.lib.viervijfzes.content import ContentApi, Program, Episode from resources.lib.viervijfzes.auth import AuthApi +from resources.lib.viervijfzes.content import ContentApi, Program, Episode _LOGGER = logging.getLogger('test-api') @@ -18,7 +18,8 @@ _LOGGER = logging.getLogger('test-api') class TestApi(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestApi, self).__init__(*args, **kwargs) - self._api = ContentApi() + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + self._api = ContentApi(auth) def test_programs(self): for channel in ['vier', 'vijf', 'zes']: @@ -39,9 +40,7 @@ class TestApi(unittest.TestCase): program = self._api.get_program('vier', 'auwch') episode = program.episodes[0] - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) - api_authed = ContentApi(auth.get_token()) - video = api_authed.get_stream(episode.channel, episode.uuid) + video = self._api.get_stream_by_uuid(episode.uuid) self.assertTrue(video) _LOGGER.info('Got video URL: %s', video) diff --git a/test/test_auth.py b/test/test_auth.py index f84d973..9109bd6 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -17,18 +17,19 @@ _LOGGER = logging.getLogger('test-auth') class TestAuth(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestAuth, self).__init__(*args, **kwargs) - self._auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) def test_login(self): # Clear any cache we have - self._auth.clear_cache() + AuthApi.clear_tokens() # We should get a token by logging in - token = self._auth.get_token() + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + token = auth.get_token() self.assertTrue(token) # Test it a second time, it should go from memory now - token = self._auth.get_token() + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + token = auth.get_token() self.assertTrue(token) diff --git a/test/test_epg.py b/test/test_epg.py index ce1df2f..569499f 100644 --- a/test/test_epg.py +++ b/test/test_epg.py @@ -49,15 +49,15 @@ class TestEpg(unittest.TestCase): epg_programs = self._epg.get_epg('vier', date.today().strftime('%Y-%m-%d')) epg_program = [program for program in epg_programs if program.video_url][0] + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + api = ContentApi(auth) + # Lookup the Episode data since we don't have an UUID - api = ContentApi() episode = api.get_episode(epg_program.channel, epg_program.video_url) self.assertIsInstance(episode, Episode) # Get stream based on the Episode's UUID - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) - api = ContentApi(auth.get_token()) - video = api.get_stream(episode.channel, episode.uuid) + video = api.get_stream_by_uuid(episode.uuid) self.assertTrue(video)