Implement program caching for more metadata in listings (#8)

Makefile

@@ -7,7 +7,7 @@ version = $(shell xmllint --xpath 'string(/addon/@version)' addon.xml)
 git_branch = $(shell git rev-parse --abbrev-ref HEAD)
 git_hash = $(shell git rev-parse --short HEAD)
 zip_name = $(name)-$(version)-$(git_branch)-$(git_hash).zip
-include_files = addon_entry.py addon.xml CHANGELOG.md LICENSE README.md resources/
+include_files = addon_entry.py addon.xml CHANGELOG.md LICENSE README.md service_entry.py resources/
 include_paths = $(patsubst %,$(name)/%,$(include_files))
 exclude_files = \*.new \*.orig \*.pyc \*.pyo
 

addon.xml

@@ -10,6 +10,7 @@
     <extension point="xbmc.python.pluginsource" library="addon_entry.py">
         <provides>video</provides>
     </extension>
+    <extension point="xbmc.service" library="service_entry.py"/>
     <extension point="xbmc.addon.metadata">
         <summary lang="en_GB">Watch content from VIER, VIJF and ZES.</summary>
         <platform>all</platform>

resources/language/resource.language.en_gb/strings.po

@@ -120,6 +120,18 @@ msgctxt "#30713"
 msgid "The requested video was not found in the guide."
 msgstr ""
 
+msgctxt "#30714"
+msgid "Local metadata is cleared."
+msgstr ""
+
+msgctxt "#30715"
+msgid "Updating metadata"
+msgstr ""
+
+msgctxt "#30716"
+msgid "Updating metadata ({index}/{total})..."
+msgstr ""
+
 msgctxt "#30717"
 msgid "This program is not available in the catalogue."
 msgstr ""
@@ -141,3 +153,24 @@ msgstr ""
 msgctxt "#30805"
 msgid "Password"
 msgstr ""
+
+msgctxt "#30820"
+msgid "Interface"
+msgstr ""
+
+msgctxt "#30827"
+msgid "Metadata"
+msgstr ""
+
+msgctxt "#30829"
+msgid "Periodically refresh metadata in the background"
+msgstr ""
+
+msgctxt "#30831"
+msgid "Update local metadata now"
+msgstr ""
+
+msgctxt "#30833"
+msgid "Clear local metadata"
+msgstr ""
+

resources/language/resource.language.nl_nl/strings.po

@@ -121,6 +121,18 @@ msgctxt "#30713"
 msgid "The requested video was not found in the guide."
 msgstr "De gevraagde video werd niet gevonden in de tv-gids."
 
+msgctxt "#30714"
+msgid "Local metadata is cleared."
+msgstr "De locale metadata is verwijderd."
+
+msgctxt "#30715"
+msgid "Updating metadata"
+msgstr "Vernieuwen metadata"
+
+msgctxt "#30716"
+msgid "Updating metadata ({index}/{total})..."
+msgstr "Vernieuwen metadata ({index}/{total})..."
+
 msgctxt "#30717"
 msgid "This program is not available in the catalogue."
 msgstr "Dit programma is niet beschikbaar in de catalogus."
@@ -142,3 +154,24 @@ msgstr "E-mailadres"
 msgctxt "#30805"
 msgid "Password"
 msgstr "Wachtwoord"
+
+msgctxt "#30820"
+msgid "Interface"
+msgstr "Interface"
+
+msgctxt "#30827"
+msgid "Metadata"
+msgstr "Metadata"
+
+msgctxt "#30829"
+msgid "Periodically refresh metadata in the background"
+msgstr "Vernieuw de metdata automatisch in de achtergrond"
+
+msgctxt "#30831"
+msgid "Update local metadata now"
+msgstr "De locale metadata nu vernieuwen"
+
+msgctxt "#30833"
+msgid "Clear local metadata"
+msgstr "De locale metadata verwijderen"
+

resources/lib/addon.py

@@ -3,12 +3,15 @@
 
 from __future__ import absolute_import, division, unicode_literals
 
+import logging
+
 from routing import Plugin
 
 from resources.lib import kodilogging
 
 kodilogging.config()
 routing = Plugin()
+_LOGGER = logging.getLogger('addon')
 
 
 @routing.route('/')
@@ -67,11 +70,11 @@ def show_catalog_program(channel, program):
     Catalog().show_program(channel, program)
 
 
-@routing.route('/program/program/<channel>/<program>/<season>')
+@routing.route('/catalog/program/<channel>/<program>/<season>')
 def show_catalog_program_season(channel, program, season):
     """ Show a program from the catalog """
     from resources.lib.modules.catalog import Catalog
-    Catalog().show_program_season(channel, program, int(season))
+    Catalog().show_program_season(channel, program, season)
 
 
 @routing.route('/search')
@@ -82,11 +85,11 @@ def show_search(query=None):
     Search().show_search(query)
 
 
-@routing.route('/play/catalog/<channel>/<uuid>')
+@routing.route('/play/catalog/<uuid>')
-def play(channel, uuid):
+def play(uuid):
     """ Play the requested item """
     from resources.lib.modules.player import Player
-    Player().play(channel, uuid)
+    Player().play(uuid)
 
 
 @routing.route('/play/page/<channel>/<page>')
@@ -101,6 +104,20 @@ def play_from_page(channel, page):
     Player().play_from_page(channel, unquote(page))
 
 
+@routing.route('/metadata/update')
+def metadata_update():
+    """ Update the metadata for the listings (called from settings) """
+    from resources.lib.modules.metadata import Metadata
+    Metadata().update()
+
+
+@routing.route('/metadata/clean')
+def metadata_clean():
+    """ Clear metadata (called from settings) """
+    from resources.lib.modules.metadata import Metadata
+    Metadata().clean()
+
+
 def run(params):
     """ Run the routing plugin """
     routing.run(params)

resources/lib/kodiutils.py

@@ -16,13 +16,14 @@ ADDON = xbmcaddon.Addon()
 SORT_METHODS = dict(
     unsorted=xbmcplugin.SORT_METHOD_UNSORTED,
     label=xbmcplugin.SORT_METHOD_LABEL_IGNORE_FOLDERS,
+    title=xbmcplugin.SORT_METHOD_TITLE,
     episode=xbmcplugin.SORT_METHOD_EPISODE,
     duration=xbmcplugin.SORT_METHOD_DURATION,
     year=xbmcplugin.SORT_METHOD_VIDEO_YEAR,
     date=xbmcplugin.SORT_METHOD_DATE,
 )
 DEFAULT_SORT_METHODS = [
-    'unsorted', 'label'
+    'unsorted', 'title'
 ]
 
 _LOGGER = logging.getLogger('kodiutils')
@@ -269,7 +270,7 @@ def set_locale():
         setlocale(LC_ALL, locale_lang)
     except (Error, ValueError) as exc:
         if locale_lang != 'en_GB':
-            _LOGGER.debug("Your system does not support locale '{locale}': {error}", locale=locale_lang, error=exc)
+            _LOGGER.debug("Your system does not support locale '%s': %s", locale_lang, exc)
             set_locale.cached = False
             return False
     set_locale.cached = True
@@ -423,14 +424,14 @@ def listdir(path):
 def mkdir(path):
     """Create a directory (using xbmcvfs)"""
     from xbmcvfs import mkdir as vfsmkdir
-    _LOGGER.debug("Create directory '{path}'.", path=path)
+    _LOGGER.debug("Create directory '%s'.", path)
     return vfsmkdir(path)
 
 
 def mkdirs(path):
     """Create directory including parents (using xbmcvfs)"""
     from xbmcvfs import mkdirs as vfsmkdirs
-    _LOGGER.debug("Recursively create directory '{path}'.", path=path)
+    _LOGGER.debug("Recursively create directory '%s'.", path)
     return vfsmkdirs(path)
 
 
@@ -458,14 +459,14 @@ def stat_file(path):
 def delete(path):
     """Remove a file (using xbmcvfs)"""
     from xbmcvfs import delete as vfsdelete
-    _LOGGER.debug("Delete file '{path}'.", path=path)
+    _LOGGER.debug("Delete file '%s'.", path)
     return vfsdelete(path)
 
 
 def container_refresh(url=None):
     """Refresh the current container or (re)load a container by URL"""
     if url:
-        _LOGGER.debug('Execute: Container.Refresh({url})', url=url)
+        _LOGGER.debug('Execute: Container.Refresh(%s)', url)
         xbmc.executebuiltin('Container.Refresh({url})'.format(url=url))
     else:
         _LOGGER.debug('Execute: Container.Refresh')
@@ -475,7 +476,7 @@ def container_refresh(url=None):
 def container_update(url):
     """Update the current container while respecting the path history."""
     if url:
-        _LOGGER.debug('Execute: Container.Update({url})', url=url)
+        _LOGGER.debug('Execute: Container.Update(%s)', url)
         xbmc.executebuiltin('Container.Update({url})'.format(url=url))
     else:
         # URL is a mandatory argument for Container.Update, use Container.Refresh instead
@@ -529,7 +530,7 @@ def get_cache(key, ttl=None):
 
     with open_file(fullpath, 'r') as fdesc:
         try:
-            _LOGGER.info('Fetching {file} from cache', file=filename)
+            _LOGGER.debug('Fetching %s from cache', filename)
             import json
             value = json.load(fdesc)
             return value
@@ -547,6 +548,21 @@ def set_cache(key, data):
         mkdirs(path)
 
     with open_file(fullpath, 'w') as fdesc:
-        _LOGGER.info('Storing to cache as {file}', file=filename)
+        _LOGGER.debug('Storing to cache as %s', filename)
         import json
         json.dump(data, fdesc)
+
+
+def invalidate_cache(ttl=None):
+    """ Clear the cache """
+    path = get_cache_path()
+    if not exists(path):
+        return
+    _, files = listdir(path)
+    import time
+    now = time.mktime(time.localtime())
+    for filename in files:
+        fullpath = path + filename
+        if ttl and now - stat_file(fullpath).st_mtime() < ttl:
+            continue
+        delete(fullpath)

resources/lib/modules/catalog.py

@@ -9,6 +9,7 @@ from resources.lib import kodiutils
 from resources.lib.kodiutils import TitleItem
 from resources.lib.modules.menu import Menu
 from resources.lib.viervijfzes import CHANNELS
+from resources.lib.viervijfzes.auth import AuthApi
 from resources.lib.viervijfzes.content import ContentApi, UnavailableException
 
 _LOGGER = logging.getLogger('catalog')
@@ -19,7 +20,8 @@ class Catalog:
 
     def __init__(self):
         """ Initialise object """
-        self._api = ContentApi()
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        self._api = ContentApi(auth)
         self._menu = Menu()
 
     def show_catalog(self):
@@ -34,9 +36,9 @@ class Catalog:
 
         listing = [self._menu.generate_titleitem(item) for item in items]
 
-        # Sort items by label, but don't put folders at the top.
+        # Sort items by title
         # Used for A-Z listing or when movies and episodes are mixed.
-        kodiutils.show_listing(listing, 30003, content='tvshows', sort='label')
+        kodiutils.show_listing(listing, 30003, content='tvshows', sort='title')
 
     def show_catalog_channel(self, channel):
         """ Show the programs of a specific channel
@@ -52,9 +54,9 @@ class Catalog:
         for item in items:
             listing.append(self._menu.generate_titleitem(item))
 
-        # Sort items by label, but don't put folders at the top.
+        # Sort items by title
         # Used for A-Z listing or when movies and episodes are mixed.
-        kodiutils.show_listing(listing, 30003, content='tvshows', sort='label')
+        kodiutils.show_listing(listing, 30003, content='tvshows', sort='title')
 
     def show_program(self, channel, program_id):
         """ Show a program from the catalog
@@ -75,7 +77,7 @@ class Catalog:
 
         # Go directly to the season when we have only one season
         if len(program.seasons) == 1:
-            self.show_program_season(channel, program_id, program.seasons.values()[0].number)
+            self.show_program_season(channel, program_id, program.seasons.values()[0].uuid)
             return
 
         studio = CHANNELS.get(program.channel, {}).get('studio_icon')
@@ -87,9 +89,8 @@ class Catalog:
             listing.append(
                 TitleItem(
                     title='* %s' % kodiutils.localize(30204),  # * All seasons
-                    path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=-1),
+                    path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season='-1'),
                     art_dict={
-                        'thumb': program.cover,
                         'fanart': program.background,
                     },
                     info_dict={
@@ -107,9 +108,8 @@ class Catalog:
             listing.append(
                 TitleItem(
                     title=s.title,  # kodiutils.localize(30205, season=s.number),  # Season {season}
-                    path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=s.number),
+                    path=kodiutils.url_for('show_catalog_program_season', channel=channel, program=program_id, season=s.uuid),
                     art_dict={
-                        'thumb': s.cover,
                         'fanart': program.background,
                     },
                     info_dict={
@@ -123,13 +123,13 @@ class Catalog:
             )
 
         # Sort by label. Some programs return seasons unordered.
-        kodiutils.show_listing(listing, 30003, content='tvshows', sort=['label'])
+        kodiutils.show_listing(listing, 30003, content='tvshows')
 
-    def show_program_season(self, channel, program_id, season):
+    def show_program_season(self, channel, program_id, season_uuid):
         """ Show the episodes of a program from the catalog
         :type channel: str
         :type program_id: str
-        :type season: int
+        :type season_uuid: str
         """
         try:
             program = self._api.get_program(channel, program_id)
@@ -138,12 +138,12 @@ class Catalog:
             kodiutils.end_of_directory()
             return
 
-        if season == -1:
+        if season_uuid == "-1":
             # Show all episodes
             episodes = program.episodes
         else:
             # Show the episodes of the season that was selected
-            episodes = [e for e in program.episodes if e.season == season]
+            episodes = [e for e in program.episodes if e.season_uuid == season_uuid]
 
         listing = [self._menu.generate_titleitem(episode) for episode in episodes]
 

resources/lib/modules/menu.py

@@ -86,7 +86,14 @@ class Menu:
                 'season': len(item.seasons) if item.seasons else None,
             })
 
-            return TitleItem(title=item.title,
+            if isinstance(item.episodes, list) and not item.episodes:
+                # We know that we don't have episodes
+                title = '[COLOR gray]' + item.title + '[/COLOR]'
+            else:
+                # We have episodes, or we don't know it
+                title = item.title
+
+            return TitleItem(title=title,
                              path=kodiutils.url_for('show_catalog_program', channel=item.channel, program=item.path),
                              art_dict=art_dict,
                              info_dict=info_dict)
@@ -113,7 +120,7 @@ class Menu:
             })
 
             return TitleItem(title=info_dict['title'],
-                             path=kodiutils.url_for('play', channel=item.channel, uuid=item.uuid),
+                             path=kodiutils.url_for('play', uuid=item.uuid),
                              art_dict=art_dict,
                              info_dict=info_dict,
                              stream_dict=stream_dict,

resources/lib/modules/metadata.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+""" Metadata module """
+
+from __future__ import absolute_import, division, unicode_literals
+
+from resources.lib import kodiutils
+from resources.lib.viervijfzes import CHANNELS
+from resources.lib.viervijfzes.content import ContentApi, Program
+
+
+class Metadata:
+    """ Code responsible for the management of the local cached metadata """
+
+    def __init__(self):
+        """ Initialise object """
+        self._api = ContentApi()
+
+    def update(self):
+        """ Update the metadata with a foreground progress indicator """
+        # Create progress indicator
+        progress = kodiutils.progress(message=kodiutils.localize(30715))  # Updating metadata
+
+        def update_status(i, total):
+            """ Update the progress indicator """
+            progress.update(int(((i + 1) / total) * 100), kodiutils.localize(30716, index=i + 1, total=total))  # Updating metadata ({index}/{total})
+            return progress.iscanceled()
+
+        self.fetch_metadata(callback=update_status)
+
+        # Close progress indicator
+        progress.close()
+
+    def fetch_metadata(self, callback=None):
+        """ Fetch the metadata for all the items in the catalog
+        :type callback: callable
+        """
+        # Fetch all items from the catalog
+        items = []
+        for channel in list(CHANNELS):
+            items.extend(self._api.get_programs(channel))
+        count = len(items)
+
+        # Loop over all of them and download the metadata
+        for index, item in enumerate(items):
+            if isinstance(item, Program):
+                self._api.get_program(item.channel, item.path)
+
+            # Run callback after every item
+            if callback and callback(index, count):
+                # Stop when callback returns False
+                return False
+
+        return True
+
+    @staticmethod
+    def clean():
+        """ Clear metadata (called from settings) """
+        kodiutils.invalidate_cache()
+        kodiutils.set_setting('metadata_last_updated', '0')
+        kodiutils.ok_dialog(message=kodiutils.localize(30714))  # Local metadata is cleared

resources/lib/modules/player.py

@@ -28,12 +28,11 @@ class Player:
         episode = ContentApi().get_episode(channel, path)
 
         # Play this now we have the uuid
-        self.play(channel, episode.uuid)
+        self.play(episode.uuid)
 
     @staticmethod
-    def play(channel, item):
+    def play(item):
         """ Play the requested item.
-        :type channel: string
         :type item: string
         """
         try:
@@ -48,17 +47,17 @@ class Player:
 
             # Fetch an auth token now
             try:
-                auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
+                auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
-                token = auth.get_token()
+
+                # Get stream information
+                resolved_stream = ContentApi(auth).get_stream_by_uuid(item)
+
             except (InvalidLoginException, AuthenticationException) as ex:
                 _LOGGER.error(ex)
-                kodiutils.ok_dialog(message=kodiutils.localize(30702, error=ex.message))
+                kodiutils.ok_dialog(message=kodiutils.localize(30702, error=str(ex)))
                 kodiutils.end_of_directory()
                 return
 
-            # Get stream information
-            resolved_stream = ContentApi(token).get_stream(channel, item)
-
         except GeoblockedException:
             kodiutils.ok_dialog(heading=kodiutils.localize(30709), message=kodiutils.localize(30710))  # This video is geo-blocked...
             return

resources/lib/modules/tvguide.py

@@ -170,4 +170,4 @@ class TvGuide:
             return
 
         kodiutils.container_update(
-            kodiutils.url_for('play', channel=channel, uuid=broadcast.video_url))
+            kodiutils.url_for('play', uuid=broadcast.video_url))

resources/lib/service.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+""" Background service code """
+
+from __future__ import absolute_import, division, unicode_literals
+
+import hashlib
+import logging
+from time import time
+
+from xbmc import Monitor
+
+from resources.lib import kodilogging, kodiutils
+from resources.lib.viervijfzes.auth import AuthApi
+
+kodilogging.config()
+_LOGGER = logging.getLogger('service')
+
+
+class BackgroundService(Monitor):
+    """ Background service code """
+
+    def __init__(self):
+        Monitor.__init__(self)
+        self.update_interval = 24 * 3600  # Every 24 hours
+        self.cache_expiry = 30 * 24 * 3600  # One month
+
+    def run(self):
+        """ Background loop for maintenance tasks """
+        _LOGGER.info('Service started')
+
+        while not self.abortRequested():
+            # Update every `update_interval` after the last update
+            if kodiutils.get_setting_bool('metadata_update') and int(kodiutils.get_setting('metadata_last_updated', 0)) + self.update_interval < time():
+                self._update_metadata()
+
+            # Stop when abort requested
+            if self.waitForAbort(10):
+                break
+
+        _LOGGER.info('Service stopped')
+
+    def onSettingsChanged(self):
+        """ Callback when a setting has changed """
+        if self._has_credentials_changed():
+            _LOGGER.info('Clearing auth tokens due to changed credentials')
+            AuthApi.clear_tokens()
+
+            # Refresh container
+            kodiutils.container_refresh()
+
+    @staticmethod
+    def _has_credentials_changed():
+        """ Check if credentials have changed """
+        old_hash = kodiutils.get_setting('credentials_hash')
+        new_hash = ''
+        if kodiutils.get_setting('username') or kodiutils.get_setting('password'):
+            new_hash = hashlib.md5((kodiutils.get_setting('username') + kodiutils.get_setting('password')).encode('utf-8')).hexdigest()
+        if new_hash != old_hash:
+            kodiutils.set_setting('credentials_hash', new_hash)
+            return True
+        return False
+
+    def _update_metadata(self):
+        """ Update the metadata for the listings """
+        from resources.lib.modules.metadata import Metadata
+
+        # Clear outdated metadata
+        kodiutils.invalidate_cache(self.cache_expiry)
+
+        def update_status(_i, _total):
+            """ Allow to cancel the background job """
+            return self.abortRequested() or not kodiutils.get_setting_bool('metadata_update')
+
+        success = Metadata().fetch_metadata(callback=update_status)
+
+        # Update metadata_last_updated
+        if success:
+            kodiutils.set_setting('metadata_last_updated', str(int(time())))
+
+
+def run():
+    """ Run the BackgroundService """
+    BackgroundService().run()

resources/lib/viervijfzes/auth.py

@@ -5,9 +5,9 @@ from __future__ import absolute_import, division, unicode_literals
 
 import json
 import logging
-import os
 import time
 
+from resources.lib import kodiutils
 from resources.lib.viervijfzes.auth_awsidp import AwsIdp, InvalidLoginException, AuthenticationException
 
 _LOGGER = logging.getLogger('auth-api')
@@ -21,25 +21,24 @@ class AuthApi:
 
     TOKEN_FILE = 'auth-tokens.json'
 
-    def __init__(self, username, password, cache_dir=None):
+    def __init__(self, username, password):
         """ Initialise object """
         self._username = username
         self._password = password
-        self._cache_dir = cache_dir
+        self._cache_dir = kodiutils.get_tokens_path()
         self._id_token = None
         self._expiry = 0
         self._refresh_token = None
 
-        if self._cache_dir:
+        # Load tokens from cache
-            # Load tokens from cache
+        try:
-            try:
+            with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'rb') as f:
-                with open(self._cache_dir + self.TOKEN_FILE, 'rb') as f:
+                data_json = json.loads(f.read())
-                    data_json = json.loads(f.read())
+                self._id_token = data_json.get('id_token')
-                    self._id_token = data_json.get('id_token')
+                self._refresh_token = data_json.get('refresh_token')
-                    self._refresh_token = data_json.get('refresh_token')
+                self._expiry = int(data_json.get('expiry', 0))
-                    self._expiry = int(data_json.get('expiry', 0))
+        except (IOError, TypeError, ValueError):
-            except (IOError, TypeError, ValueError):
+            _LOGGER.info('We could not use the cache since it is invalid or non-existant.')
-                _LOGGER.info('We could not use the cache since it is invalid or non-existant.')
 
     def get_token(self):
         """ Get a valid token """
@@ -59,7 +58,7 @@ class AuthApi:
                 self._expiry = now + 3600
                 _LOGGER.debug('Got an id token by refreshing: %s', self._id_token)
             except (InvalidLoginException, AuthenticationException) as e:
-                _LOGGER.error('Error logging in: %s', e.message)
+                _LOGGER.error('Error logging in: %s', str(e))
                 self._id_token = None
                 self._refresh_token = None
                 self._expiry = 0
@@ -74,33 +73,23 @@ class AuthApi:
             self._expiry = now + 3600
             _LOGGER.debug('Got an id token by logging in: %s', self._id_token)
 
-        if self._cache_dir:
+        # Store new tokens in cache
-            if not os.path.isdir(self._cache_dir):
+        if not kodiutils.exists(self._cache_dir):
-                os.mkdir(self._cache_dir)
+            kodiutils.mkdirs(self._cache_dir)
-
+        with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'wb') as f:
-            # Store new tokens in cache
+            data = json.dumps(dict(
-            with open(self._cache_dir + self.TOKEN_FILE, 'wb') as f:
+                id_token=self._id_token,
-                data = json.dumps(dict(
+                refresh_token=self._refresh_token,
-                    id_token=self._id_token,
+                expiry=self._expiry,
-                    refresh_token=self._refresh_token,
+            ))
-                    expiry=self._expiry,
+            f.write(data.encode('utf8'))
-                ))
-                f.write(data.encode('utf8'))
 
         return self._id_token
 
-    def clear_cache(self):
+    @staticmethod
+    def clear_tokens():
         """ Remove the cached tokens. """
-        if not self._cache_dir:
+        kodiutils.delete(kodiutils.get_tokens_path() + AuthApi.TOKEN_FILE)
-            return
-
-        # Remove cache
-        os.remove(self._cache_dir + self.TOKEN_FILE)
-
-        # Clear tokens in memory
-        self._id_token = None
-        self._refresh_token = None
-        self._expiry = 0
 
     @staticmethod
     def _authenticate(username, password):

resources/lib/viervijfzes/auth_awsidp.py

@@ -22,12 +22,10 @@ _LOGGER = logging.getLogger('auth-awsidp')
 
 class InvalidLoginException(Exception):
     """ The login credentials are invalid """
-    pass
 
 
 class AuthenticationException(Exception):
     """ Something went wrong while logging in """
-    pass
 
 
 class AwsIdp:
@@ -90,7 +88,7 @@ class AwsIdp:
             "Content-Type": "application/x-amz-json-1.1"
         }
         auth_response = self._session.post(self.url, auth_data, headers=auth_headers)
-        auth_response_json = json.loads(auth_response.content)
+        auth_response_json = json.loads(auth_response.text)
         challenge_parameters = auth_response_json.get("ChallengeParameters")
         _LOGGER.debug(challenge_parameters)
 
@@ -106,7 +104,7 @@ class AwsIdp:
             "Content-Type": "application/x-amz-json-1.1"
         }
         auth_response = self._session.post(self.url, challenge_data, headers=challenge_headers)
-        auth_response_json = json.loads(auth_response.content)
+        auth_response_json = json.loads(auth_response.text)
         _LOGGER.debug("Got response: %s", auth_response_json)
 
         if "message" in auth_response_json:
@@ -138,7 +136,7 @@ class AwsIdp:
         }
         refresh_request_data = json.dumps(refresh_request)
         refresh_response = self._session.post(self.url, refresh_request_data, headers=refresh_headers)
-        refresh_json = json.loads(refresh_response.content)
+        refresh_json = json.loads(refresh_response.text)
 
         if "message" in refresh_json:
             raise AuthenticationException(refresh_json.get("message"))

resources/lib/viervijfzes/content.py

@@ -11,10 +11,15 @@ from datetime import datetime
 from six.moves.html_parser import HTMLParser
 import requests
 
+from resources.lib import kodiutils
 from resources.lib.viervijfzes import CHANNELS
 
 _LOGGER = logging.getLogger('content-api')
 
+CACHE_AUTO = 1  # Allow to use the cache, and query the API if no cache is available
+CACHE_ONLY = 2  # Only use the cache, don't use the API
+CACHE_PREVENT = 3  # Don't use the cache
+
 
 class UnavailableException(Exception):
     """ Is thrown when an item is unavailable. """
@@ -89,8 +94,7 @@ class Episode:
     """ Defines an Episode. """
 
     def __init__(self, uuid=None, nodeid=None, path=None, channel=None, program_title=None, title=None, description=None, cover=None, duration=None,
-                 season=None, number=None,
+                 season=None, season_uuid=None, number=None, rating=None, aired=None, expiry=None):
-                 rating=None, aired=None, expiry=None):
         """
         :type uuid: str
         :type nodeid: str
@@ -102,6 +106,7 @@ class Episode:
         :type cover: str
         :type duration: int
         :type season: int
+        :type season_uuid: str
         :type number: int
         :type rating: str
         :type aired: datetime
@@ -117,6 +122,7 @@ class Episode:
         self.cover = cover
         self.duration = duration
         self.season = season
+        self.season_uuid = season_uuid
         self.number = number
         self.rating = rating
         self.aired = aired
@@ -129,30 +135,69 @@ class Episode:
 class ContentApi:
     """ VIER/VIJF/ZES Content API"""
     API_ENDPOINT = 'https://api.viervijfzes.be'
+    SITE_APIS = {
+        'vier': 'https://www.vier.be/api',
+        'vijf': 'https://www.vijf.be/api',
+        'zes': 'https://www.zestv.be/api',
+    }
 
-    def __init__(self, token=None):
+    def __init__(self, auth=None):
         """ Initialise object """
         self._session = requests.session()
-        self._token = token
+        self._auth = auth
 
     def get_notifications(self):
         """ Get a list of notifications for your account.
         :rtype list[dict]
         """
-        response = self._get_url(self.API_ENDPOINT + '/notifications')
+        response = self._get_url(self.API_ENDPOINT + '/notifications', authentication=True)
         data = json.loads(response)
         return data
 
-    def get_stream(self, _channel, uuid):
+    def get_content_tree(self, channel):
+        """ Get a list of all the content.
+        :type channel: str
+        :rtype list[dict]
+        """
+        if channel not in self.SITE_APIS:
+            raise Exception('Unknown channel %s' % channel)
+
+        response = self._get_url(self.SITE_APIS[channel] + '/content_tree', authentication=True)
+        data = json.loads(response)
+        return data
+
+    def get_stream_by_uuid(self, uuid):
         """ Get the stream URL to use for this video.
-        :type _channel: str
         :type uuid: str
         :rtype str
         """
-        response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid)
+        response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True)
         data = json.loads(response)
         return data['video']['S']
 
+    def get_programs_new(self, channel):
+        """ Get a list of all programs of the specified channel.
+        :type channel: str
+        :rtype list[Program]
+        """
+        if channel not in CHANNELS:
+            raise Exception('Unknown channel %s' % channel)
+
+        # Request all content from this channel
+        content_tree = self.get_content_tree(channel)
+
+        programs = []
+        for p in content_tree['programs']:
+            try:
+                program = self.get_program_by_uuid(p)
+                program.channel = channel
+                programs.append(program)
+            except UnavailableException:
+                # Some programs are not available, but do occur in the content tree
+                pass
+
+        return programs
+
     def get_programs(self, channel):
         """ Get a list of all programs of the specified channel.
         :type channel: str
@@ -171,36 +216,90 @@ class ContentApi:
                                     r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?'
                                     r'</a>', re.DOTALL)
 
-        programs = [
+        programs = []
-            Program(channel=channel,
+        for item in regex_programs.finditer(data):
-                    path=program.group('path').lstrip('/'),
+            path = item.group('path').lstrip('/')
-                    title=h.unescape(program.group('title').strip()))
+
-            for program in regex_programs.finditer(data)
+            program = self.get_program(channel, path, CACHE_ONLY)  # Get program details, but from cache only
-        ]
+            if program:
+                # Use program with metadata from cache
+                programs.append(program)
+            else:
+                # Use program with the values that we've parsed from the page
+                programs.append(Program(channel=channel,
+                                        path=path,
+                                        title=h.unescape(item.group('title').strip())))
 
         return programs
 
-    def get_program(self, channel, path):
+    def get_program(self, channel, path, cache=CACHE_AUTO):
         """ Get a Program object from the specified page.
         :type channel: str
         :type path: str
+        :type cache: int
         :rtype Program
         NOTE: This function doesn't use an API.
         """
         if channel not in CHANNELS:
             raise Exception('Unknown channel %s' % channel)
 
-        # Load webpage
+        if cache in [CACHE_AUTO, CACHE_ONLY]:
-        page = self._get_url(CHANNELS[channel]['url'] + '/' + path)
+            # Try to fetch from cache
+            data = kodiutils.get_cache(['program', channel, path])
+            if data is None and cache == CACHE_ONLY:
+                return None
+        else:
+            data = None
+
+        if data is None:
+            # Fetch webpage
+            page = self._get_url(CHANNELS[channel]['url'] + '/' + path)
+
+            # Extract JSON
+            regex_program = re.compile(r'data-hero="([^"]+)', re.DOTALL)
+            json_data = HTMLParser().unescape(regex_program.search(page).group(1))
+            data = json.loads(json_data)['data']
+
+            # Store response in cache
+            kodiutils.set_cache(['program', channel, path], data)
 
-        # Extract JSON
-        regex_program = re.compile(r'data-hero="([^"]+)', re.DOTALL)
-        json_data = HTMLParser().unescape(regex_program.search(page).group(1))
-        data = json.loads(json_data)['data']
         program = self._parse_program_data(data)
 
         return program
 
+    def get_program_by_uuid(self, uuid, cache=CACHE_AUTO):
+        """ Get a Program object.
+        :type uuid: str
+        :type cache: int
+        :rtype Program
+        """
+        if cache in [CACHE_AUTO, CACHE_ONLY]:
+            # Try to fetch from cache
+            data = kodiutils.get_cache(['program', uuid])
+            if data is None and cache == CACHE_ONLY:
+                return None
+        else:
+            data = None
+
+        if data is None:
+            # Fetch from API
+            response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True)
+            data = json.loads(response)
+
+            if not data:
+                raise UnavailableException()
+
+            # Store response in cache
+            kodiutils.set_cache(['program', uuid], data)
+
+        return Program(
+            uuid=uuid,
+            path=data['url']['S'].strip('/'),
+            title=data['label']['S'],
+            description=data['description']['S'],
+            cover=data['image']['S'],
+        )
+
     def get_episode(self, channel, path):
         """ Get a Episode object from the specified page.
         :type channel: str
@@ -254,7 +353,7 @@ class ContentApi:
 
         # Create Season info
         program.seasons = {
-            playlist['episodes'][0]['seasonNumber']: Season(
+            key: Season(
                 uuid=playlist['id'],
                 path=playlist['link'].lstrip('/'),
                 channel=playlist['pageInfo']['site'],
@@ -262,12 +361,12 @@ class ContentApi:
                 description=playlist['pageInfo']['description'],
                 number=playlist['episodes'][0]['seasonNumber'],  # You did not see this
             )
-            for playlist in data['playlists']
+            for key, playlist in enumerate(data['playlists'])
         }
 
         # Create Episodes info
         program.episodes = [
-            ContentApi._parse_episode_data(episode)
+            ContentApi._parse_episode_data(episode, playlist['id'])
             for playlist in data['playlists']
             for episode in playlist['episodes']
         ]
@@ -275,9 +374,10 @@ class ContentApi:
         return program
 
     @staticmethod
-    def _parse_episode_data(data):
+    def _parse_episode_data(data, season_uuid):
         """ Parse the Episode JSON.
         :type data: dict
+        :type season_uuid: str
         :rtype Episode
         """
 
@@ -291,22 +391,18 @@ class ContentApi:
             else:
                 episode_number = None
 
-        if data.get('episodeTitle'):
-            episode_title = data.get('episodeTitle')
-        else:
-            episode_title = data.get('title')
-
         episode = Episode(
             uuid=data.get('videoUuid'),
             nodeid=data.get('pageInfo', {}).get('nodeId'),
             path=data.get('link').lstrip('/'),
             channel=data.get('pageInfo', {}).get('site'),
             program_title=data.get('program', {}).get('title'),
-            title=episode_title,
+            title=data.get('title'),
             description=data.get('pageInfo', {}).get('description'),
             cover=data.get('image'),
             duration=data.get('duration'),
             season=data.get('seasonNumber'),
+            season_uuid=season_uuid,
             number=episode_number,
             aired=datetime.fromtimestamp(data.get('createdDate')),
             expiry=datetime.fromtimestamp(data.get('unpublishDate')) if data.get('unpublishDate') else None,
@@ -314,19 +410,22 @@ class ContentApi:
         )
         return episode
 
-    def _get_url(self, url, params=None):
+    def _get_url(self, url, params=None, authentication=False):
         """ Makes a GET request for the specified URL.
         :type url: str
         :rtype str
         """
-        if self._token:
+        if authentication:
+            if not self._auth:
+                raise Exception('Requested to authenticate, but not auth object passed')
             response = self._session.get(url, params=params, headers={
-                'authorization': self._token,
+                'authorization': self._auth.get_token(),
             })
         else:
             response = self._session.get(url, params=params)
 
         if response.status_code != 200:
+            _LOGGER.error(response.text)
             raise Exception('Could not fetch data')
 
         return response.text

resources/lib/viervijfzes/search.py

@@ -42,7 +42,7 @@ class SearchApi:
         if response.status_code != 200:
             raise Exception('Could not search')
 
-        data = json.loads(response.content)
+        data = json.loads(response.text)
 
         results = []
         for hit in data['hits']['hits']:

resources/settings.xml

@@ -1,8 +1,15 @@
 <?xml version="1.0" encoding="utf-8" standalone="yes"?>
 <settings>
+    <setting id="metadata_last_updated" visible="false"/>
     <category label="30800"> <!-- Credentials -->
         <setting label="30801" type="lsep"/> <!-- Credentials -->
         <setting label="30803" type="text" id="username"/>
         <setting label="30805" type="text" id="password" option="hidden"/>
     </category>
+    <category label="30820"> <!-- Interface -->
+        <setting label="30827" type="lsep"/> <!-- Metadata -->
+        <setting label="30829" type="bool" id="metadata_update" default="true" subsetting="true"/>
+        <setting label="30831" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/update)"/>
+        <setting label="30833" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/clean)"/>
+    </category>
 </settings>

service_entry.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+""" Service entry point """
+
+from __future__ import absolute_import, division, unicode_literals
+
+from resources.lib import service
+
+service.run()

test/test_api.py

@@ -9,8 +9,8 @@ import logging
 import unittest
 
 import resources.lib.kodiutils as kodiutils
-from resources.lib.viervijfzes.content import ContentApi, Program, Episode
 from resources.lib.viervijfzes.auth import AuthApi
+from resources.lib.viervijfzes.content import ContentApi, Program, Episode
 
 _LOGGER = logging.getLogger('test-api')
 
@@ -18,7 +18,8 @@ _LOGGER = logging.getLogger('test-api')
 class TestApi(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestApi, self).__init__(*args, **kwargs)
-        self._api = ContentApi()
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        self._api = ContentApi(auth)
 
     def test_programs(self):
         for channel in ['vier', 'vijf', 'zes']:
@@ -39,9 +40,7 @@ class TestApi(unittest.TestCase):
         program = self._api.get_program('vier', 'auwch')
         episode = program.episodes[0]
 
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
+        video = self._api.get_stream_by_uuid(episode.uuid)
-        api_authed = ContentApi(auth.get_token())
-        video = api_authed.get_stream(episode.channel, episode.uuid)
         self.assertTrue(video)
 
         _LOGGER.info('Got video URL: %s', video)

test/test_auth.py

@@ -17,18 +17,19 @@ _LOGGER = logging.getLogger('test-auth')
 class TestAuth(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestAuth, self).__init__(*args, **kwargs)
-        self._auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
 
     def test_login(self):
         # Clear any cache we have
-        self._auth.clear_cache()
+        AuthApi.clear_tokens()
 
         # We should get a token by logging in
-        token = self._auth.get_token()
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        token = auth.get_token()
         self.assertTrue(token)
 
         # Test it a second time, it should go from memory now
-        token = self._auth.get_token()
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        token = auth.get_token()
         self.assertTrue(token)
 
 

test/test_epg.py

@@ -49,15 +49,15 @@ class TestEpg(unittest.TestCase):
         epg_programs = self._epg.get_epg('vier', date.today().strftime('%Y-%m-%d'))
         epg_program = [program for program in epg_programs if program.video_url][0]
 
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        api = ContentApi(auth)
+
         # Lookup the Episode data since we don't have an UUID
-        api = ContentApi()
         episode = api.get_episode(epg_program.channel, epg_program.video_url)
         self.assertIsInstance(episode, Episode)
 
         # Get stream based on the Episode's UUID
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
+        video = api.get_stream_by_uuid(episode.uuid)
-        api = ContentApi(auth.get_token())
-        video = api.get_stream(episode.channel, episode.uuid)
         self.assertTrue(video)