Rework cache (#15)

* Rework cache

* Cancel at 5%

* Check before removing auth token

* Fix cache refreshing from service

resources/language/resource.language.en_gb/strings.po

@@ -177,8 +177,3 @@ msgstr ""
 msgctxt "#30831"
 msgid "Update local metadata now"
 msgstr ""
-
-msgctxt "#30833"
-msgid "Clear local metadata"
-msgstr ""
-

resources/language/resource.language.nl_nl/strings.po

@@ -178,7 +178,3 @@ msgstr "Vernieuw de lokale metdata automatisch in de achtergrond"
 msgctxt "#30831"
 msgid "Update local metadata now"
 msgstr "De lokale metadata nu vernieuwen"
-
-msgctxt "#30833"
-msgid "Clear local metadata"
-msgstr "De lokale metadata verwijderen"

resources/lib/addon.py

@@ -111,13 +111,6 @@ def metadata_update():
     Metadata().update()
 
 
-@routing.route('/metadata/clean')
-def metadata_clean():
-    """ Clear metadata (called from settings) """
-    from resources.lib.modules.metadata import Metadata
-    Metadata().clean()
-
-
 def run(params):
     """ Run the routing plugin """
     routing.run(params)

resources/lib/kodiutils.py

@@ -4,7 +4,6 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import logging
-from contextlib import contextmanager
 
 import xbmc
 import xbmcaddon
@@ -420,54 +419,6 @@ def get_addon_info(key):
     return to_unicode(ADDON.getAddonInfo(key))
 
 
-def listdir(path):
-    """Return all files in a directory (using xbmcvfs)"""
-    from xbmcvfs import listdir as vfslistdir
-    return vfslistdir(path)
-
-
-def mkdir(path):
-    """Create a directory (using xbmcvfs)"""
-    from xbmcvfs import mkdir as vfsmkdir
-    _LOGGER.debug("Create directory '%s'.", path)
-    return vfsmkdir(path)
-
-
-def mkdirs(path):
-    """Create directory including parents (using xbmcvfs)"""
-    from xbmcvfs import mkdirs as vfsmkdirs
-    _LOGGER.debug("Recursively create directory '%s'.", path)
-    return vfsmkdirs(path)
-
-
-def exists(path):
-    """Whether the path exists (using xbmcvfs)"""
-    from xbmcvfs import exists as vfsexists
-    return vfsexists(path)
-
-
-@contextmanager
-def open_file(path, flags='r'):
-    """Open a file (using xbmcvfs)"""
-    from xbmcvfs import File
-    fdesc = File(path, flags)
-    yield fdesc
-    fdesc.close()
-
-
-def stat_file(path):
-    """Return information about a file (using xbmcvfs)"""
-    from xbmcvfs import Stat
-    return Stat(path)
-
-
-def delete(path):
-    """Remove a file (using xbmcvfs)"""
-    from xbmcvfs import delete as vfsdelete
-    _LOGGER.debug("Delete file '%s'.", path)
-    return vfsdelete(path)
-
-
 def container_refresh(url=None):
     """Refresh the current container or (re)load a container by URL"""
     if url:
@@ -518,56 +469,3 @@ def jsonrpc(*args, **kwargs):
     if kwargs.get('jsonrpc') is None:
         kwargs.update(jsonrpc='2.0')
     return loads(xbmc.executeJSONRPC(dumps(kwargs)))
-
-
-def get_cache(key, ttl=None):
-    """ Get an item from the cache """
-    import time
-    path = get_cache_path()
-    filename = '.'.join(key)
-    fullpath = path + filename
-
-    if not exists(fullpath):
-        return None
-
-    if ttl and time.mktime(time.localtime()) - stat_file(fullpath).st_mtime() > ttl:
-        return None
-
-    with open_file(fullpath, 'r') as fdesc:
-        try:
-            _LOGGER.debug('Fetching %s from cache', filename)
-            import json
-            value = json.load(fdesc)
-            return value
-        except (ValueError, TypeError):
-            return None
-
-
-def set_cache(key, data):
-    """ Store an item in the cache """
-    path = get_cache_path()
-    filename = '.'.join(key)
-    fullpath = path + filename
-
-    if not exists(path):
-        mkdirs(path)
-
-    with open_file(fullpath, 'w') as fdesc:
-        _LOGGER.debug('Storing to cache as %s', filename)
-        import json
-        json.dump(data, fdesc)
-
-
-def invalidate_cache(ttl=None):
-    """ Clear the cache """
-    path = get_cache_path()
-    if not exists(path):
-        return
-    _, files = listdir(path)
-    import time
-    now = time.mktime(time.localtime())
-    for filename in files:
-        fullpath = path + filename
-        if ttl and now - stat_file(fullpath).st_mtime() < ttl:
-            continue
-        delete(fullpath)

resources/lib/modules/catalog.py

@@ -20,8 +20,8 @@ class Catalog:
 
     def __init__(self):
         """ Initialise object """
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
-        self._api = ContentApi(auth)
+        self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path())
         self._menu = Menu()
 
     def show_catalog(self):

resources/lib/modules/metadata.py

@@ -5,7 +5,7 @@ from __future__ import absolute_import, division, unicode_literals
 
 from resources.lib import kodiutils
 from resources.lib.viervijfzes import CHANNELS
-from resources.lib.viervijfzes.content import ContentApi, Program
+from resources.lib.viervijfzes.content import ContentApi, Program, CACHE_PREVENT, CACHE_AUTO
 
 
 class Metadata:
@@ -13,7 +13,7 @@ class Metadata:
 
     def __init__(self):
         """ Initialise object """
-        self._api = ContentApi()
+        self._api = ContentApi(cache_path=kodiutils.get_cache_path())
 
     def update(self):
         """ Update the metadata with a foreground progress indicator """
@@ -25,25 +25,26 @@ class Metadata:
             progress.update(int(((i + 1) / total) * 100), kodiutils.localize(30716, index=i + 1, total=total))  # Updating metadata ({index}/{total})
             return progress.iscanceled()
 
-        self.fetch_metadata(callback=update_status)
+        self.fetch_metadata(callback=update_status, refresh=True)
 
         # Close progress indicator
         progress.close()
 
-    def fetch_metadata(self, callback=None):
+    def fetch_metadata(self, callback=None, refresh=False):
         """ Fetch the metadata for all the items in the catalog
         :type callback: callable
+        :type refresh: bool
         """
         # Fetch all items from the catalog
         items = []
         for channel in list(CHANNELS):
-            items.extend(self._api.get_programs(channel))
+            items.extend(self._api.get_programs(channel, CACHE_PREVENT))
         count = len(items)
 
         # Loop over all of them and download the metadata
         for index, item in enumerate(items):
             if isinstance(item, Program):
-                self._api.get_program(item.channel, item.path)
+                self._api.get_program(item.channel, item.path, CACHE_PREVENT if refresh else CACHE_AUTO)
 
             # Run callback after every item
             if callback and callback(index, count):
@@ -51,10 +52,3 @@ class Metadata:
                 return False
 
         return True
-
-    @staticmethod
-    def clean():
-        """ Clear metadata (called from settings) """
-        kodiutils.invalidate_cache()
-        kodiutils.set_setting('metadata_last_updated', '0')
-        kodiutils.ok_dialog(message=kodiutils.localize(30714))  # Local metadata is cleared

resources/lib/modules/player.py

@@ -53,7 +53,7 @@ class Player:
 
             # Fetch an auth token now
             try:
-                auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+                auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
 
                 # Get stream information
                 resolved_stream = ContentApi(auth).get_stream_by_uuid(item)

resources/lib/service.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import, division, unicode_literals
 
 import hashlib
 import logging
+import os
 from time import time
 
 from xbmc import Monitor
@@ -23,6 +24,7 @@ class BackgroundService(Monitor):
         Monitor.__init__(self)
         self.update_interval = 24 * 3600  # Every 24 hours
         self.cache_expiry = 30 * 24 * 3600  # One month
+        self._auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
 
     def run(self):
         """ Background loop for maintenance tasks """
@@ -43,7 +45,7 @@ class BackgroundService(Monitor):
         """ Callback when a setting has changed """
         if self._has_credentials_changed():
             _LOGGER.info('Clearing auth tokens due to changed credentials')
-            AuthApi.clear_tokens()
+            self._auth.clear_tokens()
 
             # Refresh container
             kodiutils.container_refresh()
@@ -64,19 +66,34 @@ class BackgroundService(Monitor):
         """ Update the metadata for the listings """
         from resources.lib.modules.metadata import Metadata
 
-        # Clear outdated metadata
-        kodiutils.invalidate_cache(self.cache_expiry)
-
         def update_status(_i, _total):
             """ Allow to cancel the background job """
             return self.abortRequested() or not kodiutils.get_setting_bool('metadata_update')
 
+        # Clear metadata that has expired for 30 days
+        self._remove_expired_metadata(30 * 24 * 60 * 60)
+
+        # Fetch new metadata
         success = Metadata().fetch_metadata(callback=update_status)
 
         # Update metadata_last_updated
         if success:
             kodiutils.set_setting('metadata_last_updated', str(int(time())))
 
+    @staticmethod
+    def _remove_expired_metadata(keep_expired=None):
+        """ Clear the cache """
+        path = kodiutils.get_cache_path()
+        if not os.path.exists(path):
+            return
+
+        now = time()
+        for filename in os.listdir(path):
+            fullpath = path + filename
+            if keep_expired and os.stat(fullpath).st_mtime + keep_expired > now:
+                continue
+            os.unlink(fullpath)
+
 
 def run():
     """ Run the BackgroundService """

resources/lib/viervijfzes/auth.py

@@ -5,9 +5,9 @@ from __future__ import absolute_import, division, unicode_literals
 
 import json
 import logging
+import os
 import time
 
-from resources.lib import kodiutils
 from resources.lib.viervijfzes.auth_awsidp import AwsIdp, InvalidLoginException, AuthenticationException
 
 _LOGGER = logging.getLogger('auth-api')
@@ -21,18 +21,18 @@ class AuthApi:
 
     TOKEN_FILE = 'auth-tokens.json'
 
-    def __init__(self, username, password):
+    def __init__(self, username, password, token_path):
         """ Initialise object """
         self._username = username
         self._password = password
-        self._cache_dir = kodiutils.get_tokens_path()
+        self._token_path = token_path
         self._id_token = None
         self._expiry = 0
         self._refresh_token = None
 
         # Load tokens from cache
         try:
-            with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'rb') as fdesc:
+            with open(self._token_path + self.TOKEN_FILE, 'rb') as fdesc:
                 data_json = json.loads(fdesc.read())
                 self._id_token = data_json.get('id_token')
                 self._refresh_token = data_json.get('refresh_token')
@@ -72,9 +72,9 @@ class AuthApi:
             self._expiry = now + 3600
 
         # Store new tokens in cache
-        if not kodiutils.exists(self._cache_dir):
+        if not os.path.exists(self._token_path):
-            kodiutils.mkdirs(self._cache_dir)
+            os.mkdir(self._token_path)
-        with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'wb') as fdesc:
+        with open(self._token_path + self.TOKEN_FILE, 'wb') as fdesc:
             data = json.dumps(dict(
                 id_token=self._id_token,
                 refresh_token=self._refresh_token,
@@ -84,10 +84,10 @@ class AuthApi:
 
         return self._id_token
 
-    @staticmethod
+    def clear_tokens(self):
-    def clear_tokens():
         """ Remove the cached tokens. """
-        kodiutils.delete(kodiutils.get_tokens_path() + AuthApi.TOKEN_FILE)
+        if os.path.exists(self._token_path + AuthApi.TOKEN_FILE):
+            os.unlink(self._token_path + AuthApi.TOKEN_FILE)
 
     @staticmethod
     def _authenticate(username, password):

resources/lib/viervijfzes/content.py

@@ -5,13 +5,14 @@ from __future__ import absolute_import, division, unicode_literals
 
 import json
 import logging
+import os
 import re
+import time
 from datetime import datetime
 
 from six.moves.html_parser import HTMLParser
 import requests
 
-from resources.lib import kodiutils
 from resources.lib.viervijfzes import CHANNELS
 
 _LOGGER = logging.getLogger('content-api')
@@ -141,33 +142,49 @@ class ContentApi:
         'zes': 'https://www.zestv.be/api',
     }
 
-    def __init__(self, auth=None):
+    def __init__(self, auth=None, cache_path=None):
         """ Initialise object """
         self._session = requests.session()
         self._auth = auth
+        self._cache_path = cache_path
 
-    def get_programs(self, channel):
+    def get_programs(self, channel, cache=CACHE_AUTO):
         """ Get a list of all programs of the specified channel.
         :type channel: str
+        :type cache: str
         :rtype list[Program]
-        NOTE: This function doesn't use an API.
         """
         if channel not in CHANNELS:
             raise Exception('Unknown channel %s' % channel)
 
-        # Load webpage
+        def update():
-        data = self._get_url(CHANNELS[channel]['url'])
+            """ Fetch the program listing by scraping """
+            # Load webpage
+            raw_html = self._get_url(CHANNELS[channel]['url'])
 
-        # Parse programs
+            # Parse programs
-        parser = HTMLParser()
+            parser = HTMLParser()
-        regex_programs = re.compile(r'<a class="program-overview__link" href="(?P<path>[^"]+)">\s+'
+            regex_programs = re.compile(r'<a class="program-overview__link" href="(?P<path>[^"]+)">\s+'
-                                    r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?'
+                                        r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?'
-                                    r'</a>', re.DOTALL)
+                                        r'</a>', re.DOTALL)
+            data = {
+                item.group('path').lstrip('/'): parser.unescape(item.group('title').strip())
+                for item in regex_programs.finditer(raw_html)
+            }
+
+            if not data:
+                raise Exception('No programs found for %s' % channel)
+
+            return data
+
+        # Fetch listing from cache or update if needed
+        data = self._handle_cache(key=['programs', channel], cache_mode=cache, update=update, ttl=30 * 5)
+        if not data:
+            return []
 
         programs = []
-        for item in regex_programs.finditer(data):
+        for path in data:
-            path = item.group('path').lstrip('/')
+            title = data[path]
-
             program = self.get_program(channel, path, CACHE_ONLY)  # Get program details, but from cache only
             if program:
                 # Use program with metadata from cache
@@ -176,7 +193,7 @@ class ContentApi:
                 # Use program with the values that we've parsed from the page
                 programs.append(Program(channel=channel,
                                         path=path,
-                                        title=parser.unescape(item.group('title').strip())))
+                                        title=title))
         return programs
 
     def get_program(self, channel, path, cache=CACHE_AUTO):
@@ -185,20 +202,12 @@ class ContentApi:
         :type path: str
         :type cache: int
         :rtype Program
-        NOTE: This function doesn't use an API.
         """
         if channel not in CHANNELS:
             raise Exception('Unknown channel %s' % channel)
 
-        if cache in [CACHE_AUTO, CACHE_ONLY]:
+        def update():
-            # Try to fetch from cache
+            """ Fetch the program metadata by scraping """
-            data = kodiutils.get_cache(['program', channel, path])
-            if data is None and cache == CACHE_ONLY:
-                return None
-        else:
-            data = None
-
-        if data is None:
             # Fetch webpage
             page = self._get_url(CHANNELS[channel]['url'] + '/' + path)
 
@@ -207,46 +216,15 @@ class ContentApi:
             json_data = HTMLParser().unescape(regex_program.search(page).group(1))
             data = json.loads(json_data)['data']
 
-            # Store response in cache
+            return data
-            kodiutils.set_cache(['program', channel, path], data)
+
+        # Fetch listing from cache or update if needed
+        data = self._handle_cache(key=['program', channel, path], cache_mode=cache, update=update)
 
         program = self._parse_program_data(data)
 
         return program
 
-    def get_program_by_uuid(self, uuid, cache=CACHE_AUTO):
-        """ Get a Program object.
-        :type uuid: str
-        :type cache: int
-        :rtype Program
-        """
-        if cache in [CACHE_AUTO, CACHE_ONLY]:
-            # Try to fetch from cache
-            data = kodiutils.get_cache(['program', uuid])
-            if data is None and cache == CACHE_ONLY:
-                return None
-        else:
-            data = None
-
-        if data is None:
-            # Fetch from API
-            response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True)
-            data = json.loads(response)
-
-            if not data:
-                raise UnavailableException()
-
-            # Store response in cache
-            kodiutils.set_cache(['program', uuid], data)
-
-        return Program(
-            uuid=uuid,
-            path=data['url']['S'].strip('/'),
-            title=data['label']['S'],
-            description=data['description']['S'],
-            cover=data['image']['S'],
-        )
-
     def get_episode(self, channel, path):
         """ Get a Episode object from the specified page.
         :type channel: str
@@ -295,6 +273,9 @@ class ContentApi:
         :type data: dict
         :rtype Program
         """
+        if data is None:
+            return None
+
         # Create Program info
         program = Program(
             uuid=data['id'],
@@ -385,3 +366,62 @@ class ContentApi:
             raise Exception('Could not fetch data')
 
         return response.text
+
+    def _handle_cache(self, key, cache_mode, update, ttl=30 * 24 * 60 * 60):
+        """ Fetch something from the cache, and update if needed """
+        if cache_mode in [CACHE_AUTO, CACHE_ONLY]:
+            # Try to fetch from cache
+            data = self._get_cache(key)
+            if data is None and cache_mode == CACHE_ONLY:
+                return None
+        else:
+            data = None
+
+        if data is None:
+            try:
+                # Fetch fresh data
+                _LOGGER.debug('Fetching fresh data for key %s', '.'.join(key))
+                data = update()
+                if data:
+                    # Store fresh response in cache
+                    self._set_cache(key, data, ttl)
+            except Exception as exc:  # pylint: disable=broad-except
+                _LOGGER.warning('Something went wrong when refreshing live data: %s. Using expired cached values.', exc)
+                data = self._get_cache(key, allow_expired=True)
+
+        return data
+
+    def _get_cache(self, key, allow_expired=False):
+        """ Get an item from the cache """
+        filename = '.'.join(key) + '.json'
+        fullpath = self._cache_path + filename
+
+        if not os.path.exists(fullpath):
+            return None
+
+        if not allow_expired and os.stat(fullpath).st_mtime < time.time():
+            return None
+
+        with open(fullpath, 'r') as fdesc:
+            try:
+                _LOGGER.debug('Fetching %s from cache', filename)
+                value = json.load(fdesc)
+                return value
+            except (ValueError, TypeError):
+                return None
+
+    def _set_cache(self, key, data, ttl):
+        """ Store an item in the cache """
+        filename = '.'.join(key) + '.json'
+        fullpath = self._cache_path + filename
+
+        if not os.path.exists(self._cache_path):
+            os.mkdir(self._cache_path)
+
+        with open(fullpath, 'w') as fdesc:
+            _LOGGER.debug('Storing to cache as %s', filename)
+            json.dump(data, fdesc)
+
+        # Set TTL by modifying modification date
+        deadline = int(time.time()) + ttl
+        os.utime(fullpath, (deadline, deadline))

resources/settings.xml

@@ -10,6 +10,5 @@
         <setting label="30827" type="lsep"/> <!-- Metadata -->
         <setting label="30829" type="bool" id="metadata_update" default="true" subsetting="true"/>
         <setting label="30831" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/update)"/>
-        <setting label="30833" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/clean)"/>
     </category>
 </settings>

test/test_api.py

@@ -18,8 +18,8 @@ _LOGGER = logging.getLogger('test-api')
 class TestApi(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestApi, self).__init__(*args, **kwargs)
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
-        self._api = ContentApi(auth)
+        self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path())
 
     def test_programs(self):
         for channel in ['vier', 'vijf', 'zes']:
@@ -40,9 +40,6 @@ class TestApi(unittest.TestCase):
         program = self._api.get_program('vier', 'auwch')
         self.assertIsInstance(program, Program)
 
-        program_by_uuid = self._api.get_program_by_uuid(program.uuid)
-        self.assertIsInstance(program_by_uuid, Program)
-
         episode = program.episodes[0]
         video = self._api.get_stream_by_uuid(episode.uuid)
         self.assertTrue(video)

test/test_auth.py

@@ -20,16 +20,16 @@ class TestAuth(unittest.TestCase):
 
     @unittest.skipUnless(kodiutils.get_setting('username') and kodiutils.get_setting('password'), 'Skipping since we have no credentials.')
     def test_login(self):
+        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
+
         # Clear any cache we have
-        AuthApi.clear_tokens()
+        auth.clear_tokens()
 
         # We should get a token by logging in
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
         token = auth.get_token()
         self.assertTrue(token)
 
         # Test it a second time, it should go from memory now
-        auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
         token = auth.get_token()
         self.assertTrue(token)
 

test/test_epg.py

@@ -9,6 +9,7 @@ import logging
 import unittest
 from datetime import date
 
+from resources.lib import kodiutils
 from resources.lib.viervijfzes.content import ContentApi, Episode
 from resources.lib.viervijfzes.epg import EpgApi, EpgProgram
 
@@ -48,7 +49,7 @@ class TestEpg(unittest.TestCase):
         epg_program = [program for program in epg_programs if program.video_url][0]
 
         # Lookup the Episode data since we don't have an UUID
-        api = ContentApi()
+        api = ContentApi(cache_path=kodiutils.get_cache_path())
         episode = api.get_episode(epg_program.channel, epg_program.video_url)
         self.assertIsInstance(episode, Episode)
 

test/test_routing.py

@@ -57,7 +57,7 @@ class TestRouting(unittest.TestCase):
         routing.run([routing.url_for(addon.show_tvguide_detail, channel='vier', date='today'), '0', ''])
 
     def test_metadata_update(self):
-        routing.run([routing.url_for(addon.metadata_clean), '0', ''])
+        routing.run([routing.url_for(addon.metadata_update), '0', ''])
 
 
 if __name__ == '__main__':

test/xbmcgui.py

@@ -129,9 +129,9 @@ class DialogProgress:
         print('\033[37;44;1mPROGRESS:\033[35;49;1m [%s] \033[37;1m%s\033[39;0m' % (heading, line1))
         sys.stdout.flush()
 
-    @staticmethod
+    def iscanceled(self):
-    def iscanceled():
         """A stub implementation for the xbmcgui DialogProgress class iscanceled() method"""
+        return self.percentage > 5  # Cancel at 5%
 
     def update(self, percentage, line1=None, line2=None, line3=None):
         """A stub implementation for the xbmcgui DialogProgress class update() method"""