diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 7e01c4a..345d60d 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -177,8 +177,3 @@ msgstr "" msgctxt "#30831" msgid "Update local metadata now" msgstr "" - -msgctxt "#30833" -msgid "Clear local metadata" -msgstr "" - diff --git a/resources/language/resource.language.nl_nl/strings.po b/resources/language/resource.language.nl_nl/strings.po index 15ea316..4aab7b5 100644 --- a/resources/language/resource.language.nl_nl/strings.po +++ b/resources/language/resource.language.nl_nl/strings.po @@ -178,7 +178,3 @@ msgstr "Vernieuw de lokale metdata automatisch in de achtergrond" msgctxt "#30831" msgid "Update local metadata now" msgstr "De lokale metadata nu vernieuwen" - -msgctxt "#30833" -msgid "Clear local metadata" -msgstr "De lokale metadata verwijderen" diff --git a/resources/lib/addon.py b/resources/lib/addon.py index e7382eb..cd326bc 100644 --- a/resources/lib/addon.py +++ b/resources/lib/addon.py @@ -111,13 +111,6 @@ def metadata_update(): Metadata().update() -@routing.route('/metadata/clean') -def metadata_clean(): - """ Clear metadata (called from settings) """ - from resources.lib.modules.metadata import Metadata - Metadata().clean() - - def run(params): """ Run the routing plugin """ routing.run(params) diff --git a/resources/lib/kodiutils.py b/resources/lib/kodiutils.py index 83d1d58..843ceed 100644 --- a/resources/lib/kodiutils.py +++ b/resources/lib/kodiutils.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, unicode_literals import logging -from contextlib import contextmanager import xbmc import xbmcaddon @@ -420,54 +419,6 @@ def get_addon_info(key): return to_unicode(ADDON.getAddonInfo(key)) -def listdir(path): - """Return all files in a directory (using xbmcvfs)""" - from xbmcvfs import listdir as vfslistdir - return vfslistdir(path) - - -def mkdir(path): - """Create a directory (using xbmcvfs)""" - from xbmcvfs import mkdir as vfsmkdir - _LOGGER.debug("Create directory '%s'.", path) - return vfsmkdir(path) - - -def mkdirs(path): - """Create directory including parents (using xbmcvfs)""" - from xbmcvfs import mkdirs as vfsmkdirs - _LOGGER.debug("Recursively create directory '%s'.", path) - return vfsmkdirs(path) - - -def exists(path): - """Whether the path exists (using xbmcvfs)""" - from xbmcvfs import exists as vfsexists - return vfsexists(path) - - -@contextmanager -def open_file(path, flags='r'): - """Open a file (using xbmcvfs)""" - from xbmcvfs import File - fdesc = File(path, flags) - yield fdesc - fdesc.close() - - -def stat_file(path): - """Return information about a file (using xbmcvfs)""" - from xbmcvfs import Stat - return Stat(path) - - -def delete(path): - """Remove a file (using xbmcvfs)""" - from xbmcvfs import delete as vfsdelete - _LOGGER.debug("Delete file '%s'.", path) - return vfsdelete(path) - - def container_refresh(url=None): """Refresh the current container or (re)load a container by URL""" if url: @@ -518,56 +469,3 @@ def jsonrpc(*args, **kwargs): if kwargs.get('jsonrpc') is None: kwargs.update(jsonrpc='2.0') return loads(xbmc.executeJSONRPC(dumps(kwargs))) - - -def get_cache(key, ttl=None): - """ Get an item from the cache """ - import time - path = get_cache_path() - filename = '.'.join(key) - fullpath = path + filename - - if not exists(fullpath): - return None - - if ttl and time.mktime(time.localtime()) - stat_file(fullpath).st_mtime() > ttl: - return None - - with open_file(fullpath, 'r') as fdesc: - try: - _LOGGER.debug('Fetching %s from cache', filename) - import json - value = json.load(fdesc) - return value - except (ValueError, TypeError): - return None - - -def set_cache(key, data): - """ Store an item in the cache """ - path = get_cache_path() - filename = '.'.join(key) - fullpath = path + filename - - if not exists(path): - mkdirs(path) - - with open_file(fullpath, 'w') as fdesc: - _LOGGER.debug('Storing to cache as %s', filename) - import json - json.dump(data, fdesc) - - -def invalidate_cache(ttl=None): - """ Clear the cache """ - path = get_cache_path() - if not exists(path): - return - _, files = listdir(path) - import time - now = time.mktime(time.localtime()) - for filename in files: - fullpath = path + filename - if ttl and now - stat_file(fullpath).st_mtime() < ttl: - continue - delete(fullpath) diff --git a/resources/lib/modules/catalog.py b/resources/lib/modules/catalog.py index 72f4ba7..d2da86c 100644 --- a/resources/lib/modules/catalog.py +++ b/resources/lib/modules/catalog.py @@ -20,8 +20,8 @@ class Catalog: def __init__(self): """ Initialise object """ - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) - self._api = ContentApi(auth) + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) + self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path()) self._menu = Menu() def show_catalog(self): diff --git a/resources/lib/modules/metadata.py b/resources/lib/modules/metadata.py index 5a938f9..866c359 100644 --- a/resources/lib/modules/metadata.py +++ b/resources/lib/modules/metadata.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, unicode_literals from resources.lib import kodiutils from resources.lib.viervijfzes import CHANNELS -from resources.lib.viervijfzes.content import ContentApi, Program +from resources.lib.viervijfzes.content import ContentApi, Program, CACHE_PREVENT, CACHE_AUTO class Metadata: @@ -13,7 +13,7 @@ class Metadata: def __init__(self): """ Initialise object """ - self._api = ContentApi() + self._api = ContentApi(cache_path=kodiutils.get_cache_path()) def update(self): """ Update the metadata with a foreground progress indicator """ @@ -25,25 +25,26 @@ class Metadata: progress.update(int(((i + 1) / total) * 100), kodiutils.localize(30716, index=i + 1, total=total)) # Updating metadata ({index}/{total}) return progress.iscanceled() - self.fetch_metadata(callback=update_status) + self.fetch_metadata(callback=update_status, refresh=True) # Close progress indicator progress.close() - def fetch_metadata(self, callback=None): + def fetch_metadata(self, callback=None, refresh=False): """ Fetch the metadata for all the items in the catalog :type callback: callable + :type refresh: bool """ # Fetch all items from the catalog items = [] for channel in list(CHANNELS): - items.extend(self._api.get_programs(channel)) + items.extend(self._api.get_programs(channel, CACHE_PREVENT)) count = len(items) # Loop over all of them and download the metadata for index, item in enumerate(items): if isinstance(item, Program): - self._api.get_program(item.channel, item.path) + self._api.get_program(item.channel, item.path, CACHE_PREVENT if refresh else CACHE_AUTO) # Run callback after every item if callback and callback(index, count): @@ -51,10 +52,3 @@ class Metadata: return False return True - - @staticmethod - def clean(): - """ Clear metadata (called from settings) """ - kodiutils.invalidate_cache() - kodiutils.set_setting('metadata_last_updated', '0') - kodiutils.ok_dialog(message=kodiutils.localize(30714)) # Local metadata is cleared diff --git a/resources/lib/modules/player.py b/resources/lib/modules/player.py index 4585bb7..3ddf60b 100644 --- a/resources/lib/modules/player.py +++ b/resources/lib/modules/player.py @@ -53,7 +53,7 @@ class Player: # Fetch an auth token now try: - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) # Get stream information resolved_stream = ContentApi(auth).get_stream_by_uuid(item) diff --git a/resources/lib/service.py b/resources/lib/service.py index e3fac5b..9f1c5fb 100644 --- a/resources/lib/service.py +++ b/resources/lib/service.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, unicode_literals import hashlib import logging +import os from time import time from xbmc import Monitor @@ -23,6 +24,7 @@ class BackgroundService(Monitor): Monitor.__init__(self) self.update_interval = 24 * 3600 # Every 24 hours self.cache_expiry = 30 * 24 * 3600 # One month + self._auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) def run(self): """ Background loop for maintenance tasks """ @@ -43,7 +45,7 @@ class BackgroundService(Monitor): """ Callback when a setting has changed """ if self._has_credentials_changed(): _LOGGER.info('Clearing auth tokens due to changed credentials') - AuthApi.clear_tokens() + self._auth.clear_tokens() # Refresh container kodiutils.container_refresh() @@ -64,19 +66,34 @@ class BackgroundService(Monitor): """ Update the metadata for the listings """ from resources.lib.modules.metadata import Metadata - # Clear outdated metadata - kodiutils.invalidate_cache(self.cache_expiry) - def update_status(_i, _total): """ Allow to cancel the background job """ return self.abortRequested() or not kodiutils.get_setting_bool('metadata_update') + # Clear metadata that has expired for 30 days + self._remove_expired_metadata(30 * 24 * 60 * 60) + + # Fetch new metadata success = Metadata().fetch_metadata(callback=update_status) # Update metadata_last_updated if success: kodiutils.set_setting('metadata_last_updated', str(int(time()))) + @staticmethod + def _remove_expired_metadata(keep_expired=None): + """ Clear the cache """ + path = kodiutils.get_cache_path() + if not os.path.exists(path): + return + + now = time() + for filename in os.listdir(path): + fullpath = path + filename + if keep_expired and os.stat(fullpath).st_mtime + keep_expired > now: + continue + os.unlink(fullpath) + def run(): """ Run the BackgroundService """ diff --git a/resources/lib/viervijfzes/auth.py b/resources/lib/viervijfzes/auth.py index 5bf14b7..76a61fc 100644 --- a/resources/lib/viervijfzes/auth.py +++ b/resources/lib/viervijfzes/auth.py @@ -5,9 +5,9 @@ from __future__ import absolute_import, division, unicode_literals import json import logging +import os import time -from resources.lib import kodiutils from resources.lib.viervijfzes.auth_awsidp import AwsIdp, InvalidLoginException, AuthenticationException _LOGGER = logging.getLogger('auth-api') @@ -21,18 +21,18 @@ class AuthApi: TOKEN_FILE = 'auth-tokens.json' - def __init__(self, username, password): + def __init__(self, username, password, token_path): """ Initialise object """ self._username = username self._password = password - self._cache_dir = kodiutils.get_tokens_path() + self._token_path = token_path self._id_token = None self._expiry = 0 self._refresh_token = None # Load tokens from cache try: - with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'rb') as fdesc: + with open(self._token_path + self.TOKEN_FILE, 'rb') as fdesc: data_json = json.loads(fdesc.read()) self._id_token = data_json.get('id_token') self._refresh_token = data_json.get('refresh_token') @@ -72,9 +72,9 @@ class AuthApi: self._expiry = now + 3600 # Store new tokens in cache - if not kodiutils.exists(self._cache_dir): - kodiutils.mkdirs(self._cache_dir) - with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'wb') as fdesc: + if not os.path.exists(self._token_path): + os.mkdir(self._token_path) + with open(self._token_path + self.TOKEN_FILE, 'wb') as fdesc: data = json.dumps(dict( id_token=self._id_token, refresh_token=self._refresh_token, @@ -84,10 +84,10 @@ class AuthApi: return self._id_token - @staticmethod - def clear_tokens(): + def clear_tokens(self): """ Remove the cached tokens. """ - kodiutils.delete(kodiutils.get_tokens_path() + AuthApi.TOKEN_FILE) + if os.path.exists(self._token_path + AuthApi.TOKEN_FILE): + os.unlink(self._token_path + AuthApi.TOKEN_FILE) @staticmethod def _authenticate(username, password): diff --git a/resources/lib/viervijfzes/content.py b/resources/lib/viervijfzes/content.py index 6a5a0ba..4a263b0 100644 --- a/resources/lib/viervijfzes/content.py +++ b/resources/lib/viervijfzes/content.py @@ -5,13 +5,14 @@ from __future__ import absolute_import, division, unicode_literals import json import logging +import os import re +import time from datetime import datetime from six.moves.html_parser import HTMLParser import requests -from resources.lib import kodiutils from resources.lib.viervijfzes import CHANNELS _LOGGER = logging.getLogger('content-api') @@ -141,33 +142,49 @@ class ContentApi: 'zes': 'https://www.zestv.be/api', } - def __init__(self, auth=None): + def __init__(self, auth=None, cache_path=None): """ Initialise object """ self._session = requests.session() self._auth = auth + self._cache_path = cache_path - def get_programs(self, channel): + def get_programs(self, channel, cache=CACHE_AUTO): """ Get a list of all programs of the specified channel. :type channel: str + :type cache: str :rtype list[Program] - NOTE: This function doesn't use an API. """ if channel not in CHANNELS: raise Exception('Unknown channel %s' % channel) - # Load webpage - data = self._get_url(CHANNELS[channel]['url']) + def update(): + """ Fetch the program listing by scraping """ + # Load webpage + raw_html = self._get_url(CHANNELS[channel]['url']) - # Parse programs - parser = HTMLParser() - regex_programs = re.compile(r'\s+' - r'\s+(?P[^<]+)</span>.*?' - r'</a>', re.DOTALL) + # Parse programs + parser = HTMLParser() + regex_programs = re.compile(r'<a class="program-overview__link" href="(?P<path>[^"]+)">\s+' + r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?' + r'</a>', re.DOTALL) + data = { + item.group('path').lstrip('/'): parser.unescape(item.group('title').strip()) + for item in regex_programs.finditer(raw_html) + } + + if not data: + raise Exception('No programs found for %s' % channel) + + return data + + # Fetch listing from cache or update if needed + data = self._handle_cache(key=['programs', channel], cache_mode=cache, update=update, ttl=30 * 5) + if not data: + return [] programs = [] - for item in regex_programs.finditer(data): - path = item.group('path').lstrip('/') - + for path in data: + title = data[path] program = self.get_program(channel, path, CACHE_ONLY) # Get program details, but from cache only if program: # Use program with metadata from cache @@ -176,7 +193,7 @@ class ContentApi: # Use program with the values that we've parsed from the page programs.append(Program(channel=channel, path=path, - title=parser.unescape(item.group('title').strip()))) + title=title)) return programs def get_program(self, channel, path, cache=CACHE_AUTO): @@ -185,20 +202,12 @@ class ContentApi: :type path: str :type cache: int :rtype Program - NOTE: This function doesn't use an API. """ if channel not in CHANNELS: raise Exception('Unknown channel %s' % channel) - if cache in [CACHE_AUTO, CACHE_ONLY]: - # Try to fetch from cache - data = kodiutils.get_cache(['program', channel, path]) - if data is None and cache == CACHE_ONLY: - return None - else: - data = None - - if data is None: + def update(): + """ Fetch the program metadata by scraping """ # Fetch webpage page = self._get_url(CHANNELS[channel]['url'] + '/' + path) @@ -207,46 +216,15 @@ class ContentApi: json_data = HTMLParser().unescape(regex_program.search(page).group(1)) data = json.loads(json_data)['data'] - # Store response in cache - kodiutils.set_cache(['program', channel, path], data) + return data + + # Fetch listing from cache or update if needed + data = self._handle_cache(key=['program', channel, path], cache_mode=cache, update=update) program = self._parse_program_data(data) return program - def get_program_by_uuid(self, uuid, cache=CACHE_AUTO): - """ Get a Program object. - :type uuid: str - :type cache: int - :rtype Program - """ - if cache in [CACHE_AUTO, CACHE_ONLY]: - # Try to fetch from cache - data = kodiutils.get_cache(['program', uuid]) - if data is None and cache == CACHE_ONLY: - return None - else: - data = None - - if data is None: - # Fetch from API - response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True) - data = json.loads(response) - - if not data: - raise UnavailableException() - - # Store response in cache - kodiutils.set_cache(['program', uuid], data) - - return Program( - uuid=uuid, - path=data['url']['S'].strip('/'), - title=data['label']['S'], - description=data['description']['S'], - cover=data['image']['S'], - ) - def get_episode(self, channel, path): """ Get a Episode object from the specified page. :type channel: str @@ -295,6 +273,9 @@ class ContentApi: :type data: dict :rtype Program """ + if data is None: + return None + # Create Program info program = Program( uuid=data['id'], @@ -385,3 +366,62 @@ class ContentApi: raise Exception('Could not fetch data') return response.text + + def _handle_cache(self, key, cache_mode, update, ttl=30 * 24 * 60 * 60): + """ Fetch something from the cache, and update if needed """ + if cache_mode in [CACHE_AUTO, CACHE_ONLY]: + # Try to fetch from cache + data = self._get_cache(key) + if data is None and cache_mode == CACHE_ONLY: + return None + else: + data = None + + if data is None: + try: + # Fetch fresh data + _LOGGER.debug('Fetching fresh data for key %s', '.'.join(key)) + data = update() + if data: + # Store fresh response in cache + self._set_cache(key, data, ttl) + except Exception as exc: # pylint: disable=broad-except + _LOGGER.warning('Something went wrong when refreshing live data: %s. Using expired cached values.', exc) + data = self._get_cache(key, allow_expired=True) + + return data + + def _get_cache(self, key, allow_expired=False): + """ Get an item from the cache """ + filename = '.'.join(key) + '.json' + fullpath = self._cache_path + filename + + if not os.path.exists(fullpath): + return None + + if not allow_expired and os.stat(fullpath).st_mtime < time.time(): + return None + + with open(fullpath, 'r') as fdesc: + try: + _LOGGER.debug('Fetching %s from cache', filename) + value = json.load(fdesc) + return value + except (ValueError, TypeError): + return None + + def _set_cache(self, key, data, ttl): + """ Store an item in the cache """ + filename = '.'.join(key) + '.json' + fullpath = self._cache_path + filename + + if not os.path.exists(self._cache_path): + os.mkdir(self._cache_path) + + with open(fullpath, 'w') as fdesc: + _LOGGER.debug('Storing to cache as %s', filename) + json.dump(data, fdesc) + + # Set TTL by modifying modification date + deadline = int(time.time()) + ttl + os.utime(fullpath, (deadline, deadline)) diff --git a/resources/settings.xml b/resources/settings.xml index 3fbdb78..277f2de 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -10,6 +10,5 @@ <setting label="30827" type="lsep"/> <!-- Metadata --> <setting label="30829" type="bool" id="metadata_update" default="true" subsetting="true"/> <setting label="30831" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/update)"/> - <setting label="30833" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/clean)"/> </category> </settings> diff --git a/test/test_api.py b/test/test_api.py index d3a0e98..f19990a 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -18,8 +18,8 @@ _LOGGER = logging.getLogger('test-api') class TestApi(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestApi, self).__init__(*args, **kwargs) - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) - self._api = ContentApi(auth) + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) + self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path()) def test_programs(self): for channel in ['vier', 'vijf', 'zes']: @@ -40,9 +40,6 @@ class TestApi(unittest.TestCase): program = self._api.get_program('vier', 'auwch') self.assertIsInstance(program, Program) - program_by_uuid = self._api.get_program_by_uuid(program.uuid) - self.assertIsInstance(program_by_uuid, Program) - episode = program.episodes[0] video = self._api.get_stream_by_uuid(episode.uuid) self.assertTrue(video) diff --git a/test/test_auth.py b/test/test_auth.py index de83c9e..50281e9 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -20,16 +20,16 @@ class TestAuth(unittest.TestCase): @unittest.skipUnless(kodiutils.get_setting('username') and kodiutils.get_setting('password'), 'Skipping since we have no credentials.') def test_login(self): + auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path()) + # Clear any cache we have - AuthApi.clear_tokens() + auth.clear_tokens() # We should get a token by logging in - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) token = auth.get_token() self.assertTrue(token) # Test it a second time, it should go from memory now - auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password')) token = auth.get_token() self.assertTrue(token) diff --git a/test/test_epg.py b/test/test_epg.py index 87d9da2..50dbbb0 100644 --- a/test/test_epg.py +++ b/test/test_epg.py @@ -9,6 +9,7 @@ import logging import unittest from datetime import date +from resources.lib import kodiutils from resources.lib.viervijfzes.content import ContentApi, Episode from resources.lib.viervijfzes.epg import EpgApi, EpgProgram @@ -48,7 +49,7 @@ class TestEpg(unittest.TestCase): epg_program = [program for program in epg_programs if program.video_url][0] # Lookup the Episode data since we don't have an UUID - api = ContentApi() + api = ContentApi(cache_path=kodiutils.get_cache_path()) episode = api.get_episode(epg_program.channel, epg_program.video_url) self.assertIsInstance(episode, Episode) diff --git a/test/test_routing.py b/test/test_routing.py index dfebaf3..3d2c432 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -57,7 +57,7 @@ class TestRouting(unittest.TestCase): routing.run([routing.url_for(addon.show_tvguide_detail, channel='vier', date='today'), '0', '']) def test_metadata_update(self): - routing.run([routing.url_for(addon.metadata_clean), '0', '']) + routing.run([routing.url_for(addon.metadata_update), '0', '']) if __name__ == '__main__': diff --git a/test/xbmcgui.py b/test/xbmcgui.py index d1103be..a9f5648 100644 --- a/test/xbmcgui.py +++ b/test/xbmcgui.py @@ -129,9 +129,9 @@ class DialogProgress: print('\033[37;44;1mPROGRESS:\033[35;49;1m [%s] \033[37;1m%s\033[39;0m' % (heading, line1)) sys.stdout.flush() - @staticmethod - def iscanceled(): + def iscanceled(self): """A stub implementation for the xbmcgui DialogProgress class iscanceled() method""" + return self.percentage > 5 # Cancel at 5% def update(self, percentage, line1=None, line2=None, line3=None): """A stub implementation for the xbmcgui DialogProgress class update() method"""