JeroenED/plugin.video.viervijfzes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Rework cache (#15)

Browse Source 
* Rework cache

* Cancel at 5%

* Check before removing auth token

* Fix cache refreshing from service
This commit is contained in:
Michaël Arnauts 2020-04-01 11:01:22 +02:00 committed by GitHub
parent 3d7a05cf24
commit 6e9a4718fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 151 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
resources/language/resource.language.en_gb/strings.po
View File

@ -177,8 +177,3 @@ msgstr ""
msgctxt "#30831"
msgid "Update local metadata now"
msgstr ""
msgctxt "#30833"
msgid "Clear local metadata"
msgstr ""

4
resources/language/resource.language.nl_nl/strings.po
View File

@ -178,7 +178,3 @@ msgstr "Vernieuw de lokale metdata automatisch in de achtergrond"
msgctxt "#30831"
msgid "Update local metadata now"
msgstr "De lokale metadata nu vernieuwen"
msgctxt "#30833"
msgid "Clear local metadata"
msgstr "De lokale metadata verwijderen"

7
resources/lib/addon.py
View File

@ -111,13 +111,6 @@ def metadata_update():
Metadata().update()
@routing.route('/metadata/clean')
def metadata_clean():
""" Clear metadata (called from settings) """
from resources.lib.modules.metadata import Metadata
Metadata().clean()
def run(params):
""" Run the routing plugin """
routing.run(params)

102
resources/lib/kodiutils.py
View File

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, unicode_literals
import logging
from contextlib import contextmanager
import xbmc
import xbmcaddon
@ -420,54 +419,6 @@ def get_addon_info(key):
return to_unicode(ADDON.getAddonInfo(key))
def listdir(path):
"""Return all files in a directory (using xbmcvfs)"""
from xbmcvfs import listdir as vfslistdir
return vfslistdir(path)
def mkdir(path):
"""Create a directory (using xbmcvfs)"""
from xbmcvfs import mkdir as vfsmkdir
_LOGGER.debug("Create directory '%s'.", path)
return vfsmkdir(path)
def mkdirs(path):
"""Create directory including parents (using xbmcvfs)"""
from xbmcvfs import mkdirs as vfsmkdirs
_LOGGER.debug("Recursively create directory '%s'.", path)
return vfsmkdirs(path)
def exists(path):
"""Whether the path exists (using xbmcvfs)"""
from xbmcvfs import exists as vfsexists
return vfsexists(path)
@contextmanager
def open_file(path, flags='r'):
"""Open a file (using xbmcvfs)"""
from xbmcvfs import File
fdesc = File(path, flags)
yield fdesc
fdesc.close()
def stat_file(path):
"""Return information about a file (using xbmcvfs)"""
from xbmcvfs import Stat
return Stat(path)
def delete(path):
"""Remove a file (using xbmcvfs)"""
from xbmcvfs import delete as vfsdelete
_LOGGER.debug("Delete file '%s'.", path)
return vfsdelete(path)
def container_refresh(url=None):
"""Refresh the current container or (re)load a container by URL"""
if url:
@ -518,56 +469,3 @@ def jsonrpc(*args, **kwargs):
if kwargs.get('jsonrpc') is None:
kwargs.update(jsonrpc='2.0')
return loads(xbmc.executeJSONRPC(dumps(kwargs)))
def get_cache(key, ttl=None):
""" Get an item from the cache """
import time
path = get_cache_path()
filename = '.'.join(key)
fullpath = path + filename
if not exists(fullpath):
return None
if ttl and time.mktime(time.localtime()) - stat_file(fullpath).st_mtime() > ttl:
return None
with open_file(fullpath, 'r') as fdesc:
try:
_LOGGER.debug('Fetching %s from cache', filename)
import json
value = json.load(fdesc)
return value
except (ValueError, TypeError):
return None
def set_cache(key, data):
""" Store an item in the cache """
path = get_cache_path()
filename = '.'.join(key)
fullpath = path + filename
if not exists(path):
mkdirs(path)
with open_file(fullpath, 'w') as fdesc:
_LOGGER.debug('Storing to cache as %s', filename)
import json
json.dump(data, fdesc)
def invalidate_cache(ttl=None):
""" Clear the cache """
path = get_cache_path()
if not exists(path):
return
_, files = listdir(path)
import time
now = time.mktime(time.localtime())
for filename in files:
fullpath = path + filename
if ttl and now - stat_file(fullpath).st_mtime() < ttl:
continue
delete(fullpath)

4
resources/lib/modules/catalog.py
View File

@ -20,8 +20,8 @@ class Catalog:
def __init__(self):
""" Initialise object """
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
self._api = ContentApi(auth)
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path())
self._menu = Menu()
def show_catalog(self):

20
resources/lib/modules/metadata.py
View File

@ -5,7 +5,7 @@ from __future__ import absolute_import, division, unicode_literals
from resources.lib import kodiutils
from resources.lib.viervijfzes import CHANNELS
from resources.lib.viervijfzes.content import ContentApi, Program
from resources.lib.viervijfzes.content import ContentApi, Program, CACHE_PREVENT, CACHE_AUTO
class Metadata:
@ -13,7 +13,7 @@ class Metadata:
def __init__(self):
""" Initialise object """
self._api = ContentApi()
self._api = ContentApi(cache_path=kodiutils.get_cache_path())
def update(self):
""" Update the metadata with a foreground progress indicator """
@ -25,25 +25,26 @@ class Metadata:
progress.update(int(((i + 1) / total) * 100), kodiutils.localize(30716, index=i + 1, total=total)) # Updating metadata ({index}/{total})
return progress.iscanceled()
self.fetch_metadata(callback=update_status)
self.fetch_metadata(callback=update_status, refresh=True)
# Close progress indicator
progress.close()
def fetch_metadata(self, callback=None):
def fetch_metadata(self, callback=None, refresh=False):
""" Fetch the metadata for all the items in the catalog
:type callback: callable
:type refresh: bool
"""
# Fetch all items from the catalog
items = []
for channel in list(CHANNELS):
items.extend(self._api.get_programs(channel))
items.extend(self._api.get_programs(channel, CACHE_PREVENT))
count = len(items)
# Loop over all of them and download the metadata
for index, item in enumerate(items):
if isinstance(item, Program):
self._api.get_program(item.channel, item.path)
self._api.get_program(item.channel, item.path, CACHE_PREVENT if refresh else CACHE_AUTO)
# Run callback after every item
if callback and callback(index, count):
@ -51,10 +52,3 @@ class Metadata:
return False
return True
@staticmethod
def clean():
""" Clear metadata (called from settings) """
kodiutils.invalidate_cache()
kodiutils.set_setting('metadata_last_updated', '0')
kodiutils.ok_dialog(message=kodiutils.localize(30714)) # Local metadata is cleared

2
resources/lib/modules/player.py
View File

@ -53,7 +53,7 @@ class Player:
# Fetch an auth token now
try:
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
# Get stream information
resolved_stream = ContentApi(auth).get_stream_by_uuid(item)

25
resources/lib/service.py
View File

@ -5,6 +5,7 @@ from __future__ import absolute_import, division, unicode_literals
import hashlib
import logging
import os
from time import time
from xbmc import Monitor
@ -23,6 +24,7 @@ class BackgroundService(Monitor):
Monitor.__init__(self)
self.update_interval = 24 * 3600 # Every 24 hours
self.cache_expiry = 30 * 24 * 3600 # One month
self._auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
def run(self):
""" Background loop for maintenance tasks """
@ -43,7 +45,7 @@ class BackgroundService(Monitor):
""" Callback when a setting has changed """
if self._has_credentials_changed():
_LOGGER.info('Clearing auth tokens due to changed credentials')
AuthApi.clear_tokens()
self._auth.clear_tokens()
# Refresh container
kodiutils.container_refresh()
@ -64,19 +66,34 @@ class BackgroundService(Monitor):
""" Update the metadata for the listings """
from resources.lib.modules.metadata import Metadata
# Clear outdated metadata
kodiutils.invalidate_cache(self.cache_expiry)
def update_status(_i, _total):
""" Allow to cancel the background job """
return self.abortRequested() or not kodiutils.get_setting_bool('metadata_update')
# Clear metadata that has expired for 30 days
self._remove_expired_metadata(30 * 24 * 60 * 60)
# Fetch new metadata
success = Metadata().fetch_metadata(callback=update_status)
# Update metadata_last_updated
if success:
kodiutils.set_setting('metadata_last_updated', str(int(time())))
@staticmethod
def _remove_expired_metadata(keep_expired=None):
""" Clear the cache """
path = kodiutils.get_cache_path()
if not os.path.exists(path):
return
now = time()
for filename in os.listdir(path):
fullpath = path + filename
if keep_expired and os.stat(fullpath).st_mtime + keep_expired > now:
continue
os.unlink(fullpath)
def run():
""" Run the BackgroundService """

20
resources/lib/viervijfzes/auth.py
View File

@ -5,9 +5,9 @@ from __future__ import absolute_import, division, unicode_literals
import json
import logging
import os
import time
from resources.lib import kodiutils
from resources.lib.viervijfzes.auth_awsidp import AwsIdp, InvalidLoginException, AuthenticationException
_LOGGER = logging.getLogger('auth-api')
@ -21,18 +21,18 @@ class AuthApi:
TOKEN_FILE = 'auth-tokens.json'
def __init__(self, username, password):
def __init__(self, username, password, token_path):
""" Initialise object """
self._username = username
self._password = password
self._cache_dir = kodiutils.get_tokens_path()
self._token_path = token_path
self._id_token = None
self._expiry = 0
self._refresh_token = None
# Load tokens from cache
try:
with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'rb') as fdesc:
with open(self._token_path + self.TOKEN_FILE, 'rb') as fdesc:
data_json = json.loads(fdesc.read())
self._id_token = data_json.get('id_token')
self._refresh_token = data_json.get('refresh_token')
@ -72,9 +72,9 @@ class AuthApi:
self._expiry = now + 3600
# Store new tokens in cache
if not kodiutils.exists(self._cache_dir):
kodiutils.mkdirs(self._cache_dir)
with kodiutils.open_file(self._cache_dir + self.TOKEN_FILE, 'wb') as fdesc:
if not os.path.exists(self._token_path):
os.mkdir(self._token_path)
with open(self._token_path + self.TOKEN_FILE, 'wb') as fdesc:
data = json.dumps(dict(
id_token=self._id_token,
refresh_token=self._refresh_token,
@ -84,10 +84,10 @@ class AuthApi:
return self._id_token
@staticmethod
def clear_tokens():
def clear_tokens(self):
""" Remove the cached tokens. """
kodiutils.delete(kodiutils.get_tokens_path() + AuthApi.TOKEN_FILE)
if os.path.exists(self._token_path + AuthApi.TOKEN_FILE):
os.unlink(self._token_path + AuthApi.TOKEN_FILE)
@staticmethod
def _authenticate(username, password):

160
resources/lib/viervijfzes/content.py
View File

@ -5,13 +5,14 @@ from __future__ import absolute_import, division, unicode_literals
import json
import logging
import os
import re
import time
from datetime import datetime
from six.moves.html_parser import HTMLParser
import requests
from resources.lib import kodiutils
from resources.lib.viervijfzes import CHANNELS
_LOGGER = logging.getLogger('content-api')
@ -141,33 +142,49 @@ class ContentApi:
'zes': 'https://www.zestv.be/api',
}
def __init__(self, auth=None):
def __init__(self, auth=None, cache_path=None):
""" Initialise object """
self._session = requests.session()
self._auth = auth
self._cache_path = cache_path
def get_programs(self, channel):
def get_programs(self, channel, cache=CACHE_AUTO):
""" Get a list of all programs of the specified channel.
:type channel: str
:type cache: str
:rtype list[Program]
NOTE: This function doesn't use an API.
"""
if channel not in CHANNELS:
raise Exception('Unknown channel %s' % channel)
# Load webpage
data = self._get_url(CHANNELS[channel]['url'])
def update():
""" Fetch the program listing by scraping """
# Load webpage
raw_html = self._get_url(CHANNELS[channel]['url'])
# Parse programs
parser = HTMLParser()
regex_programs = re.compile(r'<a class="program-overview__link" href="(?P<path>[^"]+)">\s+'
r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?'
r'</a>', re.DOTALL)
# Parse programs
parser = HTMLParser()
regex_programs = re.compile(r'<a class="program-overview__link" href="(?P<path>[^"]+)">\s+'
r'<span class="program-overview__title">\s+(?P<title>[^<]+)</span>.*?'
r'</a>', re.DOTALL)
data = {
item.group('path').lstrip('/'): parser.unescape(item.group('title').strip())
for item in regex_programs.finditer(raw_html)
}
if not data:
raise Exception('No programs found for %s' % channel)
return data
# Fetch listing from cache or update if needed
data = self._handle_cache(key=['programs', channel], cache_mode=cache, update=update, ttl=30 * 5)
if not data:
return []
programs = []
for item in regex_programs.finditer(data):
path = item.group('path').lstrip('/')
for path in data:
title = data[path]
program = self.get_program(channel, path, CACHE_ONLY) # Get program details, but from cache only
if program:
# Use program with metadata from cache
@ -176,7 +193,7 @@ class ContentApi:
# Use program with the values that we've parsed from the page
programs.append(Program(channel=channel,
path=path,
title=parser.unescape(item.group('title').strip())))
title=title))
return programs
def get_program(self, channel, path, cache=CACHE_AUTO):
@ -185,20 +202,12 @@ class ContentApi:
:type path: str
:type cache: int
:rtype Program
NOTE: This function doesn't use an API.
"""
if channel not in CHANNELS:
raise Exception('Unknown channel %s' % channel)
if cache in [CACHE_AUTO, CACHE_ONLY]:
# Try to fetch from cache
data = kodiutils.get_cache(['program', channel, path])
if data is None and cache == CACHE_ONLY:
return None
else:
data = None
if data is None:
def update():
""" Fetch the program metadata by scraping """
# Fetch webpage
page = self._get_url(CHANNELS[channel]['url'] + '/' + path)
@ -207,46 +216,15 @@ class ContentApi:
json_data = HTMLParser().unescape(regex_program.search(page).group(1))
data = json.loads(json_data)['data']
# Store response in cache
kodiutils.set_cache(['program', channel, path], data)
return data
# Fetch listing from cache or update if needed
data = self._handle_cache(key=['program', channel, path], cache_mode=cache, update=update)
program = self._parse_program_data(data)
return program
def get_program_by_uuid(self, uuid, cache=CACHE_AUTO):
""" Get a Program object.
:type uuid: str
:type cache: int
:rtype Program
"""
if cache in [CACHE_AUTO, CACHE_ONLY]:
# Try to fetch from cache
data = kodiutils.get_cache(['program', uuid])
if data is None and cache == CACHE_ONLY:
return None
else:
data = None
if data is None:
# Fetch from API
response = self._get_url(self.API_ENDPOINT + '/content/%s' % uuid, authentication=True)
data = json.loads(response)
if not data:
raise UnavailableException()
# Store response in cache
kodiutils.set_cache(['program', uuid], data)
return Program(
uuid=uuid,
path=data['url']['S'].strip('/'),
title=data['label']['S'],
description=data['description']['S'],
cover=data['image']['S'],
)
def get_episode(self, channel, path):
""" Get a Episode object from the specified page.
:type channel: str
@ -295,6 +273,9 @@ class ContentApi:
:type data: dict
:rtype Program
"""
if data is None:
return None
# Create Program info
program = Program(
uuid=data['id'],
@ -385,3 +366,62 @@ class ContentApi:
raise Exception('Could not fetch data')
return response.text
def _handle_cache(self, key, cache_mode, update, ttl=30 * 24 * 60 * 60):
""" Fetch something from the cache, and update if needed """
if cache_mode in [CACHE_AUTO, CACHE_ONLY]:
# Try to fetch from cache
data = self._get_cache(key)
if data is None and cache_mode == CACHE_ONLY:
return None
else:
data = None
if data is None:
try:
# Fetch fresh data
_LOGGER.debug('Fetching fresh data for key %s', '.'.join(key))
data = update()
if data:
# Store fresh response in cache
self._set_cache(key, data, ttl)
except Exception as exc: # pylint: disable=broad-except
_LOGGER.warning('Something went wrong when refreshing live data: %s. Using expired cached values.', exc)
data = self._get_cache(key, allow_expired=True)
return data
def _get_cache(self, key, allow_expired=False):
""" Get an item from the cache """
filename = '.'.join(key) + '.json'
fullpath = self._cache_path + filename
if not os.path.exists(fullpath):
return None
if not allow_expired and os.stat(fullpath).st_mtime < time.time():
return None
with open(fullpath, 'r') as fdesc:
try:
_LOGGER.debug('Fetching %s from cache', filename)
value = json.load(fdesc)
return value
except (ValueError, TypeError):
return None
def _set_cache(self, key, data, ttl):
""" Store an item in the cache """
filename = '.'.join(key) + '.json'
fullpath = self._cache_path + filename
if not os.path.exists(self._cache_path):
os.mkdir(self._cache_path)
with open(fullpath, 'w') as fdesc:
_LOGGER.debug('Storing to cache as %s', filename)
json.dump(data, fdesc)
# Set TTL by modifying modification date
deadline = int(time.time()) + ttl
os.utime(fullpath, (deadline, deadline))

1
resources/settings.xml
View File

@ -10,6 +10,5 @@
<setting label="30827" type="lsep"/> <!-- Metadata -->
<setting label="30829" type="bool" id="metadata_update" default="true" subsetting="true"/>
<setting label="30831" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/update)"/>
<setting label="30833" type="action" action="RunPlugin(plugin://plugin.video.viervijfzes/metadata/clean)"/>
</category>
</settings>

7
test/test_api.py
View File

@ -18,8 +18,8 @@ _LOGGER = logging.getLogger('test-api')
class TestApi(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestApi, self).__init__(*args, **kwargs)
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
self._api = ContentApi(auth)
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
self._api = ContentApi(auth, cache_path=kodiutils.get_cache_path())
def test_programs(self):
for channel in ['vier', 'vijf', 'zes']:
@ -40,9 +40,6 @@ class TestApi(unittest.TestCase):
program = self._api.get_program('vier', 'auwch')
self.assertIsInstance(program, Program)
program_by_uuid = self._api.get_program_by_uuid(program.uuid)
self.assertIsInstance(program_by_uuid, Program)
episode = program.episodes[0]
video = self._api.get_stream_by_uuid(episode.uuid)
self.assertTrue(video)

6
test/test_auth.py
View File

@ -20,16 +20,16 @@ class TestAuth(unittest.TestCase):
@unittest.skipUnless(kodiutils.get_setting('username') and kodiutils.get_setting('password'), 'Skipping since we have no credentials.')
def test_login(self):
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'), kodiutils.get_tokens_path())
# Clear any cache we have
AuthApi.clear_tokens()
auth.clear_tokens()
# We should get a token by logging in
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
token = auth.get_token()
self.assertTrue(token)
# Test it a second time, it should go from memory now
auth = AuthApi(kodiutils.get_setting('username'), kodiutils.get_setting('password'))
token = auth.get_token()
self.assertTrue(token)

3
test/test_epg.py
View File

@ -9,6 +9,7 @@ import logging
import unittest
from datetime import date
from resources.lib import kodiutils
from resources.lib.viervijfzes.content import ContentApi, Episode
from resources.lib.viervijfzes.epg import EpgApi, EpgProgram
@ -48,7 +49,7 @@ class TestEpg(unittest.TestCase):
epg_program = [program for program in epg_programs if program.video_url][0]
# Lookup the Episode data since we don't have an UUID
api = ContentApi()
api = ContentApi(cache_path=kodiutils.get_cache_path())
episode = api.get_episode(epg_program.channel, epg_program.video_url)
self.assertIsInstance(episode, Episode)

2
test/test_routing.py
View File

@ -57,7 +57,7 @@ class TestRouting(unittest.TestCase):
routing.run([routing.url_for(addon.show_tvguide_detail, channel='vier', date='today'), '0', ''])
def test_metadata_update(self):
routing.run([routing.url_for(addon.metadata_clean), '0', ''])
routing.run([routing.url_for(addon.metadata_update), '0', ''])
if __name__ == '__main__':

4
test/xbmcgui.py
View File

@ -129,9 +129,9 @@ class DialogProgress:
print('\033[37;44;1mPROGRESS:\033[35;49;1m [%s] \033[37;1m%s\033[39;0m' % (heading, line1))
sys.stdout.flush()
@staticmethod
def iscanceled():
def iscanceled(self):
"""A stub implementation for the xbmcgui DialogProgress class iscanceled() method"""
return self.percentage > 5 # Cancel at 5%
def update(self, percentage, line1=None, line2=None, line3=None):
"""A stub implementation for the xbmcgui DialogProgress class update() method"""