diff options
Diffstat (limited to 'plugin.video.mediathekview/resources/lib')
24 files changed, 3362 insertions, 0 deletions
diff --git a/plugin.video.mediathekview/resources/lib/__init__.py b/plugin.video.mediathekview/resources/lib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/__init__.py diff --git a/plugin.video.mediathekview/resources/lib/base/Logger.py b/plugin.video.mediathekview/resources/lib/base/Logger.py new file mode 100644 index 0000000..3d74680 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/base/Logger.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ + +# -- Classes ------------------------------------------------ +class Logger( object ): + def __init__( self, name, version, topic = None ): + self.name = name + self.version = version + self.setTopic( topic ) + + def getNewLogger( self, topic = None ): + pass + + def setTopic( self, topic = None ): + if topic == None: + self.prefix = '[%s-%s]: ' % ( self.name, self.version ) + else: + self.prefix = '[%s-%s:%s]: ' % ( self.name, self.version, topic ) + + def debug( self, message, *args ): + pass + + def info( self, message, *args ): + pass + + def warn( self, message, *args ): + pass + + def error( self, message, *args ): + pass diff --git a/plugin.video.mediathekview/resources/lib/base/__init__.py b/plugin.video.mediathekview/resources/lib/base/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/base/__init__.py diff --git a/plugin.video.mediathekview/resources/lib/channel.py b/plugin.video.mediathekview/resources/lib/channel.py new file mode 100644 index 0000000..d142383 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/channel.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ + +# -- Classes ------------------------------------------------ +class Channel( object ): + def __init__( self ): + self.id = 0 + self.channel = ''
\ No newline at end of file diff --git a/plugin.video.mediathekview/resources/lib/channelui.py b/plugin.video.mediathekview/resources/lib/channelui.py new file mode 100644 index 0000000..36743e8 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/channelui.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import sys, urllib +import xbmcplugin, xbmcgui + +from resources.lib.channel import Channel + +# -- Classes ------------------------------------------------ +class ChannelUI( Channel ): + def __init__( self, handle, sortmethods = None, nextdir = 'initial' ): + self.base_url = sys.argv[0] + self.nextdir = nextdir + self.handle = handle + self.sortmethods = sortmethods if sortmethods is not None else [ xbmcplugin.SORT_METHOD_TITLE ] + self.count = 0 + + def Begin( self ): + for method in self.sortmethods: + xbmcplugin.addSortMethod( self.handle, method ) + + def Add( self, altname = None ): + resultingname = self.channel if self.count == 0 else '%s (%d)' % ( self.channel, self.count, ) + li = xbmcgui.ListItem( label = resultingname if altname is None else altname ) + xbmcplugin.addDirectoryItem( + handle = self.handle, + url = self.build_url( { + 'mode': self.nextdir, + 'channel': self.id + } ), + listitem = li, + isFolder = True + ) + + def End( self ): + xbmcplugin.endOfDirectory( self.handle ) + + def build_url( self, query ): + return self.base_url + '?' + urllib.urlencode( query ) diff --git a/plugin.video.mediathekview/resources/lib/exceptions.py b/plugin.video.mediathekview/resources/lib/exceptions.py new file mode 100644 index 0000000..53d7f14 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/exceptions.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll +# + +class DatabaseCorrupted( RuntimeError ): + """This exception is raised when the database throws errors during update""" + +class DatabaseLost( RuntimeError ): + """This exception is raised when the connection to the database is lost during update""" diff --git a/plugin.video.mediathekview/resources/lib/film.py b/plugin.video.mediathekview/resources/lib/film.py new file mode 100644 index 0000000..b417078 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/film.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ + +# -- Classes ------------------------------------------------ +class Film( object ): + def __init__( self ): + self.id = 0 + self.title = u'' + self.show = u'' + self.channel = u'' + self.description = u'' + self.seconds = 0 + self.size = 0 + self.aired = u'' + self.url_sub = u'' + self.url_video = u'' + self.url_video_sd = u'' + self.url_video_hd = u'' diff --git a/plugin.video.mediathekview/resources/lib/filmui.py b/plugin.video.mediathekview/resources/lib/filmui.py new file mode 100644 index 0000000..af04798 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/filmui.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import xbmcplugin, xbmcgui + +from resources.lib.film import Film +from resources.lib.settings import Settings + +# -- Classes ------------------------------------------------ +class FilmUI( Film ): + def __init__( self, plugin, sortmethods = None ): + self.plugin = plugin + self.handle = plugin.addon_handle + self.settings = Settings() + self.sortmethods = sortmethods if sortmethods is not None else [ xbmcplugin.SORT_METHOD_TITLE, xbmcplugin.SORT_METHOD_DATE, xbmcplugin.SORT_METHOD_DURATION, xbmcplugin.SORT_METHOD_SIZE ] + self.showshows = False + self.showchannels = False + + def Begin( self, showshows, showchannels ): + self.showshows = showshows + self.showchannels = showchannels + # xbmcplugin.setContent( self.handle, 'tvshows' ) + for method in self.sortmethods: + xbmcplugin.addSortMethod( self.handle, method ) + + def Add( self, alttitle = None, totalItems = None ): + # get the best url + videourl = self.url_video_hd if ( self.url_video_hd != "" and self.settings.preferhd ) else self.url_video if self.url_video != "" else self.url_video_sd + videohds = " (HD)" if ( self.url_video_hd != "" and self.settings.preferhd ) else "" + # exit if no url supplied + if videourl == "": + return + + if alttitle is not None: + resultingtitle = alttitle + else: + if self.showshows: + resultingtitle = self.show + ': ' + self.title + else: + resultingtitle = self.title + if self.showchannels: + resultingtitle += ' [' + self.channel + ']' + + infoLabels = { + 'title' : resultingtitle + videohds, + 'sorttitle' : resultingtitle, + 'tvshowtitle' : self.show, + 'plot' : self.description + } + + if self.size > 0: + infoLabels['size'] = self.size * 1024 * 1024 + + if self.seconds > 0: + infoLabels['duration'] = self.seconds + + if self.aired is not None: + airedstring = '%s' % self.aired + if airedstring[:4] != '1970': + infoLabels['date'] = airedstring[8:10] + '-' + airedstring[5:7] + '-' + airedstring[:4] + infoLabels['aired'] = airedstring + infoLabels['dateadded'] = airedstring + + li = xbmcgui.ListItem( resultingtitle, self.description ) + li.setInfo( type = 'video', infoLabels = infoLabels ) + li.setProperty( 'IsPlayable', 'true' ) + + # create context menu + contextmenu = [] + if self.size > 0: + # Download video + contextmenu.append( ( + self.plugin.language( 30921 ), + 'RunPlugin({})'.format( self.plugin.build_url( { 'mode': "download", 'id': self.id, 'quality': 1 } ) ) + ) ) + if self.url_video_hd: + # Download SD video + contextmenu.append( ( + self.plugin.language( 30923 ), + 'RunPlugin({})'.format( self.plugin.build_url( { 'mode': "download", 'id': self.id, 'quality': 2 } ) ) + ) ) + if self.url_video_sd: + # Download SD video + contextmenu.append( ( + self.plugin.language( 30922 ), + 'RunPlugin({})'.format( self.plugin.build_url( { 'mode': "download", 'id': self.id, 'quality': 0 } ) ) + ) ) + # Add to queue + # TODO: Enable later +# contextmenu.append( ( +# self.plugin.language( 30924 ), +# 'RunPlugin({})'.format( self.plugin.build_url( { 'mode': "enqueue", 'id': self.id } ) ) +# ) ) + li.addContextMenuItems( contextmenu ) + + if totalItems is not None: + xbmcplugin.addDirectoryItem( + handle = self.handle, + url = videourl, + listitem = li, + isFolder = False, + totalItems = totalItems + ) + else: + xbmcplugin.addDirectoryItem( + handle = self.handle, + url = videourl, + listitem = li, + isFolder = False + ) + + def End( self ): + xbmcplugin.endOfDirectory( self.handle, cacheToDisc = False ) diff --git a/plugin.video.mediathekview/resources/lib/initialui.py b/plugin.video.mediathekview/resources/lib/initialui.py new file mode 100644 index 0000000..71d9aef --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/initialui.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import sys, urllib +import xbmcplugin, xbmcgui + +# -- Classes ------------------------------------------------ +class InitialUI( object ): + def __init__( self, handle, sortmethods = None ): + self.handle = handle + self.sortmethods = sortmethods if sortmethods is not None else [ xbmcplugin.SORT_METHOD_TITLE ] + self.channelid = 0 + self.initial = '' + self.count = 0 + + def Begin( self, channelid ): + self.channelid = channelid + for method in self.sortmethods: + xbmcplugin.addSortMethod( self.handle, method ) + + def Add( self, altname = None ): + if altname is None: + resultingname = '%s (%d)' % ( self.initial if self.initial != ' ' and self.initial != '' else ' No Title', self.count ) + else: + resultingname = altname + li = xbmcgui.ListItem( label = resultingname ) + xbmcplugin.addDirectoryItem( + handle = self.handle, + url = _build_url( { + 'mode': "shows", + 'channel': self.channelid, + 'initial': self.initial, + 'count': self.count + } ), + listitem = li, + isFolder = True + ) + + def End( self ): + xbmcplugin.endOfDirectory( self.handle ) + +# -- Functions ---------------------------------------------- + +def _build_url( query ): + return sys.argv[0] + '?' + urllib.urlencode( query ) diff --git a/plugin.video.mediathekview/resources/lib/kodi/KodiAddon.py b/plugin.video.mediathekview/resources/lib/kodi/KodiAddon.py new file mode 100644 index 0000000..f9c0cd7 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/kodi/KodiAddon.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import sys, urllib +import xbmc, xbmcgui, xbmcaddon, xbmcplugin + +from resources.lib.kodi.KodiLogger import KodiLogger + +# -- Classes ------------------------------------------------ +class KodiAddon( KodiLogger ): + + def __init__( self ): + self.addon = xbmcaddon.Addon() + self.addon_id = self.addon.getAddonInfo( 'id' ) + self.icon = self.addon.getAddonInfo( 'icon' ) + self.fanart = self.addon.getAddonInfo( 'fanart' ) + self.version = self.addon.getAddonInfo( 'version' ) + self.path = self.addon.getAddonInfo( 'path' ) + self.datapath = xbmc.translatePath( self.addon.getAddonInfo('profile').decode('utf-8') ) + self.language = self.addon.getLocalizedString + KodiLogger.__init__( self, self.addon_id, self.version ) + + def getSetting( self, setting_id ): + return self.addon.getSetting( setting_id ) + + def setSetting( self, setting_id, value ): + return self.addon.setSetting( setting_id, value ) + + def doAction( self, action ): + xbmc.executebuiltin( 'Action({})'.format( action ) ) + +class KodiService( KodiAddon ): + def __init__( self ): + KodiAddon.__init__( self ) + +class KodiPlugin( KodiAddon ): + def __init__( self ): + KodiAddon.__init__( self ) + self.base_url = sys.argv[0] + self.addon_handle = int( sys.argv[1] ) + + def build_url( self, query ): + return self.base_url + '?' + urllib.urlencode( query ) + + def runPlugin( self, params ): + xbmc.executebuiltin( 'RunPlugin({})'.format( self.build_url( params ) ) ) + + def addActionItem( self, name, params ): + self.addDirectoryItem( name, params, False ) + + def addFolderItem( self, name, params ): + self.addDirectoryItem( name, params, True ) + + def addDirectoryItem( self, name, params, isFolder ): + if isinstance( name, int ): + name = self.language( name ) + li = xbmcgui.ListItem( name ) + xbmcplugin.addDirectoryItem( + handle = self.addon_handle, + url = self.build_url( params ), + listitem = li, + isFolder = isFolder + ) + + def endOfDirectory( self, succeeded = True, updateListing = False, cacheToDisc = True ): + xbmcplugin.endOfDirectory( self.addon_handle, succeeded, updateListing, cacheToDisc ) diff --git a/plugin.video.mediathekview/resources/lib/kodi/KodiLogger.py b/plugin.video.mediathekview/resources/lib/kodi/KodiLogger.py new file mode 100644 index 0000000..c57ec84 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/kodi/KodiLogger.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import xbmc + +from resources.lib.base.Logger import Logger + +# -- Classes ------------------------------------------------ +class KodiLogger( Logger ): + + def __init__( self, name, version, topic = None ): + super( KodiLogger, self ).__init__( name, version, topic) + + def getNewLogger( self, topic = None ): + return KodiLogger( self.name, self.version, topic ) + + def debug( self, message, *args ): + self._log( xbmc.LOGDEBUG, message, *args ) + + def info( self, message, *args ): + self._log( xbmc.LOGNOTICE, message, *args ) + + def warn( self, message, *args ): + self._log( xbmc.LOGWARNING, message, *args ) + + def error( self, message, *args ): + self._log( xbmc.LOGERROR, message, *args ) + + def _log( self, level, message, *args ): + parts = [] + for arg in args: + part = arg + if isinstance( arg, basestring ): + part = arg # arg.decode('utf-8') + parts.append( part ) + xbmc.log( self.prefix + message.format( *parts ), level = level ) diff --git a/plugin.video.mediathekview/resources/lib/kodi/KodiUI.py b/plugin.video.mediathekview/resources/lib/kodi/KodiUI.py new file mode 100644 index 0000000..dee8dbb --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/kodi/KodiUI.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import xbmc, xbmcgui + +# -- Classes ------------------------------------------------ +class KodiUI( object ): + + def __init__( self ): + self.bgdialog = None + + def GetEnteredText( self, deftext = '', heading = '', hidden = False ): + keyboard = xbmc.Keyboard( deftext, heading, 1 if hidden else 0 ) + keyboard.doModal() + if keyboard.isConfirmed(): + return keyboard.getText() + return deftext + + def ShowNotification( self, heading, message, icon = xbmcgui.NOTIFICATION_INFO, time = 5000, sound = True ): + xbmcgui.Dialog().notification( heading, message, icon, time, sound ) + + def ShowWarning( self, heading, message, time = 5000, sound = True ): + xbmcgui.Dialog().notification( heading, message, xbmcgui.NOTIFICATION_WARNING, time, sound ) + + def ShowError( self, heading, message, time = 5000, sound = True ): + xbmcgui.Dialog().notification( heading, message, xbmcgui.NOTIFICATION_ERROR, time, sound ) + + def ShowBGDialog( self, heading = None, message = None ): + if self.bgdialog is None: + self.bgdialog = xbmcgui.DialogProgressBG() + self.bgdialog.create( heading, message ) + else: + self.bgdialog.update( 0, heading, message ) + + def UpdateBGDialog( self, percent, heading = None, message = None ): + if self.bgdialog is not None: + self.bgdialog.update( percent, heading, message ) + + def CloseBGDialog( self ): + if self.bgdialog is not None: + self.bgdialog.close() + del self.bgdialog + self.bgdialog = None + +class KodiBGDialog( object ): + def __init__( self ): + self.bgdialog= None + + def __del__( self ): + self.Close() + + def Create( self, heading = None, message = None ): + if self.bgdialog is None: + self.bgdialog = xbmcgui.DialogProgressBG() + self.bgdialog.create( heading, message ) + else: + self.bgdialog.update( 0, heading, message ) + + def Update( self, percent, heading = None, message = None ): + if self.bgdialog is not None: + self.bgdialog.update( percent, heading, message ) + + def UrlRetrieveHook( self, blockcount, blocksize, totalsize ): + downloaded = blockcount * blocksize + if totalsize > 0: + percent = int( (downloaded * 100) / totalsize ) + if self.bgdialog is not None: + self.bgdialog.update( percent ) + + def Close( self ): + if self.bgdialog is not None: + self.bgdialog.close() + del self.bgdialog + self.bgdialog = None diff --git a/plugin.video.mediathekview/resources/lib/kodi/__init__.py b/plugin.video.mediathekview/resources/lib/kodi/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/kodi/__init__.py diff --git a/plugin.video.mediathekview/resources/lib/mvupdate.py b/plugin.video.mediathekview/resources/lib/mvupdate.py new file mode 100644 index 0000000..7595417 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/mvupdate.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017-2018, Leo Moll + +# -- Imports ------------------------------------------------ +import os +import sys +import argparse +import datetime +import xml.etree.ElementTree as ET + +from resources.lib.base.Logger import Logger +from resources.lib.updater import MediathekViewUpdater + +# -- Classes ------------------------------------------------ +class Settings( object ): + def __init__( self, args ): + self.datapath = args.path if args.dbtype == 'sqlite' else './' + self.type = { 'sqlite' : '0', 'mysql' : '1' }.get( args.dbtype, '0' ) + if self.type == '1': + self.host = args.host + self.port = int( args.port ) + self.user = args.user + self.password = args.password + self.database = args.database + self.nofuture = True + self.minlength = 0 + self.groupshows = False + self.updenabled = True + self.updinterval = 3600 + +class AppLogger( Logger ): + + def __init__( self, name, version, topic = None, verbosity = 0 ): + super( AppLogger, self ).__init__( name, version, topic ) + self.verbosity = verbosity + + def getNewLogger( self, topic = None ): + return AppLogger( self.name, self.version, topic, self.verbosity ) + + def debug( self, message, *args ): + self._log( 2, message, *args ) + + def info( self, message, *args ): + self._log( 1, message, *args ) + + def warn( self, message, *args ): + self._log( 0, message, *args ) + + def error( self, message, *args ): + self._log( -1, message, *args ) + + def _log( self, level, message, *args ): + parts = [] + for arg in args: + part = arg + if isinstance( arg, basestring ): + part = arg # arg.decode('utf-8') + parts.append( part ) + output = '{} {} {}{}'.format( + datetime.datetime.now(), + { -1: 'ERROR', 0: 'WARNING', 1: 'NOTICE', 2: 'DEBUG' }.get( level, 2 ), + self.prefix, + message.format( *parts ) + ) + + if level < 0: + # error + sys.stderr.write( output + '\n' ) + sys.stderr.flush() + elif self.verbosity >= level: + # other message + print( output ) + +class Notifier( object ): + def __init__( self ): + pass + def GetEnteredText( self, deftext = '', heading = '', hidden = False ): + pass + def ShowNotification( self, heading, message, icon = None, time = 5000, sound = True ): + pass + def ShowWarning( self, heading, message, time = 5000, sound = True ): + pass + def ShowError( self, heading, message, time = 5000, sound = True ): + pass + def ShowBGDialog( self, heading = None, message = None ): + pass + def UpdateBGDialog( self, percent, heading = None, message = None ): + pass + def CloseBGDialog( self ): + pass + def ShowDatabaseError( self, err ): + pass + def ShowDownloadError( self, name, err ): + pass + def ShowMissingExtractorError( self ): + pass + def ShowDownloadProgress( self ): + pass + def UpdateDownloadProgress( self, percent, message = None ): + pass + def CloseDownloadProgress( self ): + pass + def ShowUpdateProgress( self ): + pass + def UpdateUpdateProgress( self, percent, count, channels, shows, movies ): + pass + def CloseUpdateProgress( self ): + pass + +class MediathekViewMonitor( object ): + def abortRequested( self ): + return False + +class UpdateApp( AppLogger ): + def __init__( self ): + try: + self.mypath = os.path.dirname( sys.argv[0] ) + tree = ET.parse( self.mypath + '/addon.xml' ) + version = tree.getroot().attrib['version'] + AppLogger.__init__( self, os.path.basename( sys.argv[0] ), version ) + except Exception: + AppLogger.__init__( self, os.path.basename( sys.argv[0] ), '0.0' ) + + def Init( self ): + parser = argparse.ArgumentParser( + formatter_class = argparse.ArgumentDefaultsHelpFormatter, + description = 'This is the standalone database updater. It downloads the current database update from mediathekview.de and integrates it in a local database' + ) + parser.add_argument( + '-v', '--verbose', + default = 0, + action = 'count', + help = 'Show progress messages' + ) + subparsers = parser.add_subparsers( + dest = 'dbtype', + help = 'target database' + ) + sqliteopts = subparsers.add_parser( 'sqlite', formatter_class = argparse.ArgumentDefaultsHelpFormatter ) + sqliteopts.add_argument( + '-p', '--path', + dest = 'path', + help = 'alternative path for the sqlite database', + default = './' + ) + mysqlopts = subparsers.add_parser( 'mysql', formatter_class = argparse.ArgumentDefaultsHelpFormatter ) + mysqlopts.add_argument( + '-H', '--host', + dest = 'host', + help = 'hostname or ip address', + default = 'localhost' + ) + mysqlopts.add_argument( + '-P', '--port', + dest = 'port', + help = 'connection port', + default = '3306' + ) + mysqlopts.add_argument( + '-u', '--user', + dest = 'user', + help = 'connection username', + default = 'mediathekview' + ) + mysqlopts.add_argument( + '-p', '--password', + dest = 'password', + help = 'connection password', + default = None + ) + mysqlopts.add_argument( + '-d', '--database', + dest = 'database', + default = 'mediathekview', + help = 'database name' + ) + self.args = parser.parse_args() + self.verbosity = self.args.verbose + + self.info( 'Startup' ) + self.settings = Settings( self.args ) + self.notifier = Notifier() + self.monitor = MediathekViewMonitor() + self.updater = MediathekViewUpdater( self.getNewLogger( 'MediathekViewUpdater' ), self.notifier, self.settings, self.monitor ) + self.updater.Init() + + def Run( self ): + self.info( 'Starting up...' ) + updateop = self.updater.GetCurrentUpdateOperation() + if updateop == 1: + # full update + self.info( 'Initiating full update...' ) + self.updater.Update( True ) + elif updateop == 2: + # differential update + self.info( 'Initiating differential update...' ) + self.updater.Update( False ) + self.info( 'Exiting...' ) + + def Exit( self ): + self.updater.Exit() diff --git a/plugin.video.mediathekview/resources/lib/mvutils.py b/plugin.video.mediathekview/resources/lib/mvutils.py new file mode 100644 index 0000000..d4c7663 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/mvutils.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017-2018, Leo Moll + +# -- Imports ------------------------------------------------ +import os +import stat +import string +import urllib2 +import xbmcvfs + +from contextlib import closing + +# -- Functions ---------------------------------------------- +def dir_exists( name ): + try: + s = os.stat( name ) + return stat.S_ISDIR( s.st_mode ) + except OSError: + return False + +def file_exists( name ): + try: + s = os.stat( name ) + return stat.S_ISREG( s.st_mode ) + except OSError: + return False + +def file_size( name ): + try: + s = os.stat( name ) + return s.st_size + except OSError: + return 0 + +def find_xz(): + for xzbin in [ '/bin/xz', '/usr/bin/xz', '/usr/local/bin/xz' ]: + if file_exists( xzbin ): + return xzbin + return None + +def make_search_string( val ): + cset = string.letters + string.digits + ' _-#' + search = ''.join( [ c for c in val if c in cset ] ) + return search.upper().strip() + +def make_duration( val ): + if val == "00:00:00": + return None + elif val is None: + return None + x = val.split( ':' ) + if len( x ) != 3: + return None + return int( x[0] ) * 3600 + int( x[1] ) * 60 + int( x[2] ) + +def cleanup_filename( val ): + cset = string.letters + string.digits + u' _-#äöüÄÖÜßáàâéèêíìîóòôúùûÁÀÉÈÍÌÓÒÚÙçÇœ' + search = ''.join( [ c for c in val if c in cset ] ) + return search.strip() + +def url_retrieve( url, filename, reporthook, chunk_size = 8192 ): + with closing( urllib2.urlopen( url ) ) as u, closing( open( filename, 'wb' ) ) as f: + total_size = int( u.info().getheader( 'Content-Length' ).strip() ) if u.info() and u.info().getheader( 'Content-Length' ) else 0 + total_chunks = 0 + + while True: + reporthook( total_chunks, chunk_size, total_size ) + chunk = u.read( chunk_size ) + if not chunk: + break + f.write( chunk ) + total_chunks += 1 + return ( filename, [], ) + +def url_retrieve_vfs( videourl, filename, reporthook, chunk_size = 8192 ): + with closing( urllib2.urlopen( videourl ) ) as u, closing( xbmcvfs.File( filename, 'wb' ) ) as f: + total_size = int( u.info().getheader( 'Content-Length' ).strip() ) if u.info() and u.info().getheader( 'Content-Length' ) else 0 + total_chunks = 0 + + while True: + reporthook( total_chunks, chunk_size, total_size ) + chunk = u.read( chunk_size ) + if not chunk: + break + f.write( chunk ) + total_chunks += 1 + return ( filename, [], ) diff --git a/plugin.video.mediathekview/resources/lib/notifier.py b/plugin.video.mediathekview/resources/lib/notifier.py new file mode 100644 index 0000000..d257a52 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/notifier.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import xbmcaddon + +from resources.lib.kodi.KodiUI import KodiUI + +# -- Classes ------------------------------------------------ +class Notifier( KodiUI ): + def __init__( self ): + super( Notifier, self ).__init__() + self.language = xbmcaddon.Addon().getLocalizedString + + def ShowDatabaseError( self, err ): + self.ShowError( self.language( 30951 ), '{}'.format( err ) ) + + def ShowDownloadError( self, name, err ): + self.ShowError( self.language( 30952 ), self.language( 30953 ).format( name, err ) ) + + def ShowMissingExtractorError( self ): + self.ShowError( self.language( 30952 ), self.language( 30954 ), time = 10000 ) + + def ShowLimitResults( self, maxresults ): + self.ShowNotification( self.language( 30980 ), self.language( 30981 ).format( maxresults ) ) + + def ShowDownloadProgress( self ): + self.ShowBGDialog( self.language( 30955 ) ) + + def UpdateDownloadProgress( self, percent, message = None ): + self.UpdateBGDialog( percent, message = message ) + + def CloseDownloadProgress( self ): + self.CloseBGDialog() + + def ShowUpdateProgress( self ): + self.ShowBGDialog( self.language( 30956 ) ) + + def UpdateUpdateProgress( self, percent, count, channels, shows, movies ): + message = self.language( 30957 ) % ( count, channels, shows, movies ) + self.UpdateBGDialog( percent, message = message ) + + def CloseUpdateProgress( self ): + self.CloseBGDialog() diff --git a/plugin.video.mediathekview/resources/lib/settings.py b/plugin.video.mediathekview/resources/lib/settings.py new file mode 100644 index 0000000..32bd95a --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/settings.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import xbmc,xbmcaddon + +# -- Classes ------------------------------------------------ +class Settings( object ): + def __init__( self ): + self.addon = xbmcaddon.Addon() + self.Reload() + + def Reload( self ): + self.datapath = xbmc.translatePath( self.addon.getAddonInfo('profile').decode('utf-8') ) + self.firstrun = self.addon.getSetting( 'firstrun' ) == 'true' + self.preferhd = self.addon.getSetting( 'quality' ) == 'true' + self.nofuture = self.addon.getSetting( 'nofuture' ) == 'true' + self.minlength = int( float( self.addon.getSetting( 'minlength' ) ) ) * 60 + self.groupshows = self.addon.getSetting( 'groupshows' ) == 'true' + self.maxresults = int( self.addon.getSetting( 'maxresults' ) ) + self.downloadpath = self.addon.getSetting( 'downloadpath' ) + self.type = self.addon.getSetting( 'dbtype' ) + self.host = self.addon.getSetting( 'dbhost' ) + self.port = int( self.addon.getSetting( 'dbport' ) ) + self.user = self.addon.getSetting( 'dbuser' ) + self.password = self.addon.getSetting( 'dbpass' ) + self.database = self.addon.getSetting( 'dbdata' ) + self.updenabled = self.addon.getSetting( 'updenabled' ) == 'true' + self.updinterval = int( float( self.addon.getSetting( 'updinterval' ) ) ) * 3600 + + def HandleFirstRun( self ): + if self.firstrun: + self.firstrun = False + self.addon.setSetting( 'firstrun', 'false' ) + return True + return False diff --git a/plugin.video.mediathekview/resources/lib/show.py b/plugin.video.mediathekview/resources/lib/show.py new file mode 100644 index 0000000..e0c9127 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/show.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ + +# -- Classes ------------------------------------------------ +class Show( object ): + def __init__( self ): + self.id = 0 + self.channelid = 0 + self.show = '' + self.channel = ''
\ No newline at end of file diff --git a/plugin.video.mediathekview/resources/lib/showui.py b/plugin.video.mediathekview/resources/lib/showui.py new file mode 100644 index 0000000..b0bf7f9 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/showui.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll and Dominik Schlösser +# + +# -- Imports ------------------------------------------------ +import sys, urllib +import xbmcplugin, xbmcgui + +from resources.lib.show import Show + +# -- Classes ------------------------------------------------ +class ShowUI( Show ): + def __init__( self, handle, sortmethods = None ): + self.base_url = sys.argv[0] + self.handle = handle + self.sortmethods = sortmethods if sortmethods is not None else [ xbmcplugin.SORT_METHOD_TITLE ] + self.querychannelid = 0 + + def Begin( self, channelid ): + self.querychannelid = channelid + for method in self.sortmethods: + xbmcplugin.addSortMethod( self.handle, method ) + + def Add( self, altname = None ): + if altname is not None: + resultingname = altname + elif self.querychannelid == '0': + resultingname = self.show + ' [' + self.channel + ']' + else: + resultingname = self.show + li = xbmcgui.ListItem( label = resultingname ) + xbmcplugin.addDirectoryItem( + handle = self.handle, + url = self.build_url( { + 'mode': "films", + 'show': self.id + } ), + listitem = li, + isFolder = True + ) + + def End( self ): + xbmcplugin.endOfDirectory( self.handle ) + + def build_url( self, query ): + return self.base_url + '?' + urllib.urlencode( query ) diff --git a/plugin.video.mediathekview/resources/lib/store.py b/plugin.video.mediathekview/resources/lib/store.py new file mode 100644 index 0000000..2312a3c --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/store.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll +# + +# -- Imports ------------------------------------------------ +from resources.lib.storemysql import StoreMySQL +from resources.lib.storesqlite import StoreSQLite + +# -- Classes ------------------------------------------------ +class Store( object ): + def __init__( self, logger, notifier, settings ): + self.logger = logger + self.notifier = notifier + self.settings = settings + # load storage engine + if settings.type == '0': + self.logger.info( 'Database driver: Internal (sqlite)' ) + self.db = StoreSQLite( logger.getNewLogger( 'StoreSQLite' ), notifier, self.settings ) + elif settings.type == '1': + self.logger.info( 'Database driver: External (mysql)' ) + self.db = StoreMySQL( logger.getNewLogger( 'StoreMySQL' ), notifier, self.settings ) + else: + self.logger.warn( 'Unknown Database driver selected' ) + self.db = None + + def Init( self, reset = False ): + if self.db is not None: + self.db.Init( reset ) + + def Exit( self ): + if self.db is not None: + self.db.Exit() + + def Search( self, search, filmui ): + if self.db is not None: + self.db.Search( search, filmui ) + + def SearchFull( self, search, filmui ): + if self.db is not None: + self.db.SearchFull( search, filmui ) + + def GetRecents( self, channelid, filmui ): + if self.db is not None: + self.db.GetRecents( channelid, filmui ) + + def GetLiveStreams( self, filmui ): + if self.db is not None: + self.db.GetLiveStreams( filmui ) + + def GetChannels( self, channelui ): + if self.db is not None: + self.db.GetChannels( channelui ) + + def GetRecentChannels( self, channelui ): + if self.db is not None: + self.db.GetRecentChannels( channelui ) + + def GetInitials( self, channelid, initialui ): + if self.db is not None: + self.db.GetInitials( channelid, initialui ) + + def GetShows( self, channelid, initial, showui ): + if self.db is not None: + self.db.GetShows( channelid, initial, showui ) + + def GetFilms( self, showid, filmui ): + if self.db is not None: + self.db.GetFilms( showid, filmui ) + + def RetrieveFilmInfo( self, filmid ): + if self.db is not None: + return self.db.RetrieveFilmInfo( filmid ) + + def GetStatus( self ): + if self.db is not None: + return self.db.GetStatus() + else: + return { + 'modified': int( time.time() ), + 'status': 'UNINIT', + 'lastupdate': 0, + 'filmupdate': 0, + 'fullupdate': 0, + 'add_chn': 0, + 'add_shw': 0, + 'add_mov': 0, + 'del_chn': 0, + 'del_shw': 0, + 'del_mov': 0, + 'tot_chn': 0, + 'tot_shw': 0, + 'tot_mov': 0 + } + + def UpdateStatus( self, status = None, lastupdate = None, filmupdate = None, fullupdate = None, add_chn = None, add_shw = None, add_mov = None, del_chn = None, del_shw = None, del_mov = None, tot_chn = None, tot_shw = None, tot_mov = None ): + if self.db is not None: + self.db.UpdateStatus( status, lastupdate, filmupdate, fullupdate, add_chn, add_shw, add_mov, del_chn, del_shw, del_mov, tot_chn, tot_shw, tot_mov ) + + def SupportsUpdate( self ): + if self.db is not None: + return self.db.SupportsUpdate() + return False + + def ftInit( self ): + if self.db is not None: + return self.db.ftInit() + return False + + def ftUpdateStart( self, full ): + if self.db is not None: + return self.db.ftUpdateStart( full ) + return ( 0, 0, 0, ) + + def ftUpdateEnd( self, delete ): + if self.db is not None: + return self.db.ftUpdateEnd( delete ) + return ( 0, 0, 0, 0, 0, 0, ) + + def ftInsertFilm( self, film, commit = True ): + if self.db is not None: + return self.db.ftInsertFilm( film, commit ) + return ( 0, 0, 0, 0, ) diff --git a/plugin.video.mediathekview/resources/lib/storemysql.py b/plugin.video.mediathekview/resources/lib/storemysql.py new file mode 100644 index 0000000..e2419cc --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/storemysql.py @@ -0,0 +1,913 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll +# + +# -- Imports ------------------------------------------------ +import time +import mysql.connector + +import resources.lib.mvutils as mvutils + +from resources.lib.film import Film + +# -- Classes ------------------------------------------------ +class StoreMySQL( object ): + def __init__( self, logger, notifier, settings ): + self.conn = None + self.logger = logger + self.notifier = notifier + self.settings = settings + # useful query fragments + self.sql_query_films = "SELECT film.id,`title`,`show`,`channel`,`description`,TIME_TO_SEC(`duration`) AS `seconds`,`size`,`aired`,`url_sub`,`url_video`,`url_video_sd`,`url_video_hd` FROM `film` LEFT JOIN `show` ON show.id=film.showid LEFT JOIN `channel` ON channel.id=film.channelid" + self.sql_query_filmcnt = "SELECT COUNT(*) FROM `film` LEFT JOIN `show` ON show.id=film.showid LEFT JOIN `channel` ON channel.id=film.channelid" + self.sql_cond_recent = "( TIMESTAMPDIFF(HOUR,`aired`,CURRENT_TIMESTAMP()) < 24 )" + self.sql_cond_nofuture = " AND ( ( `aired` IS NULL ) OR ( TIMESTAMPDIFF(HOUR,`aired`,CURRENT_TIMESTAMP()) > 0 ) )" if settings.nofuture else "" + self.sql_cond_minlength = " AND ( ( `duration` IS NULL ) OR ( TIME_TO_SEC(`duration`) >= %d ) )" % settings.minlength if settings.minlength > 0 else "" + + def Init( self, reset = False ): + self.logger.info( 'Using MySQL connector version {}', mysql.connector.__version__ ) + try: + self.conn = mysql.connector.connect( + host = self.settings.host, + port = self.settings.port, + user = self.settings.user, + password = self.settings.password + ) + self.conn.database = self.settings.database + except mysql.connector.Error as err: + if err.errno == mysql.connector.errorcode.ER_BAD_DB_ERROR: + self.logger.info( '=== DATABASE {} DOES NOT EXIST. TRYING TO CREATE IT ===', self.settings.database ) + self._handle_database_initialization() + return + self.conn = None + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def Exit( self ): + if self.conn is not None: + self.conn.close() + + def Search( self, search, filmui ): + self._Search_Condition( '( ( `title` LIKE "%%%s%%" ) OR ( `show` LIKE "%%%s%%" ) )' % ( search, search, ), filmui, True, True, self.settings.maxresults ) + + def SearchFull( self, search, filmui ): + self._Search_Condition( '( ( `title` LIKE "%%%s%%" ) OR ( `show` LIKE "%%%s%%" ) ) OR ( `description` LIKE "%%%s%%") )' % ( search, search, search ), filmui, True, True, self.settings.maxresults ) + + def GetRecents( self, channelid, filmui ): + sql_cond_channel = ' AND ( film.channelid=' + str( channelid ) + ' ) ' if channelid != '0' else '' + self._Search_Condition( self.sql_cond_recent + sql_cond_channel, filmui, True, False, 10000 ) + + def GetLiveStreams( self, filmui ): + self._Search_Condition( '( show.search="LIVESTREAM" )', filmui, False, False, 10000 ) + + def GetChannels( self, channelui ): + self._Channels_Condition( None, channelui ) + + def GetRecentChannels( self, channelui ): + self._Channels_Condition( self.sql_cond_recent, channelui ) + + def GetInitials( self, channelid, initialui ): + if self.conn is None: + return + try: + condition = 'WHERE ( `channelid`=' + str( channelid ) + ' ) ' if channelid != '0' else '' + self.logger.info( 'MySQL Query: {}', + 'SELECT LEFT(`search`,1) AS letter,COUNT(*) AS `count` FROM `show` ' + + condition + + 'GROUP BY LEFT(search,1)' + ) + cursor = self.conn.cursor() + cursor.execute( + 'SELECT LEFT(`search`,1) AS letter,COUNT(*) AS `count` FROM `show` ' + + condition + + 'GROUP BY LEFT(`search`,1)' + ) + initialui.Begin( channelid ) + for ( initialui.initial, initialui.count ) in cursor: + initialui.Add() + initialui.End() + cursor.close() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def GetShows( self, channelid, initial, showui ): + if self.conn is None: + return + try: + if channelid == '0' and self.settings.groupshows: + query = 'SELECT GROUP_CONCAT(show.id),GROUP_CONCAT(`channelid`),`show`,GROUP_CONCAT(`channel`) FROM `show` LEFT JOIN `channel` ON channel.id=show.channelid WHERE ( `show` LIKE "%s%%" ) GROUP BY `show`' % initial + elif channelid == '0': + query = 'SELECT show.id,show.channelid,show.show,channel.channel FROM `show` LEFT JOIN channel ON channel.id=show.channelid WHERE ( `show` LIKE "%s%%" )' % initial + else: + query = 'SELECT show.id,show.channelid,show.show,channel.channel FROM `show` LEFT JOIN channel ON channel.id=show.channelid WHERE ( `channelid`=%s ) AND ( `show` LIKE "%s%%" )' % ( channelid, initial ) + self.logger.info( 'MySQL Query: {}', query ) + cursor = self.conn.cursor() + cursor.execute( query ) + showui.Begin( channelid ) + for ( showui.id, showui.channelid, showui.show, showui.channel ) in cursor: + showui.Add() + showui.End() + cursor.close() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def GetFilms( self, showid, filmui ): + if self.conn is None: + return + if showid.find( ',' ) == -1: + # only one channel id + condition = '( `showid`=%s )' % showid + showchannels = False + else: + # multiple channel ids + condition = '( `showid` IN ( %s ) )' % showid + showchannels = True + self._Search_Condition( condition, filmui, False, showchannels, 10000 ) + + def _Channels_Condition( self, condition, channelui): + if self.conn is None: + return + try: + if condition is None: + query = 'SELECT `id`,`channel`,0 AS `count` FROM `channel`' + else: + query = 'SELECT channel.id AS `id`,`channel`,COUNT(*) AS `count` FROM `film` LEFT JOIN `channel` ON channel.id=film.channelid WHERE ' + condition + ' GROUP BY channel.id' + self.logger.info( 'MySQL Query: {}', query ) + cursor = self.conn.cursor() + cursor.execute( query ) + channelui.Begin() + for ( channelui.id, channelui.channel, channelui.count ) in cursor: + channelui.Add() + channelui.End() + cursor.close() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def _Search_Condition( self, condition, filmui, showshows, showchannels, maxresults ): + if self.conn is None: + return + try: + self.logger.info( 'MySQL Query: {}', + self.sql_query_films + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + ) + cursor = self.conn.cursor() + cursor.execute( + self.sql_query_filmcnt + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + + ' LIMIT {}'.format( maxresults + 1 ) if maxresults else '' + ) + ( results, ) = cursor.fetchone() + if maxresults and results > maxresults: + self.notifier.ShowLimitResults( maxresults ) + cursor.execute( + self.sql_query_films + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + + ' LIMIT {}'.format( maxresults + 1 ) if maxresults else '' + ) + filmui.Begin( showshows, showchannels ) + for ( filmui.id, filmui.title, filmui.show, filmui.channel, filmui.description, filmui.seconds, filmui.size, filmui.aired, filmui.url_sub, filmui.url_video, filmui.url_video_sd, filmui.url_video_hd ) in cursor: + filmui.Add( totalItems = results ) + filmui.End() + cursor.close() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def RetrieveFilmInfo( self, filmid ): + if self.conn is None: + return None + try: + condition = '( film.id={} )'.format( filmid ) + self.logger.info( 'MySQL Query: {}', + self.sql_query_films + + ' WHERE ' + + condition + ) + cursor = self.conn.cursor() + cursor.execute( + self.sql_query_films + + ' WHERE ' + + condition + ) + film = Film() + for ( film.id, film.title, film.show, film.channel, film.description, film.seconds, film.size, film.aired, film.url_sub, film.url_video, film.url_video_sd, film.url_video_hd ) in cursor: + cursor.close() + return film + cursor.close() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return None + + def GetStatus( self ): + status = { + 'modified': int( time.time() ), + 'status': '', + 'lastupdate': 0, + 'filmupdate': 0, + 'fullupdate': 0, + 'add_chn': 0, + 'add_shw': 0, + 'add_mov': 0, + 'del_chn': 0, + 'del_shw': 0, + 'del_mov': 0, + 'tot_chn': 0, + 'tot_shw': 0, + 'tot_mov': 0 + } + if self.conn is None: + status['status'] = "UNINIT" + return status + try: + cursor = self.conn.cursor() + cursor.execute( 'SELECT * FROM `status` LIMIT 1' ) + r = cursor.fetchall() + cursor.close() + self.conn.commit() + if len( r ) == 0: + status['status'] = "NONE" + return status + status['modified'] = r[0][0] + status['status'] = r[0][1] + status['lastupdate'] = r[0][2] + status['filmupdate'] = r[0][3] + status['fullupdate'] = r[0][4] + status['add_chn'] = r[0][5] + status['add_shw'] = r[0][6] + status['add_mov'] = r[0][7] + status['del_chn'] = r[0][8] + status['del_shw'] = r[0][9] + status['del_mov'] = r[0][10] + status['tot_chn'] = r[0][11] + status['tot_shw'] = r[0][12] + status['tot_mov'] = r[0][13] + return status + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + status['status'] = "UNINIT" + return status + + def UpdateStatus( self, status = None, lastupdate = None, filmupdate = None, fullupdate = None, add_chn = None, add_shw = None, add_mov = None, del_chn = None, del_shw = None, del_mov = None, tot_chn = None, tot_shw = None, tot_mov = None ): + if self.conn is None: + return + new = self.GetStatus() + old = new['status'] + if status is not None: + new['status'] = status + if lastupdate is not None: + new['lastupdate'] = lastupdate + if filmupdate is not None: + new['filmupdate'] = filmupdate + if fullupdate is not None: + new['fullupdate'] = fullupdate + if add_chn is not None: + new['add_chn'] = add_chn + if add_shw is not None: + new['add_shw'] = add_shw + if add_mov is not None: + new['add_mov'] = add_mov + if del_chn is not None: + new['del_chn'] = del_chn + if del_shw is not None: + new['del_shw'] = del_shw + if del_mov is not None: + new['del_mov'] = del_mov + if tot_chn is not None: + new['tot_chn'] = tot_chn + if tot_shw is not None: + new['tot_shw'] = tot_shw + if tot_mov is not None: + new['tot_mov'] = tot_mov + # TODO: we should only write, if we have changed something... + new['modified'] = int( time.time() ) + try: + cursor = self.conn.cursor() + if old == "NONE": + # insert status + cursor.execute( + """ + INSERT INTO `status` ( + `modified`, + `status`, + `lastupdate`, + `filmupdate`, + `fullupdate`, + `add_chn`, + `add_shw`, + `add_mov`, + `del_chm`, + `del_shw`, + `del_mov`, + `tot_chn`, + `tot_shw`, + `tot_mov` + ) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + """, ( + new['modified'], + new['status'], + new['lastupdate'], + new['filmupdate'], + new['fullupdate'], + new['add_chn'], + new['add_shw'], + new['add_mov'], + new['del_chn'], + new['del_shw'], + new['del_mov'], + new['tot_chn'], + new['tot_shw'], + new['tot_mov'], + ) + ) + else: + # update status + cursor.execute( + """ + UPDATE `status` + SET `modified` = %s, + `status` = %s, + `lastupdate` = %s, + `filmupdate` = %s, + `fullupdate` = %s, + `add_chn` = %s, + `add_shw` = %s, + `add_mov` = %s, + `del_chm` = %s, + `del_shw` = %s, + `del_mov` = %s, + `tot_chn` = %s, + `tot_shw` = %s, + `tot_mov` = %s + """, ( + new['modified'], + new['status'], + new['lastupdate'], + new['filmupdate'], + new['fullupdate'], + new['add_chn'], + new['add_shw'], + new['add_mov'], + new['del_chn'], + new['del_shw'], + new['del_mov'], + new['tot_chn'], + new['tot_shw'], + new['tot_mov'], + ) + ) + cursor.close() + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def SupportsUpdate( self ): + return True + + def ftInit( self ): + # prevent concurrent updating + cursor = self.conn.cursor() + cursor.execute( + """ + UPDATE `status` + SET `modified` = %s, + `status` = 'UPDATING' + WHERE ( `status` != 'UPDATING' ) + OR + ( `modified` < %s ) + """, ( + int( time.time() ), + int( time.time() ) - 86400 + ) + ) + retval = cursor.rowcount > 0 + self.conn.commit() + cursor.close() + self.ft_channel = None + self.ft_channelid = None + self.ft_show = None + self.ft_showid = None + return retval + + def ftUpdateStart( self, full ): + param = ( 1, ) if full else ( 0, ) + try: + cursor = self.conn.cursor() + cursor.callproc( 'ftUpdateStart', param ) + for result in cursor.stored_results(): + for ( cnt_chn, cnt_shw, cnt_mov ) in result: + cursor.close() + self.conn.commit() + return ( cnt_chn, cnt_shw, cnt_mov ) + # should never happen + cursor.close() + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return ( 0, 0, 0, ) + + def ftUpdateEnd( self, delete ): + param = ( 1, ) if delete else ( 0, ) + try: + cursor = self.conn.cursor() + cursor.callproc( 'ftUpdateEnd', param ) + for result in cursor.stored_results(): + for ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov ) in result: + cursor.close() + self.conn.commit() + return ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov ) + # should never happen + cursor.close() + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return ( 0, 0, 0, 0, 0, 0, ) + + def ftInsertFilm( self, film, commit ): + newchn = False + inschn = 0 + insshw = 0 + insmov = 0 + + # handle channel + if self.ft_channel != film['channel']: + # process changed channel + newchn = True + self.ft_channel = film['channel'] + ( self.ft_channelid, inschn ) = self._insert_channel( self.ft_channel ) + if self.ft_channelid == 0: + self.logger.info( 'Undefined error adding channel "{}"', self.ft_channel ) + return ( 0, 0, 0, 0, ) + + if newchn or self.ft_show != film['show']: + # process changed show + self.ft_show = film['show'] + ( self.ft_showid, insshw ) = self._insert_show( self.ft_channelid, self.ft_show, mvutils.make_search_string( self.ft_show ) ) + if self.ft_showid == 0: + self.logger.info( 'Undefined error adding show "{}"', self.ft_show ) + return ( 0, 0, 0, 0, ) + + try: + cursor = self.conn.cursor() + cursor.callproc( 'ftInsertFilm', ( + self.ft_channelid, + self.ft_showid, + film["title"], + mvutils.make_search_string( film['title'] ), + film["aired"], + film["duration"], + film["size"], + film["description"], + film["website"], + film["url_sub"], + film["url_video"], + film["url_video_sd"], + film["url_video_hd"], + film["airedepoch"], + ) ) + for result in cursor.stored_results(): + for ( filmid, insmov ) in result: + cursor.close() + if commit: + self.conn.commit() + return ( filmid, inschn, insshw, insmov ) + # should never happen + cursor.close() + if commit: + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return ( 0, 0, 0, 0, ) + + def _insert_channel( self, channel ): + try: + cursor = self.conn.cursor() + cursor.callproc( 'ftInsertChannel', ( channel, ) ) + for result in cursor.stored_results(): + for ( id, added ) in result: + cursor.close() + self.conn.commit() + return ( id, added ) + # should never happen + cursor.close() + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return ( 0, 0, ) + + def _insert_show( self, channelid, show, search ): + try: + cursor = self.conn.cursor() + cursor.callproc( 'ftInsertShow', ( channelid, show, search, ) ) + for result in cursor.stored_results(): + for ( idd, added ) in result: + cursor.close() + self.conn.commit() + return ( idd, added ) + # should never happen + cursor.close() + self.conn.commit() + except mysql.connector.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return ( 0, 0, ) + + def _handle_database_initialization( self ): + cursor = None + dbcreated = False + try: + cursor = self.conn.cursor() + cursor.execute( 'CREATE DATABASE `{}` DEFAULT CHARACTER SET utf8'.format( self.settings.database ) ) + dbcreated = True + self.conn.database = self.settings.database + cursor.execute( 'SET FOREIGN_KEY_CHECKS=0' ) + self.conn.commit() + cursor.execute( + """ +CREATE TABLE `channel` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `dtCreated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `touched` smallint(1) NOT NULL DEFAULT '1', + `channel` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `channel` (`channel`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + """ + ) + self.conn.commit() + + cursor.execute( """ +CREATE TABLE `film` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `dtCreated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `touched` smallint(1) NOT NULL DEFAULT '1', + `channelid` int(11) NOT NULL, + `showid` int(11) NOT NULL, + `title` varchar(255) NOT NULL, + `search` varchar(255) NOT NULL, + `aired` timestamp NULL DEFAULT NULL, + `duration` time DEFAULT NULL, + `size` int(11) DEFAULT NULL, + `description` longtext, + `website` varchar(384) DEFAULT NULL, + `url_sub` varchar(384) DEFAULT NULL, + `url_video` varchar(384) DEFAULT NULL, + `url_video_sd` varchar(384) DEFAULT NULL, + `url_video_hd` varchar(384) DEFAULT NULL, + `airedepoch` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `index_1` (`showid`,`title`), + KEY `index_2` (`channelid`,`title`), + KEY `dupecheck` (`channelid`,`showid`,`url_video`), + CONSTRAINT `FK_FilmChannel` FOREIGN KEY (`channelid`) REFERENCES `channel` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION, + CONSTRAINT `FK_FilmShow` FOREIGN KEY (`showid`) REFERENCES `show` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE TABLE `show` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `dtCreated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `touched` smallint(1) NOT NULL DEFAULT '1', + `channelid` int(11) NOT NULL, + `show` varchar(255) NOT NULL, + `search` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `show` (`show`), + KEY `search` (`search`), + KEY `combined_1` (`channelid`,`search`), + KEY `combined_2` (`channelid`,`show`), + CONSTRAINT `FK_ShowChannel` FOREIGN KEY (`channelid`) REFERENCES `channel` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE TABLE `status` ( + `modified` int(11) NOT NULL, + `status` varchar(255) NOT NULL, + `lastupdate` int(11) NOT NULL, + `filmupdate` int(11) NOT NULL, + `fullupdate` int(1) NOT NULL, + `add_chn` int(11) NOT NULL, + `add_shw` int(11) NOT NULL, + `add_mov` int(11) NOT NULL, + `del_chm` int(11) NOT NULL, + `del_shw` int(11) NOT NULL, + `del_mov` int(11) NOT NULL, + `tot_chn` int(11) NOT NULL, + `tot_shw` int(11) NOT NULL, + `tot_mov` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + """ ) + self.conn.commit() + + cursor.execute( 'INSERT INTO `status` VALUES (0,"IDLE",0,0,0,0,0,0,0,0,0,0,0,0);' ) + self.conn.commit() + + cursor.execute( 'SET FOREIGN_KEY_CHECKS=1' ) + self.conn.commit() + + cursor.execute( """ +CREATE PROCEDURE `ftInsertChannel`( + _channel VARCHAR(255) +) +BEGIN + DECLARE channelid_ INT(11); + DECLARE touched_ INT(1); + DECLARE added_ INT(1) DEFAULT 0; + + SELECT `id`, + `touched` + INTO channelid_, + touched_ + FROM `channel` + WHERE ( `channel`.`channel` = _channel ); + + IF ( channelid_ IS NULL ) THEN + INSERT INTO `channel` ( + `channel` + ) + VALUES ( + _channel + ); + SET channelid_ = LAST_INSERT_ID(); + SET added_ = 1; + ELSE + UPDATE `channel` + SET `touched` = 1 + WHERE ( `id` = channelid_ ); + END IF; + + SELECT channelid_ AS `id`, + added_ AS `added`; +END + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE PROCEDURE `ftInsertFilm`( + _channelid INT(11), + _showid INT(11), + _title VARCHAR(255), + _search VARCHAR(255), + _aired TIMESTAMP, + _duration TIME, + _size INT(11), + _description LONGTEXT, + _website VARCHAR(384), + _url_sub VARCHAR(384), + _url_video VARCHAR(384), + _url_video_sd VARCHAR(384), + _url_video_hd VARCHAR(384), + _airedepoch INT(11) +) +BEGIN + DECLARE id_ INT; + DECLARE added_ INT DEFAULT 0; + + SELECT `id` + INTO id_ + FROM `film` AS f + WHERE ( f.channelid = _channelid ) + AND + ( f.showid = _showid ) + AND + ( f.url_video = _url_video ); + + IF ( id_ IS NULL ) THEN + INSERT INTO `film` ( + `channelid`, + `showid`, + `title`, + `search`, + `aired`, + `duration`, + `size`, + `description`, + `website`, + `url_sub`, + `url_video`, + `url_video_sd`, + `url_video_hd`, + `airedepoch` + ) + VALUES ( + _channelid, + _showid, + _title, + _search, + IF(_aired = "1980-01-01 00:00:00", NULL, _aired), + IF(_duration = "00:00:00", NULL, _duration), + _size, + _description, + _website, + _url_sub, + _url_video, + _url_video_sd, + _url_video_hd, + _airedepoch + ); + SET id_ = LAST_INSERT_ID(); + SET added_ = 1; + ELSE + UPDATE `film` + SET `touched` = 1 + WHERE ( `id` = id_ ); + END IF; + SELECT id_ AS `id`, + added_ AS `added`; +END + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE PROCEDURE `ftInsertShow`( + _channelid INT(11), + _show VARCHAR(255), + _search VARCHAR(255) +) +BEGIN + DECLARE showid_ INT(11); + DECLARE touched_ INT(1); + DECLARE added_ INT(1) DEFAULT 0; + + SELECT `id`, + `touched` + INTO showid_, + touched_ + FROM `show` + WHERE ( `show`.`channelid` = _channelid ) + AND + ( `show`.`show` = _show ); + + IF ( showid_ IS NULL ) THEN + INSERT INTO `show` ( + `channelid`, + `show`, + `search` + ) + VALUES ( + _channelid, + _show, + _search + ); + SET showid_ = LAST_INSERT_ID(); + SET added_ = 1; + ELSE + UPDATE `show` + SET `touched` = 1 + WHERE ( `id` = showid_ ); + END IF; + + + SELECT showid_ AS `id`, + added_ AS `added`; +END + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE PROCEDURE `ftUpdateEnd`( + _full INT(1) +) +BEGIN + DECLARE del_chn_ INT DEFAULT 0; + DECLARE del_shw_ INT DEFAULT 0; + DECLARE del_mov_ INT DEFAULT 0; + DECLARE cnt_chn_ INT DEFAULT 0; + DECLARE cnt_shw_ INT DEFAULT 0; + DECLARE cnt_mov_ INT DEFAULT 0; + + IF ( _full = 1 ) THEN + SELECT COUNT(*) + INTO del_chn_ + FROM `channel` + WHERE ( `touched` = 0 ); + + SELECT COUNT(*) + INTO del_shw_ + FROM `show` + WHERE ( `touched` = 0 ); + + SELECT COUNT(*) + INTO del_mov_ + FROM `film` + WHERE ( `touched` = 0 ); + + DELETE FROM `show` + WHERE ( `show`.`touched` = 0 ) + AND + ( ( SELECT SUM( `film`.`touched` ) FROM `film` WHERE `film`.`showid` = `show`.`id` ) = 0 ); + + DELETE FROM `film` + WHERE ( `touched` = 0 ); + ELSE + SET del_chn_ = 0; + SET del_shw_ = 0; + SET del_mov_ = 0; + END IF; + + SELECT del_chn_ AS `del_chn`, + del_shw_ AS `del_shw`, + del_mov_ AS `del_mov`, + cnt_chn_ AS `cnt_chn`, + cnt_shw_ AS `cnt_shw`, + cnt_mov_ AS `cnt_mov`; +END + """ ) + self.conn.commit() + + cursor.execute( """ +CREATE PROCEDURE `ftUpdateStart`( + _full INT(1) +) +BEGIN + DECLARE cnt_chn_ INT DEFAULT 0; + DECLARE cnt_shw_ INT DEFAULT 0; + DECLARE cnt_mov_ INT DEFAULT 0; + + IF ( _full = 1 ) THEN + UPDATE `channel` + SET `touched` = 0; + + UPDATE `show` + SET `touched` = 0; + + UPDATE `film` + SET `touched` = 0; + END IF; + + SELECT COUNT(*) + INTO cnt_chn_ + FROM `channel`; + + SELECT COUNT(*) + INTO cnt_shw_ + FROM `show`; + + SELECT COUNT(*) + INTO cnt_mov_ + FROM `film`; + + SELECT cnt_chn_ AS `cnt_chn`, + cnt_shw_ AS `cnt_shw`, + cnt_mov_ AS `cnt_mov`; +END + """ ) + self.conn.commit() + + cursor.close() + self.logger.info( 'Database creation successfully completed' ) + except mysql.connector.Error as err: + self.logger.error( '=== DATABASE CREATION ERROR: {} ===', err ) + self.notifier.ShowDatabaseError( err ) + try: + if dbcreated: + cursor.execute( 'DROP DATABASE `{}`'.format( self.settings.database ) ) + self.conn.commit() + if cursor is not None: + cursor.close() + del cursor + if self.conn is not None: + self.conn.close() + self.conn = None + except mysql.connector.Error as err: + # should never happen + self.conn = None diff --git a/plugin.video.mediathekview/resources/lib/storesqlite.py b/plugin.video.mediathekview/resources/lib/storesqlite.py new file mode 100644 index 0000000..601db67 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/storesqlite.py @@ -0,0 +1,753 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll +# + +# -- Imports ------------------------------------------------ +import os, time +import sqlite3 + +import resources.lib.mvutils as mvutils + +from resources.lib.film import Film +from resources.lib.exceptions import DatabaseCorrupted + +# -- Classes ------------------------------------------------ +class StoreSQLite( object ): + def __init__( self, logger, notifier, settings ): + self.logger = logger + self.notifier = notifier + self.settings = settings + # internals + self.conn = None + self.dbfile = os.path.join( self.settings.datapath, 'filmliste-v1.db' ) + # useful query fragments + self.sql_query_films = "SELECT film.id,title,show,channel,description,duration,size,datetime(aired, 'unixepoch', 'localtime'),url_sub,url_video,url_video_sd,url_video_hd FROM film LEFT JOIN show ON show.id=film.showid LEFT JOIN channel ON channel.id=film.channelid" + self.sql_query_filmcnt = "SELECT COUNT(*) FROM film LEFT JOIN show ON show.id=film.showid LEFT JOIN channel ON channel.id=film.channelid" + self.sql_cond_recent = "( ( UNIX_TIMESTAMP() - aired ) <= 86400 )" + self.sql_cond_nofuture = " AND ( ( aired IS NULL ) OR ( ( UNIX_TIMESTAMP() - aired ) > 0 ) )" if settings.nofuture else "" + self.sql_cond_minlength = " AND ( ( duration IS NULL ) OR ( duration >= %d ) )" % settings.minlength if settings.minlength > 0 else "" + + def Init( self, reset = False ): + self.logger.info( 'Using SQLite version {}, python library sqlite3 version {}', sqlite3.sqlite_version, sqlite3.version ) + if not mvutils.dir_exists( self.settings.datapath ): + os.mkdir( self.settings.datapath ) + if reset == True or not mvutils.file_exists( self.dbfile ): + self.logger.info( '===== RESET: Database will be deleted and regenerated =====' ) + self._file_remove( self.dbfile ) + self.conn = sqlite3.connect( self.dbfile, timeout = 60 ) + self._handle_database_initialization() + else: + try: + self.conn = sqlite3.connect( self.dbfile, timeout = 60 ) + except sqlite3.DatabaseError as err: + self.logger.error( 'Error while opening database: {}. trying to fully reset the Database...', err ) + self.Init( reset = True ) + + self.conn.execute( 'pragma journal_mode=off' ) # 3x speed-up, check mode 'WAL' + self.conn.execute( 'pragma synchronous=off' ) # that is a bit dangerous :-) but faaaast + + self.conn.create_function( 'UNIX_TIMESTAMP', 0, UNIX_TIMESTAMP ) + self.conn.create_aggregate( 'GROUP_CONCAT', 1, GROUP_CONCAT ) + + def Exit( self ): + if self.conn is not None: + self.conn.close() + self.conn = None + + def Search( self, search, filmui ): + self._Search_Condition( '( ( title LIKE "%%%s%%" ) OR ( show LIKE "%%%s%%" ) )' % ( search, search ), filmui, True, True, self.settings.maxresults ) + + def SearchFull( self, search, filmui ): + self._Search_Condition( '( ( title LIKE "%%%s%%" ) OR ( show LIKE "%%%s%%" ) OR ( description LIKE "%%%s%%") )' % ( search, search, search ), filmui, True, True, self.settings.maxresults ) + + def GetRecents( self, channelid, filmui ): + sql_cond_channel = ' AND ( film.channelid=' + str( channelid ) + ' ) ' if channelid != '0' else '' + self._Search_Condition( self.sql_cond_recent + sql_cond_channel, filmui, True, False, 10000 ) + + def GetLiveStreams( self, filmui ): + self._Search_Condition( '( show.search="LIVESTREAM" )', filmui, False, False, 10000 ) + + def GetChannels( self, channelui ): + self._Channels_Condition( None, channelui ) + + def GetRecentChannels( self, channelui ): + self._Channels_Condition( self.sql_cond_recent, channelui ) + + def GetInitials( self, channelid, initialui ): + if self.conn is None: + return + try: + condition = 'WHERE ( channelid=' + str( channelid ) + ' ) ' if channelid != '0' else '' + self.logger.info( 'SQlite Query: {}', + 'SELECT SUBSTR(search,1,1),COUNT(*) FROM show ' + + condition + + 'GROUP BY LEFT(search,1)' + ) + cursor = self.conn.cursor() + cursor.execute( + 'SELECT SUBSTR(search,1,1),COUNT(*) FROM show ' + + condition + + 'GROUP BY SUBSTR(search,1,1)' + ) + initialui.Begin( channelid ) + for ( initialui.initial, initialui.count ) in cursor: + initialui.Add() + initialui.End() + cursor.close() + except sqlite3.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def GetShows( self, channelid, initial, showui ): + if self.conn is None: + return + try: + if channelid == '0' and self.settings.groupshows: + query = 'SELECT GROUP_CONCAT(show.id),GROUP_CONCAT(channelid),show,GROUP_CONCAT(channel) FROM show LEFT JOIN channel ON channel.id=show.channelid WHERE ( show LIKE "%s%%" ) GROUP BY show' % initial + elif channelid == '0': + query = 'SELECT show.id,show.channelid,show.show,channel.channel FROM show LEFT JOIN channel ON channel.id=show.channelid WHERE ( show LIKE "%s%%" )' % initial + else: + query = 'SELECT show.id,show.channelid,show.show,channel.channel FROM show LEFT JOIN channel ON channel.id=show.channelid WHERE ( channelid=%s ) AND ( show LIKE "%s%%" )' % ( channelid, initial ) + self.logger.info( 'SQLite Query: {}', query ) + cursor = self.conn.cursor() + cursor.execute( query ) + showui.Begin( channelid ) + for ( showui.id, showui.channelid, showui.show, showui.channel ) in cursor: + showui.Add() + showui.End() + cursor.close() + except sqlite3.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def GetFilms( self, showid, filmui ): + if self.conn is None: + return + if showid.find( ',' ) == -1: + # only one channel id + condition = '( showid=%s )' % showid + showchannels = False + else: + # multiple channel ids + condition = '( showid IN ( %s ) )' % showid + showchannels = True + self._Search_Condition( condition, filmui, False, showchannels, 10000 ) + + def _Channels_Condition( self, condition, channelui ): + if self.conn is None: + return + try: + if condition is None: + query = 'SELECT id,channel,0 AS `count` FROM channel' + else: + query = 'SELECT channel.id AS `id`,channel,COUNT(*) AS `count` FROM film LEFT JOIN channel ON channel.id=film.channelid WHERE ' + condition + ' GROUP BY channel' + self.logger.info( 'SQLite Query: {}', query ) + cursor = self.conn.cursor() + cursor.execute( query ) + channelui.Begin() + for ( channelui.id, channelui.channel, channelui.count ) in cursor: + channelui.Add() + channelui.End() + cursor.close() + except sqlite3.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def _Search_Condition( self, condition, filmui, showshows, showchannels, maxresults ): + if self.conn is None: + return + try: + maxresults = int( maxresults ) + self.logger.info( 'SQLite Query: {}', + self.sql_query_films + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + ) + cursor = self.conn.cursor() + cursor.execute( + self.sql_query_filmcnt + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + + ' LIMIT {}'.format( maxresults + 1 ) if maxresults else '' + ) + ( results, ) = cursor.fetchone() + if maxresults and results > maxresults: + self.notifier.ShowLimitResults( maxresults ) + cursor.execute( + self.sql_query_films + + ' WHERE ' + + condition + + self.sql_cond_nofuture + + self.sql_cond_minlength + + ' LIMIT {}'.format( maxresults ) if maxresults else '' + ) + filmui.Begin( showshows, showchannels ) + for ( filmui.id, filmui.title, filmui.show, filmui.channel, filmui.description, filmui.seconds, filmui.size, filmui.aired, filmui.url_sub, filmui.url_video, filmui.url_video_sd, filmui.url_video_hd ) in cursor: + filmui.Add( totalItems = results ) + filmui.End() + cursor.close() + except sqlite3.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + + def RetrieveFilmInfo( self, filmid ): + if self.conn is None: + return None + try: + condition = '( film.id={} )'.format( filmid ) + self.logger.info( 'SQLite Query: {}', + self.sql_query_films + + ' WHERE ' + + condition + ) + cursor = self.conn.cursor() + cursor.execute( + self.sql_query_films + + ' WHERE ' + + condition + ) + film = Film() + for ( film.id, film.title, film.show, film.channel, film.description, film.seconds, film.size, film.aired, film.url_sub, film.url_video, film.url_video_sd, film.url_video_hd ) in cursor: + cursor.close() + return film + cursor.close() + except sqlite3.Error as err: + self.logger.error( 'Database error: {}', err ) + self.notifier.ShowDatabaseError( err ) + return None + + def GetStatus( self ): + status = { + 'modified': int( time.time() ), + 'status': '', + 'lastupdate': 0, + 'filmupdate': 0, + 'fullupdate': 0, + 'add_chn': 0, + 'add_shw': 0, + 'add_mov': 0, + 'del_chn': 0, + 'del_shw': 0, + 'del_mov': 0, + 'tot_chn': 0, + 'tot_shw': 0, + 'tot_mov': 0 + } + if self.conn is None: + status['status'] = "UNINIT" + return status + self.conn.commit() + cursor = self.conn.cursor() + cursor.execute( 'SELECT * FROM `status` LIMIT 1' ) + r = cursor.fetchall() + cursor.close() + if len( r ) == 0: + status['status'] = "NONE" + return status + status['modified'] = r[0][0] + status['status'] = r[0][1] + status['lastupdate'] = r[0][2] + status['filmupdate'] = r[0][3] + status['fullupdate'] = r[0][4] + status['add_chn'] = r[0][5] + status['add_shw'] = r[0][6] + status['add_mov'] = r[0][7] + status['del_chn'] = r[0][8] + status['del_shw'] = r[0][9] + status['del_mov'] = r[0][10] + status['tot_chn'] = r[0][11] + status['tot_shw'] = r[0][12] + status['tot_mov'] = r[0][13] + return status + + def UpdateStatus( self, status = None, lastupdate = None, filmupdate = None, fullupdate = None, add_chn = None, add_shw = None, add_mov = None, del_chn = None, del_shw = None, del_mov = None, tot_chn = None, tot_shw = None, tot_mov = None ): + if self.conn is None: + return + new = self.GetStatus() + old = new['status'] + if status is not None: + new['status'] = status + if lastupdate is not None: + new['lastupdate'] = lastupdate + if filmupdate is not None: + new['filmupdate'] = filmupdate + if fullupdate is not None: + new['fullupdate'] = fullupdate + if add_chn is not None: + new['add_chn'] = add_chn + if add_shw is not None: + new['add_shw'] = add_shw + if add_mov is not None: + new['add_mov'] = add_mov + if del_chn is not None: + new['del_chn'] = del_chn + if del_shw is not None: + new['del_shw'] = del_shw + if del_mov is not None: + new['del_mov'] = del_mov + if tot_chn is not None: + new['tot_chn'] = tot_chn + if tot_shw is not None: + new['tot_shw'] = tot_shw + if tot_mov is not None: + new['tot_mov'] = tot_mov + # TODO: we should only write, if we have changed something... + new['modified'] = int( time.time() ) + cursor = self.conn.cursor() + if old == "NONE": + # insert status + cursor.execute( + """ + INSERT INTO `status` ( + `modified`, + `status`, + `lastupdate`, + `filmupdate`, + `fullupdate`, + `add_chn`, + `add_shw`, + `add_mov`, + `del_chm`, + `del_shw`, + `del_mov`, + `tot_chn`, + `tot_shw`, + `tot_mov` + ) + VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ? + ) + """, ( + new['modified'], + new['status'], + new['lastupdate'], + new['filmupdate'], + new['fullupdate'], + new['add_chn'], + new['add_shw'], + new['add_mov'], + new['del_chn'], + new['del_shw'], + new['del_mov'], + new['tot_chn'], + new['tot_shw'], + new['tot_mov'], + ) + ) + else: + # update status + cursor.execute( + """ + UPDATE `status` + SET `modified` = ?, + `status` = ?, + `lastupdate` = ?, + `filmupdate` = ?, + `fullupdate` = ?, + `add_chn` = ?, + `add_shw` = ?, + `add_mov` = ?, + `del_chm` = ?, + `del_shw` = ?, + `del_mov` = ?, + `tot_chn` = ?, + `tot_shw` = ?, + `tot_mov` = ? + """, ( + new['modified'], + new['status'], + new['lastupdate'], + new['filmupdate'], + new['fullupdate'], + new['add_chn'], + new['add_shw'], + new['add_mov'], + new['del_chn'], + new['del_shw'], + new['del_mov'], + new['tot_chn'], + new['tot_shw'], + new['tot_mov'], + ) + ) + cursor.close() + self.conn.commit() + + def SupportsUpdate( self ): + return True + + def ftInit( self ): + try: + # prevent concurrent updating + self.conn.commit() + cursor = self.conn.cursor() + cursor.execute( + """ + UPDATE `status` + SET `modified` = ?, + `status` = 'UPDATING' + WHERE ( `status` != 'UPDATING' ) + OR + ( `modified` < ? ) + """, ( + int( time.time() ), + int( time.time() ) - 86400 + ) + ) + retval = cursor.rowcount > 0 + self.conn.commit() + cursor.close() + self.ft_channel = None + self.ft_channelid = None + self.ft_show = None + self.ft_showid = None + return retval + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + + def ftUpdateStart( self, full ): + try: + cursor = self.conn.cursor() + if full: + cursor.executescript( """ + UPDATE `channel` + SET `touched` = 0; + + UPDATE `show` + SET `touched` = 0; + + UPDATE `film` + SET `touched` = 0; + """ ) + cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) + r1 = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show`' ) + r2 = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film`' ) + r3 = cursor.fetchone() + cursor.close() + self.conn.commit() + return ( r1[0], r2[0], r3[0], ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + + def ftUpdateEnd( self, delete ): + try: + cursor = self.conn.cursor() + cursor.execute( 'SELECT COUNT(*) FROM `channel` WHERE ( touched = 0 )' ) + ( del_chn, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show` WHERE ( touched = 0 )' ) + ( del_shw, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film` WHERE ( touched = 0 )' ) + ( del_mov, ) = cursor.fetchone() + if delete: + cursor.execute( 'DELETE FROM `show` WHERE ( show.touched = 0 ) AND ( ( SELECT SUM( film.touched ) FROM `film` WHERE film.showid = show.id ) = 0 )' ) + cursor.execute( 'DELETE FROM `film` WHERE ( touched = 0 )' ) + else: + del_chn = 0 + del_shw = 0 + del_mov = 0 + cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) + ( cnt_chn, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show`' ) + ( cnt_shw, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film`' ) + ( cnt_mov, ) = cursor.fetchone() + cursor.close() + self.conn.commit() + return ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov, ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + + def ftInsertFilm( self, film, commit ): + try: + cursor = self.conn.cursor() + newchn = False + inschn = 0 + insshw = 0 + insmov = 0 + + # handle channel + if self.ft_channel != film['channel']: + # process changed channel + newchn = True + cursor.execute( 'SELECT `id`,`touched` FROM `channel` WHERE channel.channel=?', ( film['channel'], ) ) + r = cursor.fetchall() + if len( r ) > 0: + # get the channel data + self.ft_channel = film['channel'] + self.ft_channelid = r[0][0] + if r[0][1] == 0: + # updated touched + cursor.execute( 'UPDATE `channel` SET `touched`=1 WHERE ( channel.id=? )', ( self.ft_channelid, ) ) + else: + # insert the new channel + inschn = 1 + cursor.execute( 'INSERT INTO `channel` ( `dtCreated`,`channel` ) VALUES ( ?,? )', ( int( time.time() ), film['channel'] ) ) + self.ft_channel = film['channel'] + self.ft_channelid = cursor.lastrowid + + # handle show + if newchn or self.ft_show != film['show']: + # process changed show + cursor.execute( 'SELECT `id`,`touched` FROM `show` WHERE ( show.channelid=? ) AND ( show.show=? )', ( self.ft_channelid, film['show'] ) ) + r = cursor.fetchall() + if len( r ) > 0: + # get the show data + self.ft_show = film['show'] + self.ft_showid = r[0][0] + if r[0][1] == 0: + # updated touched + cursor.execute( 'UPDATE `show` SET `touched`=1 WHERE ( show.id=? )', ( self.ft_showid, ) ) + else: + # insert the new show + insshw = 1 + cursor.execute( + """ + INSERT INTO `show` ( + `dtCreated`, + `channelid`, + `show`, + `search` + ) + VALUES ( + ?, + ?, + ?, + ? + ) + """, ( + int( time.time() ), + self.ft_channelid, film['show'], + mvutils.make_search_string( film['show'] ) + ) + ) + self.ft_show = film['show'] + self.ft_showid = cursor.lastrowid + + # check if the movie is there + cursor.execute( """ + SELECT `id`, + `touched` + FROM `film` + WHERE ( film.channelid = ? ) + AND + ( film.showid = ? ) + AND + ( film.url_video = ? ) + """, ( self.ft_channelid, self.ft_showid, film['url_video'] ) ) + r = cursor.fetchall() + if len( r ) > 0: + # film found + filmid = r[0][0] + if r[0][1] == 0: + # update touched + cursor.execute( 'UPDATE `film` SET `touched`=1 WHERE ( film.id=? )', ( filmid, ) ) + else: + # insert the new film + insmov = 1 + cursor.execute( + """ + INSERT INTO `film` ( + `dtCreated`, + `channelid`, + `showid`, + `title`, + `search`, + `aired`, + `duration`, + `size`, + `description`, + `website`, + `url_sub`, + `url_video`, + `url_video_sd`, + `url_video_hd` + ) + VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ? + ) + """, ( + int( time.time() ), + self.ft_channelid, + self.ft_showid, + film['title'], + mvutils.make_search_string( film['title'] ), + film['airedepoch'], + mvutils.make_duration( film['duration'] ), + film['size'], + film['description'], + film['website'], + film['url_sub'], + film['url_video'], + film['url_video_sd'], + film['url_video_hd'] + ) + ) + filmid = cursor.lastrowid + if commit: + self.conn.commit() + cursor.close() + return ( filmid, inschn, insshw, insmov ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + + def _handle_database_corruption( self, err ): + self.logger.error( 'Database error during critical operation: {} - Database will be rebuilt from scratch.', err ) + self.notifier.ShowDatabaseError( err ) + self.Exit() + self.Init( reset = True ) + + def _handle_database_initialization( self ): + self.conn.executescript( """ +PRAGMA foreign_keys = false; + +-- ---------------------------- +-- Table structure for channel +-- ---------------------------- +DROP TABLE IF EXISTS "channel"; +CREATE TABLE "channel" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "dtCreated" integer(11,0) NOT NULL DEFAULT 0, + "touched" integer(1,0) NOT NULL DEFAULT 1, + "channel" TEXT(255,0) NOT NULL +); + +-- ---------------------------- +-- Table structure for film +-- ---------------------------- +DROP TABLE IF EXISTS "film"; +CREATE TABLE "film" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "dtCreated" integer(11,0) NOT NULL DEFAULT 0, + "touched" integer(1,0) NOT NULL DEFAULT 1, + "channelid" INTEGER(11,0) NOT NULL, + "showid" INTEGER(11,0) NOT NULL, + "title" TEXT(255,0) NOT NULL, + "search" TEXT(255,0) NOT NULL, + "aired" integer(11,0), + "duration" integer(11,0), + "size" integer(11,0), + "description" TEXT(2048,0), + "website" TEXT(384,0), + "url_sub" TEXT(384,0), + "url_video" TEXT(384,0), + "url_video_sd" TEXT(384,0), + "url_video_hd" TEXT(384,0), + CONSTRAINT "FK_FilmShow" FOREIGN KEY ("showid") REFERENCES "show" ("id") ON DELETE CASCADE, + CONSTRAINT "FK_FilmChannel" FOREIGN KEY ("channelid") REFERENCES "channel" ("id") ON DELETE CASCADE +); + +-- ---------------------------- +-- Table structure for show +-- ---------------------------- +DROP TABLE IF EXISTS "show"; +CREATE TABLE "show" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "dtCreated" integer(11,0) NOT NULL DEFAULT 0, + "touched" integer(1,0) NOT NULL DEFAULT 1, + "channelid" INTEGER(11,0) NOT NULL DEFAULT 0, + "show" TEXT(255,0) NOT NULL, + "search" TEXT(255,0) NOT NULL, + CONSTRAINT "FK_ShowChannel" FOREIGN KEY ("channelid") REFERENCES "channel" ("id") ON DELETE CASCADE +); + +-- ---------------------------- +-- Table structure for status +-- ---------------------------- +DROP TABLE IF EXISTS "status"; +CREATE TABLE "status" ( + "modified" integer(11,0), + "status" TEXT(32,0), + "lastupdate" integer(11,0), + "filmupdate" integer(11,0), + "fullupdate" integer(1,0), + "add_chn" integer(11,0), + "add_shw" integer(11,0), + "add_mov" integer(11,0), + "del_chm" integer(11,0), + "del_shw" integer(11,0), + "del_mov" integer(11,0), + "tot_chn" integer(11,0), + "tot_shw" integer(11,0), + "tot_mov" integer(11,0) +); + +-- ---------------------------- +-- Indexes structure for table film +-- ---------------------------- +CREATE INDEX "dupecheck" ON film ("channelid", "showid", "url_video"); +CREATE INDEX "index_1" ON film ("channelid", "title" COLLATE NOCASE); +CREATE INDEX "index_2" ON film ("showid", "title" COLLATE NOCASE); + +-- ---------------------------- +-- Indexes structure for table show +-- ---------------------------- +CREATE INDEX "category" ON show ("category"); +CREATE INDEX "search" ON show ("search"); +CREATE INDEX "combined_1" ON show ("channelid", "search"); +CREATE INDEX "combined_2" ON show ("channelid", "show"); + +PRAGMA foreign_keys = true; + """ ) + self.UpdateStatus( 'IDLE' ) + + def _file_remove( self, name ): + if mvutils.file_exists( name ): + try: + os.remove( name ) + return True + except OSError as err: + self.logger.error( 'Failed to remove {}: error {}', name, err ) + return False + +def UNIX_TIMESTAMP(): + return int( time.time() ) + +class GROUP_CONCAT: + def __init__( self ): + self.value = '' + + def step( self, value ): + if value is not None: + if self.value == '': + self.value = '{0}'.format( value ) + else: + self.value = '{0},{1}'.format( self.value, value ) + + def finalize(self): + return self.value diff --git a/plugin.video.mediathekview/resources/lib/ttml2srt.py b/plugin.video.mediathekview/resources/lib/ttml2srt.py new file mode 100644 index 0000000..7bfdc47 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/ttml2srt.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Laura Klünder +# See https://github.com/codingcatgirl/ttml2srt +# +# MIT License +# +# Copyright (c) 2017 Laura Klünder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import re +import io +from datetime import timedelta +from xml.etree import ElementTree as ET + +def ttml2srt( infile, outfile ): + tree = ET.parse( infile ) + root = tree.getroot() + + # strip namespaces + for elem in root.getiterator(): + elem.tag = elem.tag.split('}', 1)[-1] + elem.attrib = {name.split('}', 1) + [-1]: value for name, value in elem.attrib.items()} + + # get styles + styles = {} + for elem in root.findall('./head/styling/style'): + style = {} + if 'color' in elem.attrib: + color = elem.attrib['color'] + if color not in ('#FFFFFF', '#000000'): + style['color'] = color + if 'fontStyle' in elem.attrib: + fontstyle = elem.attrib['fontStyle'] + if fontstyle in ('italic', ): + style['fontstyle'] = fontstyle + styles[elem.attrib['id']] = style + + body = root.find('./body') + + # parse correct start and end times + def parse_time_expression(expression, default_offset=timedelta(0)): + offset_time = re.match(r'^([0-9]+(\.[0-9]+)?)(h|m|s|ms|f|t)$', expression) + if offset_time: + time_value, fraction, metric = offset_time.groups() + time_value = float(time_value) + if metric == 'h': + return default_offset + timedelta(hours=time_value) + elif metric == 'm': + return default_offset + timedelta(minutes=time_value) + elif metric == 's': + return default_offset + timedelta(seconds=time_value) + elif metric == 'ms': + return default_offset + timedelta(milliseconds=time_value) + elif metric == 'f': + raise NotImplementedError( + 'Parsing time expressions by frame is not supported!') + elif metric == 't': + raise NotImplementedError( + 'Parsing time expressions by ticks is not supported!') + + clock_time = re.match( + r'^([0-9]{2,}):([0-9]{2,}):([0-9]{2,}(\.[0-9]+)?)$', expression) + if clock_time: + hours, minutes, seconds, fraction = clock_time.groups() + return timedelta(hours=int(hours), minutes=int(minutes), seconds=float(seconds)) + + clock_time_frames = re.match( + r'^([0-9]{2,}):([0-9]{2,}):([0-9]{2,}):([0-9]{2,}(\.[0-9]+)?)$', expression) + if clock_time_frames: + raise NotImplementedError( + 'Parsing time expressions by frame is not supported!') + + raise ValueError('unknown time expression: %s' % expression) + + + def parse_times(elem, default_begin=timedelta(0)): + if 'begin' in elem.attrib: + begin = parse_time_expression( + elem.attrib['begin'], default_offset=default_begin) + else: + begin = default_begin + elem.attrib['{abs}begin'] = begin + + end = None + if 'end' in elem.attrib: + end = parse_time_expression( + elem.attrib['end'], default_offset=default_begin) + + dur = None + if 'dur' in elem.attrib: + dur = parse_time_expression(elem.attrib['dur']) + + if dur is not None: + if end is None: + end = begin + dur + else: + end = min(end, begin + dur) + + elem.attrib['{abs}end'] = end + + for child in elem: + parse_times(child, default_begin=begin) + + + parse_times(body) + + timestamps = set() + for elem in body.findall('.//*[@{abs}begin]'): + timestamps.add(elem.attrib['{abs}begin']) + + for elem in body.findall('.//*[@{abs}end]'): + timestamps.add(elem.attrib['{abs}end']) + + timestamps.discard(None) + + # render subtitles on each timestamp + + + def render_subtitles(elem, timestamp, parent_style=None): + + if timestamp < elem.attrib['{abs}begin']: + return '' + if elem.attrib['{abs}end'] is not None and timestamp >= elem.attrib['{abs}end']: + return '' + + result = '' + + style = parent_style.copy() if parent_style is not None else {} + if 'style' in elem.attrib: + style.update(styles[elem.attrib['style']]) + + if 'color' in style: + result += '<font color="%s">' % style['color'] + + if style.get('fontstyle') == 'italic': + result += '<i>' + + if elem.text: + result += elem.text.strip() + if len(elem): + for child in elem: + result += render_subtitles(child, timestamp) + if child.tail: + result += child.tail.strip() + + if 'color' in style: + result += '</font>' + + if style.get('fontstyle') == 'italic': + result += '</i>' + + if elem.tag in ('div', 'p', 'br'): + result += '\n' + + return result + + + rendered = [] + for timestamp in sorted(timestamps): + rendered.append((timestamp, re.sub(r'\n\n\n+', '\n\n', + render_subtitles(body, timestamp)).strip())) + + if not rendered: + exit(0) + + # group timestamps together if nothing changes + rendered_grouped = [] + last_text = None + for timestamp, content in rendered: + if content != last_text: + rendered_grouped.append((timestamp, content)) + last_text = content + + # output srt + rendered_grouped.append((rendered_grouped[-1][0] + timedelta(hours=24), '')) + + + def format_timestamp(timestamp): + return ('%02d:%02d:%02.3f' % (timestamp.total_seconds() // 3600, + timestamp.total_seconds() // 60 % 60, + timestamp.total_seconds() % 60)).replace('.', ',') + + + if isinstance( outfile, str ) or isinstance( outfile, unicode ): + file = io.open( outfile, 'w', encoding='utf-8' ) + else: + file = outfile + + srt_i = 1 + for i, (timestamp, content) in enumerate(rendered_grouped[:-1]): + if content == '': + continue + file.write( bytearray( '%d\n' % srt_i, 'utf-8' ) ) + file.write( bytearray( + format_timestamp( timestamp ) + + ' --> ' + + format_timestamp( rendered_grouped[i + 1][0] ) + + '\n' + ) ) + file.write( bytearray( content + '\n\n', 'utf-8' ) ) + srt_i += 1 + file.close() diff --git a/plugin.video.mediathekview/resources/lib/updater.py b/plugin.video.mediathekview/resources/lib/updater.py new file mode 100644 index 0000000..afdf840 --- /dev/null +++ b/plugin.video.mediathekview/resources/lib/updater.py @@ -0,0 +1,466 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017-2018, Leo Moll + +# -- Imports ------------------------------------------------ +import os, urllib2, subprocess, ijson, datetime, time +import xml.etree.ElementTree as etree + +import resources.lib.mvutils as mvutils + +from operator import itemgetter +#from resources.lib.utils import * +from resources.lib.store import Store +from resources.lib.exceptions import DatabaseCorrupted +from resources.lib.exceptions import DatabaseLost + +# -- Unpacker support --------------------------------------- +upd_can_bz2 = False +upd_can_gz = False + +try: + import bz2 + upd_can_bz2 = True +except ImportError: + pass + +try: + import gzip + upd_can_gz = True +except ImportError: + pass + +# -- Constants ---------------------------------------------- +FILMLISTE_AKT_URL = 'https://res.mediathekview.de/akt.xml' +FILMLISTE_DIF_URL = 'https://res.mediathekview.de/diff.xml' + +# -- Classes ------------------------------------------------ +class MediathekViewUpdater( object ): + def __init__( self, logger, notifier, settings, monitor = None ): + self.logger = logger + self.notifier = notifier + self.settings = settings + self.monitor = monitor + self.db = None + self.use_xz = mvutils.find_xz() is not None + + def Init( self ): + if self.db is not None: + self.Exit() + self.db = Store( self.logger, self.notifier, self.settings ) + self.db.Init() + + def Exit( self ): + if self.db is not None: + self.db.Exit() + del self.db + self.db = None + + def IsEnabled( self ): + return self.settings.updenabled + + def GetCurrentUpdateOperation( self ): + if not self.IsEnabled() or self.db is None: + # update disabled or not possible + self.logger.info( 'update disabled or not possible' ) + return 0 + status = self.db.GetStatus() + tsnow = int( time.time() ) + tsold = status['lastupdate'] + dtnow = datetime.datetime.fromtimestamp( tsnow ).date() + dtold = datetime.datetime.fromtimestamp( tsold ).date() + if status['status'] == 'UNINIT': + # database not initialized + self.logger.debug( 'database not initialized' ) + return 0 + elif status['status'] == "UPDATING" and tsnow - tsold > 10800: + # process was probably killed during update + self.logger.info( 'Stuck update pretending to run since epoch {} reset', tsold ) + self.db.UpdateStatus( 'ABORTED' ) + return 0 + elif status['status'] == "UPDATING": + # already updating + self.logger.debug( 'already updating' ) + return 0 + elif tsnow - tsold < self.settings.updinterval: + # last update less than the configured update interval. do nothing + self.logger.debug( 'last update less than the configured update interval. do nothing' ) + return 0 + elif dtnow != dtold: + # last update was not today. do full update once a day + self.logger.debug( 'last update was not today. do full update once a day' ) + return 1 + elif status['status'] == "ABORTED" and status['fullupdate'] == 1: + # last full update was aborted - full update needed + self.logger.debug( 'last full update was aborted - full update needed' ) + return 1 + else: + # do differential update + self.logger.debug( 'do differential update' ) + return 2 + + def Update( self, full ): + if self.db is None: + return + if self.db.SupportsUpdate(): + if self.GetNewestList( full ): + self.Import( full ) + + def Import( self, full ): + ( _, compfile, destfile, avgrecsize ) = self._get_update_info( full ) + if not mvutils.file_exists( destfile ): + self.logger.error( 'File {} does not exists', destfile ) + return False + # estimate number of records in update file + records = int( mvutils.file_size( destfile ) / avgrecsize ) + if not self.db.ftInit(): + self.logger.warn( 'Failed to initialize update. Maybe a concurrency problem?' ) + return False + try: + self.logger.info( 'Starting import of approx. {} records from {}', records, destfile ) + with open( destfile, 'r' ) as file: + parser = ijson.parse( file ) + flsm = 0 + flts = 0 + ( self.tot_chn, self.tot_shw, self.tot_mov ) = self._update_start( full ) + self.notifier.ShowUpdateProgress() + for prefix, event, value in parser: + if ( prefix, event ) == ( "X", "start_array" ): + self._init_record() + elif ( prefix, event ) == ( "X", "end_array" ): + self._end_record( records ) + if self.count % 100 == 0 and self.monitor.abortRequested(): + # kodi is shutting down. Close all + self._update_end( full, 'ABORTED' ) + self.notifier.CloseUpdateProgress() + return True + elif ( prefix, event ) == ( "X.item", "string" ): + if value is not None: + # self._add_value( value.strip().encode('utf-8') ) + self._add_value( value.strip() ) + else: + self._add_value( "" ) + elif ( prefix, event ) == ( "Filmliste", "start_array" ): + flsm += 1 + elif ( prefix, event ) == ( "Filmliste.item", "string" ): + flsm += 1 + if flsm == 2 and value is not None: + # this is the timestmap of this database update + try: + fldt = datetime.datetime.strptime( value.strip(), "%d.%m.%Y, %H:%M" ) + flts = int( time.mktime( fldt.timetuple() ) ) + self.db.UpdateStatus( filmupdate = flts ) + self.logger.info( 'Filmliste dated {}', value.strip() ) + except TypeError: + # SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507 + # Wonderful. His name is also Leopold + try: + flts = int( time.mktime( time.strptime( value.strip(), "%d.%m.%Y, %H:%M" ) ) ) + self.db.UpdateStatus( filmupdate = flts ) + self.logger.info( 'Filmliste dated {}', value.strip() ) + except Exception as err: + # If the universe hates us... + self.logger.debug( 'Could not determine date "{}" of filmliste: {}', value.strip(), err ) + except ValueError as err: + pass + + self._update_end( full, 'IDLE' ) + self.logger.info( 'Import of {} finished', destfile ) + self.notifier.CloseUpdateProgress() + return True + except KeyboardInterrupt: + self._update_end( full, 'ABORTED' ) + self.logger.info( 'Interrupted by user' ) + self.notifier.CloseUpdateProgress() + return True + except DatabaseCorrupted as err: + self.logger.error( '{}', err ) + self.notifier.CloseUpdateProgress() + except DatabaseLost as err: + self.logger.error( '{}', err ) + self.notifier.CloseUpdateProgress() + except IOError as err: + self.logger.error( 'Error {} wile processing {}', err, destfile ) + self._update_end( full, 'ABORTED' ) + self.notifier.CloseUpdateProgress() + return False + + def GetNewestList( self, full ): + ( url, compfile, destfile, avgrecsize ) = self._get_update_info( full ) + if url is None: + self.logger.error( 'No suitable archive extractor available for this system' ) + self.notifier.ShowMissingExtractorError() + return False + + # get mirrorlist + self.logger.info( 'Opening {}', url ) + try: + data = urllib2.urlopen( url ).read() + except urllib2.URLError as err: + self.logger.error( 'Failure opening {}', url ) + self.notifier.ShowDownloadError( url, err ) + return False + + root = etree.fromstring ( data ) + urls = [] + for server in root.findall( 'Server' ): + try: + URL = server.find( 'URL' ).text + Prio = server.find( 'Prio' ).text + urls.append( ( self._get_update_url( URL ), Prio ) ) + self.logger.info( 'Found mirror {} (Priority {})', URL, Prio ) + except AttributeError: + pass + urls = sorted( urls, key = itemgetter( 1 ) ) + urls = [ url[0] for url in urls ] + result = None + + # cleanup downloads + self.logger.info( 'Cleaning up old downloads...' ) + self._file_remove( compfile ) + self._file_remove( destfile ) + + # download filmliste + self.notifier.ShowDownloadProgress() + lasturl = '' + for url in urls: + try: + lasturl = url + self.logger.info( 'Trying to download {} from {}...', os.path.basename( compfile ), url ) + self.notifier.UpdateDownloadProgress( 0, url ) + result = mvutils.url_retrieve( url, filename = compfile, reporthook = self._reporthook ) + break + except urllib2.URLError as err: + self.logger.error( 'Failure downloading {}', url ) + except Exception as err: + self.logger.error( 'Failure writng {}', url ) + if result is None: + self.logger.info( 'No file downloaded' ) + self.notifier.CloseDownloadProgress() + self.notifier.ShowDownloadError( lasturl, err ) + return False + + # decompress filmliste + if self.use_xz is True: + self.logger.info( 'Trying to decompress xz file...' ) + retval = subprocess.call( [ mvutils.find_xz(), '-d', compfile ] ) + self.logger.info( 'Return {}', retval ) + elif upd_can_bz2 is True: + self.logger.info( 'Trying to decompress bz2 file...' ) + retval = self._decompress_bz2( compfile, destfile ) + self.logger.info( 'Return {}', retval ) + elif upd_can_gz is True: + self.logger.info( 'Trying to decompress gz file...' ) + retval = self._decompress_gz( compfile, destfile ) + self.logger.info( 'Return {}', retval ) + else: + # should nebver reach + pass + + self.notifier.CloseDownloadProgress() + return retval == 0 and mvutils.file_exists( destfile ) + + def _get_update_info( self, full ): + if self.use_xz is True: + ext = 'xz' + elif upd_can_bz2 is True: + ext = 'bz2' + elif upd_can_gz is True: + ext = 'gz' + else: + return ( None, None, None, 0, ) + + if full: + return ( + FILMLISTE_AKT_URL, + os.path.join( self.settings.datapath, 'Filmliste-akt.' + ext ), + os.path.join( self.settings.datapath, 'Filmliste-akt' ), + 600, + ) + else: + return ( + FILMLISTE_DIF_URL, + os.path.join( self.settings.datapath, 'Filmliste-diff.' + ext ), + os.path.join( self.settings.datapath, 'Filmliste-diff' ), + 700, + ) + + def _get_update_url( self, url ): + if self.use_xz is True: + return url + elif upd_can_bz2 is True: + return os.path.splitext( url )[0] + '.bz2' + elif upd_can_gz is True: + return os.path.splitext( url )[0] + '.gz' + else: + # should never happen since it will not be called + return None + + def _file_remove( self, name ): + if mvutils.file_exists( name ): + try: + os.remove( name ) + return True + except OSError as err: + self.logger.error( 'Failed to remove {}: error {}', name, err ) + return False + + def _reporthook( self, blockcount, blocksize, totalsize ): + downloaded = blockcount * blocksize + if totalsize > 0: + percent = int( (downloaded * 100) / totalsize ) + self.notifier.UpdateDownloadProgress( percent ) + self.logger.debug( 'Downloading blockcount={}, blocksize={}, totalsize={}', blockcount, blocksize, totalsize ) + + def _update_start( self, full ): + self.logger.info( 'Initializing update...' ) + self.add_chn = 0 + self.add_shw = 0 + self.add_mov = 0 + self.add_chn = 0 + self.add_shw = 0 + self.add_mov = 0 + self.del_chn = 0 + self.del_shw = 0 + self.del_mov = 0 + self.index = 0 + self.count = 0 + self.film = { + "channel": "", + "show": "", + "title": "", + "aired": "1980-01-01 00:00:00", + "duration": "00:00:00", + "size": 0, + "description": "", + "website": "", + "url_sub": "", + "url_video": "", + "url_video_sd": "", + "url_video_hd": "", + "airedepoch": 0, + "geo": "" + } + return self.db.ftUpdateStart( full ) + + def _update_end( self, full, status ): + self.logger.info( 'Added: channels:%d, shows:%d, movies:%d ...' % ( self.add_chn, self.add_shw, self.add_mov ) ) + ( self.del_chn, self.del_shw, self.del_mov, self.tot_chn, self.tot_shw, self.tot_mov ) = self.db.ftUpdateEnd( full and status == 'IDLE' ) + self.logger.info( 'Deleted: channels:%d, shows:%d, movies:%d' % ( self.del_chn, self.del_shw, self.del_mov ) ) + self.logger.info( 'Total: channels:%d, shows:%d, movies:%d' % ( self.tot_chn, self.tot_shw, self.tot_mov ) ) + self.db.UpdateStatus( + status, + int( time.time() ) if status != 'ABORTED' else None, + None, + 1 if full else 0, + self.add_chn, self.add_shw, self.add_mov, + self.del_chn, self.del_shw, self.del_mov, + self.tot_chn, self.tot_shw, self.tot_mov + ) + + def _init_record( self ): + self.index = 0 + self.film["title"] = "" + self.film["aired"] = "1980-01-01 00:00:00" + self.film["duration"] = "00:00:00" + self.film["size"] = 0 + self.film["description"] = "" + self.film["website"] = "" + self.film["url_sub"] = "" + self.film["url_video"] = "" + self.film["url_video_sd"] = "" + self.film["url_video_hd"] = "" + self.film["airedepoch"] = 0 + self.film["geo"] = "" + + def _end_record( self, records ): + if self.count % 1000 == 0: + percent = int( self.count * 100 / records ) + self.logger.info( 'In progress (%d%%): channels:%d, shows:%d, movies:%d ...' % ( percent, self.add_chn, self.add_shw, self.add_mov ) ) + self.notifier.UpdateUpdateProgress( percent if percent <= 100 else 100, self.count, self.add_chn, self.add_shw, self.add_mov ) + self.db.UpdateStatus( + add_chn = self.add_chn, + add_shw = self.add_shw, + add_mov = self.add_mov, + tot_chn = self.tot_chn + self.add_chn, + tot_shw = self.tot_shw + self.add_shw, + tot_mov = self.tot_mov + self.add_mov + ) + self.count = self.count + 1 + ( _, cnt_chn, cnt_shw, cnt_mov ) = self.db.ftInsertFilm( self.film, True ) + else: + self.count = self.count + 1 + ( _, cnt_chn, cnt_shw, cnt_mov ) = self.db.ftInsertFilm( self.film, False ) + self.add_chn += cnt_chn + self.add_shw += cnt_shw + self.add_mov += cnt_mov + + def _add_value( self, val ): + if self.index == 0: + if val != "": + self.film["channel"] = val + elif self.index == 1: + if val != "": + self.film["show"] = val[:255] + elif self.index == 2: + self.film["title"] = val[:255] + elif self.index == 3: + if len(val) == 10: + self.film["aired"] = val[6:] + '-' + val[3:5] + '-' + val[:2] + elif self.index == 4: + if ( self.film["aired"] != "1980-01-01 00:00:00" ) and ( len(val) == 8 ): + self.film["aired"] = self.film["aired"] + " " + val + elif self.index == 5: + if len(val) == 8: + self.film["duration"] = val + elif self.index == 6: + if val != "": + self.film["size"] = int(val) + elif self.index == 7: + self.film["description"] = val + elif self.index == 8: + self.film["url_video"] = val + elif self.index == 9: + self.film["website"] = val + elif self.index == 10: + self.film["url_sub"] = val + elif self.index == 12: + self.film["url_video_sd"] = self._make_url(val) + elif self.index == 14: + self.film["url_video_hd"] = self._make_url(val) + elif self.index == 16: + if val != "": + self.film["airedepoch"] = int(val) + elif self.index == 18: + self.film["geo"] = val + self.index = self.index + 1 + + def _make_url( self, val ): + x = val.split( '|' ) + if len( x ) == 2: + cnt = int( x[0] ) + return self.film["url_video"][:cnt] + x[1] + else: + return val + + def _decompress_bz2( self, sourcefile, destfile ): + blocksize = 8192 + try: + with open( destfile, 'wb' ) as df, open( sourcefile, 'rb' ) as sf: + decompressor = bz2.BZ2Decompressor() + for data in iter( lambda : sf.read( blocksize ), b'' ): + df.write( decompressor.decompress( data ) ) + except Exception as err: + self.logger.error( 'bz2 decompression failed: {}'.format( err ) ) + return -1 + return 0 + + def _decompress_gz( self, sourcefile, destfile ): + blocksize = 8192 + try: + with open( destfile, 'wb' ) as df, gzip.open( sourcefile ) as sf: + for data in iter( lambda : sf.read( blocksize ), b'' ): + df.write( data ) + except Exception as err: + self.logger.error( 'gz decompression failed: {}'.format( err ) ) + return -1 + return 0 |