From 1d366ee08b39e3fe8253e3280c2ed35345bc0d85 Mon Sep 17 00:00:00 2001 From: Raymond Wagner Date: Wed, 4 Jan 2012 03:00:09 -0500 Subject: [PATCH] find_orphans additions This adds a non-interactive mode that prints out the list and exists, triggered by running the script outside of a terminal. This also adds a new type, 'misplaced', indicating recordings that were found, but on a backend other than that listed in the backend. This is informative only, and does not support any actions being performed on the recordings in question. --- python/find_orphans.py | 301 +++++++++++++++++++++++------------------ 1 file changed, 173 insertions(+), 128 deletions(-) diff --git a/python/find_orphans.py b/python/find_orphans.py index 76afe70..64d22bd 100755 --- a/python/find_orphans.py +++ b/python/find_orphans.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from MythTV import MythDB, MythBE, Recorded +from MythTV import MythDB, MythBE, MythLog, Recorded as _Recorded from MythTV.utility import datetime from socket import timeout @@ -15,25 +15,38 @@ def human_size(s): o += 1 return str(round(s,1))+('B ','KB','MB','GB')[o] +class Singleton(type): + def __call__(self, *args, **kwargs): + if not hasattr(self, '_instance'): + self._instance = super(Singleton, self).__call__(*args, **kwargs) +# print 'call: %s' % type(self) +# if self.__instance is None: +# self.__instance = super(Singleton, self).__call__(*args, **kwargs) + if callable(self._instance): + return self._instance() + return self._instance + class File( str ): #Utility class to allow deletion and terminal printing of files. - def __new__(self, host, group, path, name, size): + def __new__(self, host, group, path, name, size, db): return str.__new__(self, name) - def __init__(self, host, group, path, name, size): - self.host = host + def __init__(self, host, group, path, name, size, db): + self.hosts = [host] self.group = group self.path = path self.size = int(size) + self.db = db def pprint(self): - name = '%s: %s' % (self.host, os.path.join(self.path, self)) + name = '%s: %s' % (self.hosts[0], os.path.join(self.path, self)) print u' {0:<90}{1:>8}'.format(name, human_size(self.size)) def delete(self): - be = MythBE(self.host, db=DB) + be = MythBE(self.hosts[0], db=self.db) be.deleteFile(self, self.group) + def add_host(self, host): + self.hosts.append(host) -class MyRecorded( Recorded ): +class Recorded( _Recorded ): #Utility class to allow deletion and terminal printing of orphaned recording entries. - _table = 'recorded' def pprint(self): name = u'{0.hostname}: {0.title}'.format(self) if self.subtitle: @@ -50,108 +63,159 @@ class MyRecorded( Recorded ): def printrecs(title, recs): # print out all recordings in list, followed by a count - print title - for rec in sorted(recs, key=lambda x: x.title): - rec.pprint() - print u'{0:>88}{1:>12}'.format('Count:',len(recs)) + if len(recs): + print title + for rec in sorted(recs, key=lambda x: x.title): + rec.pprint() + print u'{0:>88}{1:>12}'.format('Count:',len(recs)) def printfiles(title, files): # print out all files in list, followed by a total size - print title - for f in sorted(files, key=lambda x: x.path): - f.pprint() - size = sum([f.size for f in files]) - print u'{0:>88}{1:>12}'.format('Total:',human_size(size)) + if len(files): + print title + for f in sorted(files, key=lambda x: x.path): + f.pprint() + size = sum([f.size for f in files]) + print u'{0:>88}{1:>12}'.format('Total:',human_size(size)) -def populate(host=None): - # scan through all accessible backends to generate a new list of orphaned content - unfiltered = [] - kwargs = {'livetv':True} - if host: - # if the host was defined on the command line, check to make sure such a - # host is defined in the database - with DB as c: - c.execute("""SELECT count(1) FROM settings - WHERE hostname=%s AND value=%s""", - (host, 'BackendServerIP')) - if c.fetchone()[0] == 0: - raise Exception('Invalid hostname specified on command line.') - hosts = [host] - kwargs['hostname'] = host - else: - # else, pull a list of all defined backends from the database - with DB as c: +class populate( object ): + __metaclass__ = Singleton + def __init__(self, host=None): + self.db = MythDB() + self.db.searchRecorded.handler = Recorded + self.be = MythBE(db=self.db) + self.log = MythLog(db=self.db) + + self.set_host(host) + self.load_backends() + self.load_storagegroups() + + def set_host(self, host): + self.host = host + if host: + # if the host was defined on the command line, check + # to make sure such host is defined in the database + with self.db as c: + c.execute("""SELECT count(1) FROM settings + WHERE hostname=? AND value=?""", + (host, 'BackendServerIP')) + if c.fetchone()[0] == 0: + raise Exception('Invalid hostname specified for backend.') + + def load_backends(self): + with self.db as c: c.execute("""SELECT hostname FROM settings WHERE value='BackendServerIP'""") hosts = [r[0] for r in c.fetchall()] - for host in hosts: - for sg in DB.getStorageGroup(): - # skip special storage groups intended for MythVideo - # this list will need to be added to as additional plugins - # start using their own storage groups - if sg.groupname in ('Videos','Banners','Coverart',\ - 'Fanart','Screenshots','Trailers'): - continue + self.hosts = [] + for host in hosts: + # try to access all defined hosts, and + # store the ones currently accessible try: - dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname) - for f,s in zip(files,sizes): - newfile = File(host, sg.groupname, sg.dirname, f, s) - # each filename should be unique among all storage directories - # defined on all backends - # store one copy of a file, ignoring where the file actually exists - if newfile not in unfiltered: - unfiltered.append(newfile) + MythBE(backend=host) + self.hosts.append(host) except: pass - recs = list(DB.searchRecorded(**kwargs)) + def load_storagegroups(self): + self.storagegroups = \ + [sg for sg in self.db.getStorageGroup() \ + if sg.groupname not in ('Videos','Banners','Coverart',\ + 'Fanart','Screenshots','Trailers')] - zerorecs = [] - pendrecs = [] - orphvids = [] - for rec in list(recs): - # run through list of recordings, matching recording basenames with - # found files, and removing from both lists - if rec.basename in unfiltered: - recs.remove(rec) - i = unfiltered.index(rec.basename) - f = unfiltered.pop(i) - if f.size < 1024: - zerorecs.append(rec) - elif rec.doubleorphan: - pendrecs.append(rec) - # remove any file with the same basename, these could be snapshots, failed - # transcode temporary files, or anything else relating to a non-orphaned - # recording - name = rec.basename.rsplit('.',1)[0] - for f in list(unfiltered): - if name in f: - unfiltered.remove(f) + def flush(self): + self.misplaced = [] + self.zerorecs = [] + self.pendrecs = [] + self.orphrecs = [] + self.orphvids = [] + self.orphimgs = [] + self.dbbackup = [] + self.unfiltered = [] - # filter remaining files for those with recording extensions - for f in list(unfiltered): - if not (f.endswith('.mpg') or f.endswith('.nuv')): - continue - orphvids.append(f) - unfiltered.remove(f) + def __call__(self): + self.refresh_content() + return self - # filter remaining files for those with image extensions - orphimgs = [] - for f in list(unfiltered): - if not f.endswith('.png'): - continue - orphimgs.append(f) - unfiltered.remove(f) + def refresh_content(self): + # scan through all accessible backends to + # generate a new listof orphaned content + self.flush() - # filter remaining files for those that look like database backups - dbbackup = [] - for f in list(unfiltered): - if 'sql' not in f: - continue - dbbackup.append(f) - unfiltered.remove(f) + unfiltered = {} + for host in self.hosts: + for sg in self.storagegroups: + try: + dirs,files,sizes = self.be.getSGList(host, sg.groupname, sg.dirname) + for f,s in zip(files, sizes): + newfile = File(host, sg.groupname, sg.dirname, f, s, self.db) + # each filename should be unique among all storage directories + # defined on all backends, but may exist in the same directory + # on multiple backends if they are shared + if newfile not in unfiltered: + # add a new file to the list + unfiltered[str(newfile)] = newfile + else: + # add a reference to the host on which it was found + unfiltered[str(newfile)].add_host(host) + except: + self.log(MythLog.GENERAL, MythLog.INFO, + 'Could not access {0.groupname}@{1}{0.dirname}'.format(sg, host)) - return (recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered) + for rec in self.db.searchRecorded(livetv=True): + if rec.hostname not in self.hosts: + # recording is on an offline backend, ignore it + name = rec.basename.rsplit('.',1)[0] + for n in unfiltered.keys(): + if name in n: + # and anything related to it + del unfiltered[n] + elif rec.basename in unfiltered: + # run through list of recordings, matching basenames + # with found files, and removing file from list + f = unfiltered[rec.basename] + del unfiltered[rec.basename] + if f.size < 1024: + # file is too small to be of any worth + self.zerorecs.append(rec) + elif rec.doubleorphan: + # file is marked for deletion, but has been forgotten by the backend + self.pendrecs.append(rec) + elif rec.hostname not in f.hosts: + # recording is in the database, but not where it should be + self.misplaced.append(rec) + + name = rec.basename.rsplit('.',1)[0] + for f in unfiltered.keys(): + if name in f: + # file is related to a valid recording, ignore it + del unfiltered[f] + else: + # recording has been orphaned + self.orphrecs.append(rec) + + for n,f in unfiltered.iteritems(): + if n.endswith('.mpg') or n.endswith('.nuv'): + # filter files with recording extensions + self.orphvids.append(f) + elif n.endswith('.png'): + # filter files with image extensions + self.orphimgs.append(f) + elif 'sql' in n: + # filter for database backups + self.dbbackup.append(f) + else: + self.unfiltered.append(f) + + def print_results(self): + printrecs("Recordings found on the wrong host", self.misplaced) + printrecs("Recordings with missing files", self.orphrecs) + printrecs("Zero byte recordings", self.zerorecs) + printrecs("Forgotten pending deletions", self.pendrecs) + printfiles("Orphaned video files", self.orphvids) + printfiles("Orphaned snapshots", self.orphimgs) + printfiles("Database backups", self.dbbackup) + printfiles("Other files", self.unfiltered) def delete_recs(recs): printrecs('The following recordings will be deleted', recs) @@ -197,38 +261,23 @@ def delete_files(files): sys.exit(0) def main(host=None): - while True: - recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host) + if not sys.stdin.isatty(): + populate().print_results() + sys.exit(0) - if len(recs): - printrecs("Recordings with missing files", recs) - if len(zerorecs): - printrecs("Zero byte recordings", zerorecs) - if len(pendrecs): - printrecs("Forgotten pending deletions", pendrecs) - if len(orphvids): - printfiles("Orphaned video files", orphvids) - if len(orphimgs): - printfiles("Orphaned snapshots", orphimgs) - if len(dbbackup): - printfiles("Database backups", dbbackup) - if len(unfiltered): - printfiles("Other files", unfiltered) + while True: + results = populate(host) + results.print_results() - opts = [] - if len(recs): - opts.append(['Delete orphaned recording entries', delete_recs, recs]) - if len(zerorecs): - opts.append(['Delete zero byte recordings', delete_recs, zerorecs]) - if len(pendrecs): - opts.append(['Forgotten pending deletion recordings', delete_recs, pendrecs]) - if len(orphvids): - opts.append(['Delete orphaned video files', delete_files, orphvids]) - if len(orphimgs): - opts.append(['Delete orphaned snapshots', delete_files, orphimgs]) - if len(unfiltered): - opts.append(['Delete other files', delete_files, unfiltered]) - opts.append(['Refresh list', None, None]) + opts = [opt for opt in ( + ('Delete orphaned recording entries', delete_recs, results.orphrecs), + ('Delete zero byte recordings', delete_recs, results.zerorecs), + ('Forgotten pending deletion recordings', delete_recs, results.pendrecs), + ('Delete orphaned video files', delete_files, results.orphvids), + ('Delete orphaned snapshots', delete_files, results.orphimgs), + ('Delete other files', delete_files, results.unfiltered), + ('Refresh list', None, None)) + if (opt[2] is None) or len(opt[2])] print 'Please select from the following' for i, opt in enumerate(opts): print u' {0}. {1}'.format(i+1, opt[0]) @@ -257,10 +306,6 @@ def main(host=None): except EOFError: sys.exit(0) -DB = MythDB() -BE = MythBE(db=DB) -DB.searchRecorded.handler = MyRecorded - if __name__ == '__main__': if len(sys.argv) == 2: main(sys.argv[1])