CiaranG/mythtv-scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

find_orphans additions

Browse Source 
This adds a non-interactive mode that prints out the list and exists,
triggered by running the script outside of a terminal.

This also adds a new type, 'misplaced', indicating recordings that were
found, but on a backend other than that listed in the backend.  This
is informative only, and does not support any actions being performed on
the recordings in question.
This commit is contained in:
Raymond Wagner 2012-01-04 03:00:09 -05:00
parent 7b021b489a
commit 1d366ee08b
1 changed files with 173 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

301
python/find_orphans.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
from MythTV import MythDB, MythBE, Recorded from MythTV import MythDB, MythBE, MythLog, Recorded as _Recorded
from MythTV.utility import datetime from MythTV.utility import datetime
from socket import timeout from socket import timeout
@ -15,25 +15,38 @@ def human_size(s):
o += 1 o += 1
return str(round(s,1))+('B ','KB','MB','GB')[o] return str(round(s,1))+('B ','KB','MB','GB')[o]
class Singleton(type):
def __call__(self, *args, **kwargs):
if not hasattr(self, '_instance'):
self._instance = super(Singleton, self).__call__(*args, **kwargs)
# print 'call: %s' % type(self)
# if self.__instance is None:
# self.__instance = super(Singleton, self).__call__(*args, **kwargs)
if callable(self._instance):
return self._instance()
return self._instance
class File( str ): class File( str ):
#Utility class to allow deletion and terminal printing of files. #Utility class to allow deletion and terminal printing of files.
def __new__(self, host, group, path, name, size): def __new__(self, host, group, path, name, size, db):
return str.__new__(self, name) return str.__new__(self, name)
def __init__(self, host, group, path, name, size): def __init__(self, host, group, path, name, size, db):
self.host = host self.hosts = [host]
self.group = group self.group = group
self.path = path self.path = path
self.size = int(size) self.size = int(size)
self.db = db
def pprint(self): def pprint(self):
name = '%s: %s' % (self.host, os.path.join(self.path, self)) name = '%s: %s' % (self.hosts[0], os.path.join(self.path, self))
print u' {0:<90}{1:>8}'.format(name, human_size(self.size)) print u' {0:<90}{1:>8}'.format(name, human_size(self.size))
def delete(self): def delete(self):
be = MythBE(self.host, db=DB) be = MythBE(self.hosts[0], db=self.db)
be.deleteFile(self, self.group) be.deleteFile(self, self.group)
def add_host(self, host):
self.hosts.append(host)
class MyRecorded( Recorded ): class Recorded( _Recorded ):
#Utility class to allow deletion and terminal printing of orphaned recording entries. #Utility class to allow deletion and terminal printing of orphaned recording entries.
_table = 'recorded'
def pprint(self): def pprint(self):
name = u'{0.hostname}: {0.title}'.format(self) name = u'{0.hostname}: {0.title}'.format(self)
if self.subtitle: if self.subtitle:
@ -50,108 +63,159 @@ class MyRecorded( Recorded ):
def printrecs(title, recs): def printrecs(title, recs):
# print out all recordings in list, followed by a count # print out all recordings in list, followed by a count
print title if len(recs):
for rec in sorted(recs, key=lambda x: x.title): print title
rec.pprint() for rec in sorted(recs, key=lambda x: x.title):
print u'{0:>88}{1:>12}'.format('Count:',len(recs)) rec.pprint()
print u'{0:>88}{1:>12}'.format('Count:',len(recs))
def printfiles(title, files): def printfiles(title, files):
# print out all files in list, followed by a total size # print out all files in list, followed by a total size
print title if len(files):
for f in sorted(files, key=lambda x: x.path): print title
f.pprint() for f in sorted(files, key=lambda x: x.path):
size = sum([f.size for f in files]) f.pprint()
print u'{0:>88}{1:>12}'.format('Total:',human_size(size)) size = sum([f.size for f in files])
print u'{0:>88}{1:>12}'.format('Total:',human_size(size))
def populate(host=None): class populate( object ):
# scan through all accessible backends to generate a new list of orphaned content __metaclass__ = Singleton
unfiltered = [] def __init__(self, host=None):
kwargs = {'livetv':True} self.db = MythDB()
if host: self.db.searchRecorded.handler = Recorded
# if the host was defined on the command line, check to make sure such a self.be = MythBE(db=self.db)
# host is defined in the database self.log = MythLog(db=self.db)
with DB as c:
c.execute("""SELECT count(1) FROM settings self.set_host(host)
WHERE hostname=%s AND value=%s""", self.load_backends()
(host, 'BackendServerIP')) self.load_storagegroups()
if c.fetchone()[0] == 0:
raise Exception('Invalid hostname specified on command line.') def set_host(self, host):
hosts = [host] self.host = host
kwargs['hostname'] = host if host:
else: # if the host was defined on the command line, check
# else, pull a list of all defined backends from the database # to make sure such host is defined in the database
with DB as c: with self.db as c:
c.execute("""SELECT count(1) FROM settings
WHERE hostname=? AND value=?""",
(host, 'BackendServerIP'))
if c.fetchone()[0] == 0:
raise Exception('Invalid hostname specified for backend.')
def load_backends(self):
with self.db as c:
c.execute("""SELECT hostname FROM settings c.execute("""SELECT hostname FROM settings
WHERE value='BackendServerIP'""") WHERE value='BackendServerIP'""")
hosts = [r[0] for r in c.fetchall()] hosts = [r[0] for r in c.fetchall()]
for host in hosts: self.hosts = []
for sg in DB.getStorageGroup(): for host in hosts:
# skip special storage groups intended for MythVideo # try to access all defined hosts, and
# this list will need to be added to as additional plugins # store the ones currently accessible
# start using their own storage groups
if sg.groupname in ('Videos','Banners','Coverart',\
'Fanart','Screenshots','Trailers'):
continue
try: try:
dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname) MythBE(backend=host)
for f,s in zip(files,sizes): self.hosts.append(host)
newfile = File(host, sg.groupname, sg.dirname, f, s)
# each filename should be unique among all storage directories
# defined on all backends
# store one copy of a file, ignoring where the file actually exists
if newfile not in unfiltered:
unfiltered.append(newfile)
except: except:
pass pass
recs = list(DB.searchRecorded(**kwargs)) def load_storagegroups(self):
self.storagegroups = \
[sg for sg in self.db.getStorageGroup() \
if sg.groupname not in ('Videos','Banners','Coverart',\
'Fanart','Screenshots','Trailers')]
zerorecs = [] def flush(self):
pendrecs = [] self.misplaced = []
orphvids = [] self.zerorecs = []
for rec in list(recs): self.pendrecs = []
# run through list of recordings, matching recording basenames with self.orphrecs = []
# found files, and removing from both lists self.orphvids = []
if rec.basename in unfiltered: self.orphimgs = []
recs.remove(rec) self.dbbackup = []
i = unfiltered.index(rec.basename) self.unfiltered = []
f = unfiltered.pop(i)
if f.size < 1024:
zerorecs.append(rec)
elif rec.doubleorphan:
pendrecs.append(rec)
# remove any file with the same basename, these could be snapshots, failed
# transcode temporary files, or anything else relating to a non-orphaned
# recording
name = rec.basename.rsplit('.',1)[0]
for f in list(unfiltered):
if name in f:
unfiltered.remove(f)
# filter remaining files for those with recording extensions def __call__(self):
for f in list(unfiltered): self.refresh_content()
if not (f.endswith('.mpg') or f.endswith('.nuv')): return self
continue
orphvids.append(f)
unfiltered.remove(f)
# filter remaining files for those with image extensions def refresh_content(self):
orphimgs = [] # scan through all accessible backends to
for f in list(unfiltered): # generate a new listof orphaned content
if not f.endswith('.png'): self.flush()
continue
orphimgs.append(f)
unfiltered.remove(f)
# filter remaining files for those that look like database backups unfiltered = {}
dbbackup = [] for host in self.hosts:
for f in list(unfiltered): for sg in self.storagegroups:
if 'sql' not in f: try:
continue dirs,files,sizes = self.be.getSGList(host, sg.groupname, sg.dirname)
dbbackup.append(f) for f,s in zip(files, sizes):
unfiltered.remove(f) newfile = File(host, sg.groupname, sg.dirname, f, s, self.db)
# each filename should be unique among all storage directories
# defined on all backends, but may exist in the same directory
# on multiple backends if they are shared
if newfile not in unfiltered:
# add a new file to the list
unfiltered[str(newfile)] = newfile
else:
# add a reference to the host on which it was found
unfiltered[str(newfile)].add_host(host)
except:
self.log(MythLog.GENERAL, MythLog.INFO,
'Could not access {0.groupname}@{1}{0.dirname}'.format(sg, host))
return (recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered) for rec in self.db.searchRecorded(livetv=True):
if rec.hostname not in self.hosts:
# recording is on an offline backend, ignore it
name = rec.basename.rsplit('.',1)[0]
for n in unfiltered.keys():
if name in n:
# and anything related to it
del unfiltered[n]
elif rec.basename in unfiltered:
# run through list of recordings, matching basenames
# with found files, and removing file from list
f = unfiltered[rec.basename]
del unfiltered[rec.basename]
if f.size < 1024:
# file is too small to be of any worth
self.zerorecs.append(rec)
elif rec.doubleorphan:
# file is marked for deletion, but has been forgotten by the backend
self.pendrecs.append(rec)
elif rec.hostname not in f.hosts:
# recording is in the database, but not where it should be
self.misplaced.append(rec)
name = rec.basename.rsplit('.',1)[0]
for f in unfiltered.keys():
if name in f:
# file is related to a valid recording, ignore it
del unfiltered[f]
else:
# recording has been orphaned
self.orphrecs.append(rec)
for n,f in unfiltered.iteritems():
if n.endswith('.mpg') or n.endswith('.nuv'):
# filter files with recording extensions
self.orphvids.append(f)
elif n.endswith('.png'):
# filter files with image extensions
self.orphimgs.append(f)
elif 'sql' in n:
# filter for database backups
self.dbbackup.append(f)
else:
self.unfiltered.append(f)
def print_results(self):
printrecs("Recordings found on the wrong host", self.misplaced)
printrecs("Recordings with missing files", self.orphrecs)
printrecs("Zero byte recordings", self.zerorecs)
printrecs("Forgotten pending deletions", self.pendrecs)
printfiles("Orphaned video files", self.orphvids)
printfiles("Orphaned snapshots", self.orphimgs)
printfiles("Database backups", self.dbbackup)
printfiles("Other files", self.unfiltered)
def delete_recs(recs): def delete_recs(recs):
printrecs('The following recordings will be deleted', recs) printrecs('The following recordings will be deleted', recs)
@ -197,38 +261,23 @@ def delete_files(files):
sys.exit(0) sys.exit(0)
def main(host=None): def main(host=None):
while True: if not sys.stdin.isatty():
recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host) populate().print_results()
sys.exit(0)
if len(recs): while True:
printrecs("Recordings with missing files", recs) results = populate(host)
if len(zerorecs): results.print_results()
printrecs("Zero byte recordings", zerorecs)
if len(pendrecs):
printrecs("Forgotten pending deletions", pendrecs)
if len(orphvids):
printfiles("Orphaned video files", orphvids)
if len(orphimgs):
printfiles("Orphaned snapshots", orphimgs)
if len(dbbackup):
printfiles("Database backups", dbbackup)
if len(unfiltered):
printfiles("Other files", unfiltered)
opts = [] opts = [opt for opt in (
if len(recs): ('Delete orphaned recording entries', delete_recs, results.orphrecs),
opts.append(['Delete orphaned recording entries', delete_recs, recs]) ('Delete zero byte recordings', delete_recs, results.zerorecs),
if len(zerorecs): ('Forgotten pending deletion recordings', delete_recs, results.pendrecs),
opts.append(['Delete zero byte recordings', delete_recs, zerorecs]) ('Delete orphaned video files', delete_files, results.orphvids),
if len(pendrecs): ('Delete orphaned snapshots', delete_files, results.orphimgs),
opts.append(['Forgotten pending deletion recordings', delete_recs, pendrecs]) ('Delete other files', delete_files, results.unfiltered),
if len(orphvids): ('Refresh list', None, None))
opts.append(['Delete orphaned video files', delete_files, orphvids]) if (opt[2] is None) or len(opt[2])]
if len(orphimgs):
opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
if len(unfiltered):
opts.append(['Delete other files', delete_files, unfiltered])
opts.append(['Refresh list', None, None])
print 'Please select from the following' print 'Please select from the following'
for i, opt in enumerate(opts): for i, opt in enumerate(opts):
print u' {0}. {1}'.format(i+1, opt[0]) print u' {0}. {1}'.format(i+1, opt[0])
@ -257,10 +306,6 @@ def main(host=None):
except EOFError: except EOFError:
sys.exit(0) sys.exit(0)
DB = MythDB()
BE = MythBE(db=DB)
DB.searchRecorded.handler = MyRecorded
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) == 2: if len(sys.argv) == 2:
main(sys.argv[1]) main(sys.argv[1])