CiaranG/mythtv-scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

find_orphans additions

Browse Source 
This adds a non-interactive mode that prints out the list and exists,
triggered by running the script outside of a terminal.

This also adds a new type, 'misplaced', indicating recordings that were
found, but on a backend other than that listed in the backend.  This
is informative only, and does not support any actions being performed on
the recordings in question.
This commit is contained in:
Raymond Wagner 2012-01-04 03:00:09 -05:00
parent 7b021b489a
commit 1d366ee08b
1 changed files with 173 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
python/find_orphans.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
from MythTV import MythDB, MythBE, Recorded from MythTV import MythDB, MythBE, MythLog, Recorded as _Recorded
from MythTV.utility import datetime from MythTV.utility import datetime
from socket import timeout from socket import timeout
@ -15,25 +15,38 @@ def human_size(s):
o += 1 o += 1
return str(round(s,1))+('B ','KB','MB','GB')[o] return str(round(s,1))+('B ','KB','MB','GB')[o]
class Singleton(type):
def __call__(self, *args, **kwargs):
if not hasattr(self, '_instance'):
self._instance = super(Singleton, self).__call__(*args, **kwargs)
# print 'call: %s' % type(self)
# if self.__instance is None:
# self.__instance = super(Singleton, self).__call__(*args, **kwargs)
if callable(self._instance):
return self._instance()
return self._instance
class File( str ): class File( str ):
#Utility class to allow deletion and terminal printing of files. #Utility class to allow deletion and terminal printing of files.
def __new__(self, host, group, path, name, size): def __new__(self, host, group, path, name, size, db):
return str.__new__(self, name) return str.__new__(self, name)
def __init__(self, host, group, path, name, size): def __init__(self, host, group, path, name, size, db):
self.host = host self.hosts = [host]
self.group = group self.group = group
self.path = path self.path = path
self.size = int(size) self.size = int(size)
self.db = db
def pprint(self): def pprint(self):
name = '%s: %s' % (self.host, os.path.join(self.path, self)) name = '%s: %s' % (self.hosts[0], os.path.join(self.path, self))
print u' {0:<90}{1:>8}'.format(name, human_size(self.size)) print u' {0:<90}{1:>8}'.format(name, human_size(self.size))
def delete(self): def delete(self):
be = MythBE(self.host, db=DB) be = MythBE(self.hosts[0], db=self.db)
be.deleteFile(self, self.group) be.deleteFile(self, self.group)
def add_host(self, host):
self.hosts.append(host)
class MyRecorded( Recorded ): class Recorded( _Recorded ):
#Utility class to allow deletion and terminal printing of orphaned recording entries. #Utility class to allow deletion and terminal printing of orphaned recording entries.
_table = 'recorded'
def pprint(self): def pprint(self):
name = u'{0.hostname}: {0.title}'.format(self) name = u'{0.hostname}: {0.title}'.format(self)
if self.subtitle: if self.subtitle:
@ -50,6 +63,7 @@ class MyRecorded( Recorded ):
def printrecs(title, recs): def printrecs(title, recs):
# print out all recordings in list, followed by a count # print out all recordings in list, followed by a count
if len(recs):
print title print title
for rec in sorted(recs, key=lambda x: x.title): for rec in sorted(recs, key=lambda x: x.title):
rec.pprint() rec.pprint()
@ -57,101 +71,151 @@ def printrecs(title, recs):
def printfiles(title, files): def printfiles(title, files):
# print out all files in list, followed by a total size # print out all files in list, followed by a total size
if len(files):
print title print title
for f in sorted(files, key=lambda x: x.path): for f in sorted(files, key=lambda x: x.path):
f.pprint() f.pprint()
size = sum([f.size for f in files]) size = sum([f.size for f in files])
print u'{0:>88}{1:>12}'.format('Total:',human_size(size)) print u'{0:>88}{1:>12}'.format('Total:',human_size(size))
def populate(host=None): class populate( object ):
# scan through all accessible backends to generate a new list of orphaned content __metaclass__ = Singleton
unfiltered = [] def __init__(self, host=None):
kwargs = {'livetv':True} self.db = MythDB()
self.db.searchRecorded.handler = Recorded
self.be = MythBE(db=self.db)
self.log = MythLog(db=self.db)
self.set_host(host)
self.load_backends()
self.load_storagegroups()
def set_host(self, host):
self.host = host
if host: if host:
# if the host was defined on the command line, check to make sure such a # if the host was defined on the command line, check
# host is defined in the database # to make sure such host is defined in the database
with DB as c: with self.db as c:
c.execute("""SELECT count(1) FROM settings c.execute("""SELECT count(1) FROM settings
WHERE hostname=%s AND value=%s""", WHERE hostname=? AND value=?""",
(host, 'BackendServerIP')) (host, 'BackendServerIP'))
if c.fetchone()[0] == 0: if c.fetchone()[0] == 0:
raise Exception('Invalid hostname specified on command line.') raise Exception('Invalid hostname specified for backend.')
hosts = [host]
kwargs['hostname'] = host def load_backends(self):
else: with self.db as c:
# else, pull a list of all defined backends from the database
with DB as c:
c.execute("""SELECT hostname FROM settings c.execute("""SELECT hostname FROM settings
WHERE value='BackendServerIP'""") WHERE value='BackendServerIP'""")
hosts = [r[0] for r in c.fetchall()] hosts = [r[0] for r in c.fetchall()]
self.hosts = []
for host in hosts: for host in hosts:
for sg in DB.getStorageGroup(): # try to access all defined hosts, and
# skip special storage groups intended for MythVideo # store the ones currently accessible
# this list will need to be added to as additional plugins
# start using their own storage groups
if sg.groupname in ('Videos','Banners','Coverart',\
'Fanart','Screenshots','Trailers'):
continue
try: try:
dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname) MythBE(backend=host)
for f,s in zip(files,sizes): self.hosts.append(host)
newfile = File(host, sg.groupname, sg.dirname, f, s)
# each filename should be unique among all storage directories
# defined on all backends
# store one copy of a file, ignoring where the file actually exists
if newfile not in unfiltered:
unfiltered.append(newfile)
except: except:
pass pass
recs = list(DB.searchRecorded(**kwargs)) def load_storagegroups(self):
self.storagegroups = \
[sg for sg in self.db.getStorageGroup() \
if sg.groupname not in ('Videos','Banners','Coverart',\
'Fanart','Screenshots','Trailers')]
zerorecs = [] def flush(self):
pendrecs = [] self.misplaced = []
orphvids = [] self.zerorecs = []
for rec in list(recs): self.pendrecs = []
# run through list of recordings, matching recording basenames with self.orphrecs = []
# found files, and removing from both lists self.orphvids = []
if rec.basename in unfiltered: self.orphimgs = []
recs.remove(rec) self.dbbackup = []
i = unfiltered.index(rec.basename) self.unfiltered = []
f = unfiltered.pop(i)
if f.size < 1024: def __call__(self):
zerorecs.append(rec) self.refresh_content()
elif rec.doubleorphan: return self
pendrecs.append(rec)
# remove any file with the same basename, these could be snapshots, failed def refresh_content(self):
# transcode temporary files, or anything else relating to a non-orphaned # scan through all accessible backends to
# recording # generate a new listof orphaned content
self.flush()
unfiltered = {}
for host in self.hosts:
for sg in self.storagegroups:
try:
dirs,files,sizes = self.be.getSGList(host, sg.groupname, sg.dirname)
for f,s in zip(files, sizes):
newfile = File(host, sg.groupname, sg.dirname, f, s, self.db)
# each filename should be unique among all storage directories
# defined on all backends, but may exist in the same directory
# on multiple backends if they are shared
if newfile not in unfiltered:
# add a new file to the list
unfiltered[str(newfile)] = newfile
else:
# add a reference to the host on which it was found
unfiltered[str(newfile)].add_host(host)
except:
self.log(MythLog.GENERAL, MythLog.INFO,
'Could not access {0.groupname}@{1}{0.dirname}'.format(sg, host))
for rec in self.db.searchRecorded(livetv=True):
if rec.hostname not in self.hosts:
# recording is on an offline backend, ignore it
name = rec.basename.rsplit('.',1)[0] name = rec.basename.rsplit('.',1)[0]
for f in list(unfiltered): for n in unfiltered.keys():
if name in n:
# and anything related to it
del unfiltered[n]
elif rec.basename in unfiltered:
# run through list of recordings, matching basenames
# with found files, and removing file from list
f = unfiltered[rec.basename]
del unfiltered[rec.basename]
if f.size < 1024:
# file is too small to be of any worth
self.zerorecs.append(rec)
elif rec.doubleorphan:
# file is marked for deletion, but has been forgotten by the backend
self.pendrecs.append(rec)
elif rec.hostname not in f.hosts:
# recording is in the database, but not where it should be
self.misplaced.append(rec)
name = rec.basename.rsplit('.',1)[0]
for f in unfiltered.keys():
if name in f: if name in f:
unfiltered.remove(f) # file is related to a valid recording, ignore it
del unfiltered[f]
else:
# recording has been orphaned
self.orphrecs.append(rec)
# filter remaining files for those with recording extensions for n,f in unfiltered.iteritems():
for f in list(unfiltered): if n.endswith('.mpg') or n.endswith('.nuv'):
if not (f.endswith('.mpg') or f.endswith('.nuv')): # filter files with recording extensions
continue self.orphvids.append(f)
orphvids.append(f) elif n.endswith('.png'):
unfiltered.remove(f) # filter files with image extensions
self.orphimgs.append(f)
elif 'sql' in n:
# filter for database backups
self.dbbackup.append(f)
else:
self.unfiltered.append(f)
# filter remaining files for those with image extensions def print_results(self):
orphimgs = [] printrecs("Recordings found on the wrong host", self.misplaced)
for f in list(unfiltered): printrecs("Recordings with missing files", self.orphrecs)
if not f.endswith('.png'): printrecs("Zero byte recordings", self.zerorecs)
continue printrecs("Forgotten pending deletions", self.pendrecs)
orphimgs.append(f) printfiles("Orphaned video files", self.orphvids)
unfiltered.remove(f) printfiles("Orphaned snapshots", self.orphimgs)
printfiles("Database backups", self.dbbackup)
# filter remaining files for those that look like database backups printfiles("Other files", self.unfiltered)
dbbackup = []
for f in list(unfiltered):
if 'sql' not in f:
continue
dbbackup.append(f)
unfiltered.remove(f)
return (recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered)
def delete_recs(recs): def delete_recs(recs):
printrecs('The following recordings will be deleted', recs) printrecs('The following recordings will be deleted', recs)
@ -197,38 +261,23 @@ def delete_files(files):
sys.exit(0) sys.exit(0)
def main(host=None): def main(host=None):
if not sys.stdin.isatty():
populate().print_results()
sys.exit(0)
while True: while True:
recs, zerorecs, pendrecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host) results = populate(host)
results.print_results()
if len(recs): opts = [opt for opt in (
printrecs("Recordings with missing files", recs) ('Delete orphaned recording entries', delete_recs, results.orphrecs),
if len(zerorecs): ('Delete zero byte recordings', delete_recs, results.zerorecs),
printrecs("Zero byte recordings", zerorecs) ('Forgotten pending deletion recordings', delete_recs, results.pendrecs),
if len(pendrecs): ('Delete orphaned video files', delete_files, results.orphvids),
printrecs("Forgotten pending deletions", pendrecs) ('Delete orphaned snapshots', delete_files, results.orphimgs),
if len(orphvids): ('Delete other files', delete_files, results.unfiltered),
printfiles("Orphaned video files", orphvids) ('Refresh list', None, None))
if len(orphimgs): if (opt[2] is None) or len(opt[2])]
printfiles("Orphaned snapshots", orphimgs)
if len(dbbackup):
printfiles("Database backups", dbbackup)
if len(unfiltered):
printfiles("Other files", unfiltered)
opts = []
if len(recs):
opts.append(['Delete orphaned recording entries', delete_recs, recs])
if len(zerorecs):
opts.append(['Delete zero byte recordings', delete_recs, zerorecs])
if len(pendrecs):
opts.append(['Forgotten pending deletion recordings', delete_recs, pendrecs])
if len(orphvids):
opts.append(['Delete orphaned video files', delete_files, orphvids])
if len(orphimgs):
opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
if len(unfiltered):
opts.append(['Delete other files', delete_files, unfiltered])
opts.append(['Refresh list', None, None])
print 'Please select from the following' print 'Please select from the following'
for i, opt in enumerate(opts): for i, opt in enumerate(opts):
print u' {0}. {1}'.format(i+1, opt[0]) print u' {0}. {1}'.format(i+1, opt[0])
@ -257,10 +306,6 @@ def main(host=None):
except EOFError: except EOFError:
sys.exit(0) sys.exit(0)
DB = MythDB()
BE = MythBE(db=DB)
DB.searchRecorded.handler = MyRecorded
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) == 2: if len(sys.argv) == 2:
main(sys.argv[1]) main(sys.argv[1])