The company i work for has a lot of custom jobtypes that we use with Qube. we tend to run into issues with one of our jobtypes where the proprietary software we use crashes on the host while rendering and the job hangs until it's migrated. i'm fairly new to coding but i decided to write a script that will run either by chron or task scheduler. the plan is to check for running jobs with completed subjobs and compare those times. if the running subjobs time is greater than the average render time by completed subjobs by 1.5 it'll migrate the subjob to another host. if it's 3 times greater than the average render time it'll email me. then it's most likely a proxy.exe error on the host. that's a whole different story. any feedback would be much appreciated! the code:
jobs = qb.jobinfo(status='running', subjobs=True)
now = qb.currenttime()
for job in jobs:
subjob_avg_lst = []
txt = ['Running jobs:']
for subjob in job['subjobs']:
if subjob['status'] == 'running':
running_subjob_dict = {'%s.%s'%(job['id'], subjob['id']):float(now - subjob['timestart']) / 60}
#print running_subjob_dict
if subjob['status'] == 'complete':
#print job['id'],subjob['id']
subjob_avg = float(subjob['timecomplete'] - subjob['timestart']) / 60
#print subjob_avg
#if subjob_avg >= 0.5:
subjob_avg_lst.append(subjob_avg)
if len(subjob_avg_lst) >= 1:
job_avg = sum(subjob_avg_lst) / len(subjob_avg_lst)
print 'COMPLETED SUBJOBS'
#print job['id'], sum(subjob_avg_lst) / len(subjob_avg_lst)
for k, v in running_subjob_dict.items():
if int(job['id']) == int(k.split('.')[-2]):
if job_avg >= 1 and v >= job_avg * 3:
print 'SENDING MAIL'
txt.append('JobID %s has been running for %s on %s. The average completed subjob was %s'%(k, v, subjob['host'], job_avg))
send_mail(send_from, send_to, subject, text,files, server)
if job_avg >= 1 and v >= job_avg * 1.5:
print 'MIGRATING JOB'
print k, v, job_avg
qb.migrate(k)
