#!/usr/bin/python import getopt, os, sys, Queue, posix, posixpath, string, re, os.path, shutil sys.path.append("/usr/share/pydar/pydar") sys.path.append("pydar") from log import Log from config import Config from command import Command import xmlrpclib from SimpleXMLRPCServer import SimpleXMLRPCServer from xmlrpclib import Server import time from svnwrapper import SvnWrapper from stringfile import StringFile import mach import smtplib from svn import fs, repos from email.MIMEText import MIMEText #class SlaveRpc: # def __init__(self,mylog,myconfig): # METHOD_NAME = "SlaveRpc::__init__(..) - " # self.log = mylog # self.config = myconfig # self.log.debug(METHOD_NAME + "initialized") # # def build(self, dirName, specName, distName, archName, buildsystemName): # METHOD_NAME = "SlaveRpc::build(..) - " # self.log.debug(METHOD_NAME + "start") class Slave: def __init__(self, mylog,myconfig): METHOD_NAME = "Slave::__init__(..) - " self.log = mylog self.config = myconfig self.dirMapping = {} self.log.debug(METHOD_NAME + "initialized") def run(self): METHOD_NAME = "Slave::run(self) - " self.log.debug(METHOD_NAME + "start") self.log.debug(METHOD_NAME + "url of master: " + self.config.buildmasterurl) self.log.debug(METHOD_NAME + "local buildmachineid: " + self.config.buildmachineid) self.myserverclient = Server(self.config.buildmasterurl) # let the master now that we exist self.notifyMaster() # wait for commands from the master #self.startListener() self.doBuilds() def notifyMaster(self): # do a call to the master server so he knows # we're ready to build rpms METHOD_NAME = "Slave::notifyMaster(self) - " retval = self.myserverclient.registerSlave(self.config.buildmachineid) self.log.debug(METHOD_NAME + "retval:" + retval) for m in self.config.machroots: self.log.debug(METHOD_NAME + "machroot:" + m) retval = self.myserverclient.addMachRoot(self.config.buildmachineid, m) self.log.debug(METHOD_NAME + "retval=" + retval) self.log.debug(METHOD_NAME + "notify done") def doSvnUpdate(self): METHOD_NAME = "Slave::doSvnUpdate(..) - " self.log.debug(METHOD_NAME + "start") svnwrap = SvnWrapper(self.log,self.config) svnwrap.startTransaction() svnwrap.updateLocalCopy() svnwrap.stopTransaction() self.log.debug(METHOD_NAME + "end") def doCheckAllSpecs(self): METHOD_NAME = "Slave::doCheckAllSpecs(..) - " self.log.debug(METHOD_NAME + "start") self.dirMapping = {} rpmsDirList = posix.listdir(self.config.mastersvnrpmsdir) for rpmDir in rpmsDirList: #self.log.debug(METHOD_NAME + "rpmDir=" + rpmDir) fullRpmDir = posixpath.join(self.config.mastersvnrpmsdir,rpmDir) if posixpath.isdir(fullRpmDir): #self.log.debug(METHOD_NAME + "it's a dir!") fileList = posix.listdir(fullRpmDir) for file in fileList: #self.log.debug(METHOD_NAME + "part1:" + str(string.rfind(file,'.spec'))) #self.log.debug(METHOD_NAME + "part2:" + str(len(file) - len('.spec'))) rindex = string.rfind(file,'.spec') if rindex > 0 and rindex == len(file) - len('.spec'): #self.log.debug(METHOD_NAME + "spec match, file=" + file) self.doCheckSpec(rpmDir, fullRpmDir, file) def doCheckSpec(self,rpmDirName,fullRpmDir,specFileName): METHOD_NAME = "Slave::doCheckSpec(..) - " #self.log.debug(METHOD_NAME + "start, specFileName=" + specFileName) self.dirMapping[specFileName] = fullRpmDir; def getLastChangedSvnRevId(self, dirName, specName): svnwrap = SvnWrapper(self.log,self.config) svnwrap.startTransaction() retval = svnwrap.getLastCommitRev(dirName,dirName + "/" + specName) svnwrap.stopTransaction() return string.rstrip(str(retval)) def doBuildSpec(self,command): specFileName=command.specName lastChangedSvnRevId = self.getLastChangedSvnRevId(self.dirMapping[specFileName], command.specName) finalOutputDir = "" METHOD_NAME = "Slave::doBuildSpec(..) - " self.log.debug(METHOD_NAME + "start, specFileName=" + specFileName) buffer = StringFile() buffer2 = StringFile() buildResult = 0 srpm = "" rpms = () try: self.log.debug(METHOD_NAME + "dir will be: " + self.dirMapping[specFileName]) mach.config['quiet'] = 1 mach.config['release'] = command.releaseTag mach.config['lastchangedsvnrevid'] = lastChangedSvnRevId mach.config['defaultroot'] = command.machRoot self.log.debug(METHOD_NAME + "mach config set to quiet") os.chdir(self.dirMapping[specFileName]) shutil.copyfile(specFileName,'rpmforge-' + specFileName) commandLine = 'perl -i -npe ' + "'" + 'if ($ch==1) {chomp; $_ = $_ . " #' + lastChangedSvnRevId + '\n"; $ch=0;} if (/\%changelog/) {$ch=1;}' + "' rpmforge-" + specFileName; self.log.debug(METHOD_NAME + "command: " + commandLine) os.system(commandLine) sys.stdout = buffer2 sys.stderr = buffer2 mach.config['collect'] = 1 mach.DEBUG = 1 mach.main(mach.config,["unlock"]) mach.main(mach.config,["clean"]) mach.main(mach.config,["setup", "build"]) mach.main(mach.config,["chroot","/sbin/ldconfig"]) sys.stdout = buffer sys.stderr = buffer myparams = () if command.machRoot == "fedora-1-i386-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"fc1', '1" ', ' --define', '"dist', 'fc1"'] else: if command.machRoot == "fedora-2-i386-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"fc2', '1"', ' --define', '"dist', 'fc2"'] else: if command.machRoot == "fedora-3-i386-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"fc3', '1"', ' --define', '"dist', 'fc3"'] else: if command.machRoot == "aurora-1.92-sparc-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"fc2', '1"', ' --define', '"dist', 'fc2"'] else: if command.machRoot == "redhat-el3-i386-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"el3', '1"', ' --define', '"dist', 'el3"'] else: if command.machRoot == "redhat-el4-i386-core": myparams = ["build", 'rpmforge-' + specFileName, '--define', '"el4', '1"', ' --define', '"dist', 'el4"'] else: myparams = ["build", 'rpmforge-' + specFileName] print "myparams=", myparams (srpm, rpms) = mach.main(mach.config,myparams) self.log.debug(METHOD_NAME + "srpm:" + srpm) self.log.debug(METHOD_NAME + "rpms:" + str(rpms)) buildResult = 1 os.mkdir(str(command.commandId)) myFileName = self.getRpmFileName(srpm) # os.rename(myFileName , str(command.commandId) + "/" + myFileName) for rpmname in rpms: myFileName = self.getRpmFileName(rpmname) os.rename(myFileName, str(command.commandId) + "/" + myFileName) os.system("mv *.log *.md5sum *.*.spec " + str(command.commandId)) except: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ self.log.debug(METHOD_NAME + "in except part") self.log.print_exc() buildResult = 0 sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ self.log.debug(METHOD_NAME + "buffer contains: ") tempbuf = "" for line in buffer.readlines(): #self.log.debug(METHOD_NAME + "line: " + line) tempbuf = tempbuf + line # blabla to server self.informMasterAfterRebuild(buildResult, tempbuf, srpm, rpms, command, self.dirMapping[specFileName]) self.mailOutput(tempbuf,command.toEmail, specFileName, "build: " + str(buildResult), command) def doTestBuildSpec(self,command): METHOD_NAME = "Slave::doTestBuildSpec(..) - " specContents=command.specName specFileName=str(command.commandId) + ".spec" finalOutputDir = "" self.log.debug(METHOD_NAME + "start, specFileName=" + specFileName) buffer = StringFile() buildResult = 0 srpm = "" rpms = () try: specDir = self.config.dirtestbuilds + "/" + str(command.commandId) self.log.debug(METHOD_NAME + "dir will be: " + specDir) mach.config['quiet'] = 1 mach.config['release'] = command.releaseTag mach.config['defaultroot'] = command.machRoot self.log.debug(METHOD_NAME + "mach config set to quiet") os.mkdir(specDir) os.chdir(specDir) f = open(specFileName, 'w') f.write(specContents) f.close() sys.stdout = buffer sys.stderr = buffer mach.config['collect'] = 1 mach.DEBUG = 1 mach.main(mach.config,["unlock"]) (srpm, rpms) = mach.main(mach.config,["build",specFileName]) self.log.debug(METHOD_NAME + "srpm:" + srpm) self.log.debug(METHOD_NAME + "rpms:" + str(rpms)) buildResult = 1 os.mkdir(str(command.commandId)) myFileName = self.getRpmFileName(srpm) os.rename(myFileName , str(command.commandId) + "/" + myFileName) for rpmname in rpms: myFileName = self.getRpmFileName(rpmname) os.rename(myFileName, str(command.commandId) + "/" + myFileName) #os.system("mv *.log *.md5sum *.*.spec " + str(command.commandId)) except: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ self.log.debug(METHOD_NAME + "in except part") self.log.print_exc() buildResult = 0 sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ self.log.debug(METHOD_NAME + "buffer contains: ") tempbuf = "" for line in buffer.readlines(): self.log.debug(METHOD_NAME + "line: " + line) tempbuf = tempbuf + line # blabla to server self.informMasterAfterRebuild(buildResult, tempbuf, srpm, rpms, command, specDir) self.mailOutput(tempbuf,command.toEmail, specFileName, "build: " + str(buildResult), command) def informMasterAfterRebuild(self,buildResult, logbuffer, srpmname, rpmsnames, command, dirName): METHOD_NAME="informMasterAfterRebuild(..) - " self.log.debug(METHOD_NAME + "start") if buildResult: fileName = self.getRpmFileName(srpmname) self.myserverclient.sendResultFileName(self.config.buildmachineid, command.commandId,fileName) for rpm in rpmsnames: fileName = self.getRpmFileName(rpm) self.myserverclient.sendResultFileName(self.config.buildmachineid, command.commandId, fileName) self.myserverclient.sendBuildResult(self.config.buildmachineid, command.commandId, buildResult) def getRpmFileName(self, rpmname): METHOD_NAME = "getRpmFile(..) - " fileName = rpmname[string.rfind(rpmname, '/') + 1:] self.log.debug(METHOD_NAME + "fileName=" + fileName) return fileName def mailOutput(self,output, victim, specName, subjectPart, command): METHOD_NAME = "Slave::mailOutput(..) - " self.log.debug(METHOD_NAME + "start, victim=" + victim + ", output=" + output) output = "Results can be found at:\nhttp://buildmachine1.edesign.be/" + string.replace(specName,".spec","") + "/" + str(command.commandId) + "\n\n" + output msg = MIMEText(output) msg['Subject'] = "pydar: " + command.machRoot + " " + str(command.commandId) + " " + specName + ": " + subjectPart msg['From'] = self.config.mailFrom msg['To'] = victim me = self.config.mailFrom you = victim # copypaste from example on www.python.org s = smtplib.SMTP(self.config.smtpserver) #s.connect() s.sendmail(me, [you], msg.as_string()) s.close() def doBuilds(self): METHOD_NAME = "Slave::doBuilds(self) - " self.log.debug(METHOD_NAME + "start") self.doCheckAllSpecs() while 1: retval = self.myserverclient.getCommand(self.config.buildmachineid) if retval == "": time.sleep(self.config.sleeptime) else: commandName = retval['commandName'] userId = retval['userId'] specName = retval['specName'] toEmail = retval['toEmail'] releaseTag = retval['releaseTag'] machRoot = retval['machRoot'] commandId = retval['commandId'] aCommand = Command(commandName,userId,specName,toEmail,releaseTag,machRoot) aCommand.setCommandId(commandId) self.log.debug(METHOD_NAME + "aCommand=" + str(aCommand)) self.log.debug(METHOD_NAME + "commandid=" + str(aCommand.commandId)) if commandName == "BUILD": #self.doSvnUpdate() self.doCheckAllSpecs() self.doBuildSpec(aCommand) else: if commandName == "TESTBUILD": self.doTestBuildSpec(aCommand) else: self.log.debug(METHOD_NAME + "command is not BUILD or TESTBUILD ?? commandName=" + commandName) #else: # # do the build # # notify master of end of build # def startListener(self): # # open a listen socket and wait for commands from the master # METHOD_NAME = "Slave::startListener(self) - " # self.log.debug(METHOD_NAME + "start") # myslaveserver = SimpleXMLRPCServer((self.config.buildslavehostname,self.config.buildslaveport)) # myslaveserver.register_instance(SlaveRpc(self.log,self.config)) # myslaveserver.serve_forever() myLog = Log(None) myConfig = Config() myInstance = Slave(myLog,myConfig) myInstance.run()