#!/usr/bin/python """ The slave daemon which does the actual building """ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Dries Verachtert import getopt, os, sys, Queue, posix, posixpath, string, re, os.path, shutil, urllib sys.path.append(".") sys.path.append("pydar") sys.path.append("/usr/share/pydar2/pydar") from log4py import Logger from config import Config from slavecommand import SlaveCommand import xmlrpclib from SimpleXMLRPCServer import SimpleXMLRPCServer from xmlrpclib import Server import time import smtplib from gzip import GzipFile from svn import fs, repos from email.MIMEText import MIMEText from email.MIMEMultipart import MIMEMultipart from email.MIMEBase import MIMEBase from email import Encoders from optparse import OptionParser class Slave: def __init__(self): self.__cat = Logger().get_instance(self) self.__config = Config.getInstance() self.__cat.debug("initialized") def run(self): parser = OptionParser() parser.add_option("-s", "--slaveconfig", default=None, dest="slaveconfig", help="specify configuration file", metavar="FILE") (options, args) = parser.parse_args() Config.getInstance().specifyGetOptOptions(options) self.__cat.debug("url of master: " + self.__config.getBuildMasterUrlSlave()) self.__cat.debug("local getBuildMachineId(): " + self.__config.getBuildMachineId()) self.__myserverclient = Server(self.__config.getBuildMasterUrlSlave()) # let the master now that we exist self.__notifyMaster() self.__doBuilds() def __notifyMaster(self): # do a call to the master server so he knows # we're ready to build rpms retval = self.__myserverclient.registerSlave(self.__config.getBuildMachineId(), self.__config.getSlavePassword()) self.__cat.debug("retval:" + retval) for m in self.__config.getDistroArchTags(): self.__cat.debug("distoArchTag:" + m) retval = self.__myserverclient.addDistroArch(self.__config.getBuildMachineId(),self.__config.getSlavePassword(), m) self.__cat.debug("retval=" + str(retval)) self.__cat.debug("notify done") def __doBuilds(self): self.__cat.debug("start") while 1: tempCommandObj = self.__myserverclient.getCommand(self.__config.getBuildMachineId(), self.__config.getSlavePassword()) if tempCommandObj == "": self.__cat.debug("going to sleep") time.sleep(self.__config.getSleepTime()) else: self.__cat.debug("got a command") aCommand = SlaveCommand.fromXmlRpcPersistantObject(tempCommandObj) self.__cat.debug("aCommand=" + str(aCommand.toString())) self.__cat.debug("commandid=" + str(aCommand.getId())) if aCommand.getCommandName()[0:5] == "BUILD": self.__doBuildSpec(aCommand) else: self.__cat.error("command is not BUILD or TESTBUILD ?? commandName=" + commandName) def __doBuildSpec(self,command): # we need to do the following: # create a local directory based on the commandId and on our unique slavebuildserverid to store all the files # download the spec file and all the needed files # build it # inform the master about the result self.__doCreateBuildDir(command) self.__downloadFilesFromMaster(command) # choose one implementation brr = self.__doBuildPreparedSpec(command) self.__informMasterAfterRebuild(command,brr) self.__mailOutput(command,brr) def __downloadFilesFromMaster(self,command): self.__cat.debug("start") # get the filelist.txt # get each entry of the filelist filelisturl = self.__config.getMasterCommandsWebSite() + "/" + str(command.getId()) + "/filelist.txt" self.__cat.debug("filelisturl:" + filelisturl) fd = urllib.urlopen(filelisturl) entry = fd.readline() while entry != "": #self.__cat.debug("entry: " + entry) entry = entry[0:len(entry)-1] self.__cat.debug("entry: " + entry) urllib.urlcleanup() remotefileloc = self.__config.getMasterCommandsWebSite() + "/" + str(command.getId()) + "/" + entry self.__cat.debug("remotefileloc:" + remotefileloc) localfileloc = command.getBuildDir() + "/" + entry self.__cat.debug("localfileloc:" + localfileloc) cachefileloc = "/var/lib/pydar2/cachedsources/" + str(command.getSpecRepoId()) + "/" + str(command.getSpecFileId()) + "/" + str(command.getVersion()) + "/" + entry self.__cat.debug("cache file loc:" + cachefileloc) rindex = string.rfind(entry,'.spec') if os.path.isfile(cachefileloc) and not (rindex > 0 and rindex == len(entry) - len('.spec')): self.__cat.debug("copy from local cache") shutil.copy(cachefileloc, localfileloc) else: self.__cat.debug("get from server") urllib.urlretrieve(remotefileloc, localfileloc) entry = fd.readline() fd.close() # sideeffect: also sets this builddir in the command object def __doCreateBuildDir(self,command): slavebuilddir =os.path.join(self.__config.getSlaveDataRoot(), str(self.__config.getBuildMachineId())) self.__cat.debug("slavebuilddir:" + slavebuilddir) #os.mkdir(slavebuilddir) commandbuilddir = os.path.join(slavebuilddir,str(command.getId())) self.__cat.debug("commandbuilddir:" + commandbuilddir) os.makedirs(commandbuilddir) command.setBuildDir(commandbuilddir) def __doBuildPreparedSpec(self,command): self.__cat.debug("start") buildRoot = self.__config.getBuildRootList().getBuildRootByBuildRootTag(command.getDistroArchTag()) brr = buildRoot.cleanRoot() brr.append( buildRoot.prepareRoot() ) brr.append( buildRoot.installRpms(command.getSpecFileBuildReqs()) ) # all the previous brr results might already contain usefull information # todo: check the previous brr info brr.append( buildRoot.buildRpm(command) ) return brr def __informMasterAfterRebuild(self,command,brr): self.__cat.debug("start") # @todo buildResult = 1 for i in brr.getRpms(): self.__myserverclient.sendResultFileName(self.__config.getBuildMachineId(), self.__config.getSlavePassword(), command.getId(),i) if brr.getSrpm() != "": self.__myserverclient.sendResultFileName(self.__config.getBuildMachineId(), self.__config.getSlavePassword(), command.getId(),brr.getSrpm()) self.__myserverclient.sendBuildResult(self.__config.getBuildMachineId(), self.__config.getSlavePassword(), command.getId(), buildResult) def __getSubjectPartFromBuildLog(self, log): # get the last line mo = re.search(re.compile('(.*)$'), log) if mo != None: lastline = mo.group(1) self.__cat.debug("lastline: " + lastline) mo = re.search(re.compile(' *(.*) is needed by .*'),lastline) if mo != None: return "dep " + mo.group(1) + " not found" mo = re.search(re.compile('error: File .var.lib.pydar2.roots.*usr.src.redhat.SOURCES/(.*): No such file or directory'), lastline) if mo != None: return "file " + mo.group(1) + " not found" return "rpm error" else: self.__cat.debug("lastline not found") return "" def __mailOutput(self, command,brr): self.__cat.debug("start") subject = "" output = "" rpmspart = "" # first read the contents of the buildlog in a string # it might be used in the subject and it might be used as a mimepart fp = GzipFile(command.getBuildDir() + "/buildlog.txt.gz", "rb") buildlog = fp.read() fp.close() if len(brr.getRpms()) > 0: subject = "pydar2:: build ok: " + command.getDistroArchTag() + " - " + command.getSpecFileShortFileName() rpmspart = "rpms: \n" for i in brr.getRpms(): rpmspart = rpmspart + " " + i + "\n" rpmspart = rpmspart + "srpm: \n" rpmspart = rpmspart + " " + brr.getSrpm() else: subject = "pydar2:: build failed: " + command.getDistroArchTag() + " - " + command.getSpecFileShortFileName() + ":" + self.__getSubjectPartFromBuildLog(buildlog) output = output + brr.getContainsErrorsText() + "\n" output = output + command.toMultiLineString() output = output + rpmspart + "\n" output = output + "\nLog:\n" output = output + brr.getLog() msg = MIMEMultipart() msg['Subject'] = subject msg['From'] = self.__config.getSlaveFromEmail() msg['To'] = command.getToEmail() me = self.__config.getSlaveFromEmail() you = command.getToEmail() msg.attach(MIMEText(output)) code = self.__config.getSlaveAttachBuildLog() if code == "gzip": msgpart = MIMEBase("application", "x-zip") fp = open(command.getBuildDir() + "/buildlog.txt.gz", "rb") strarr = fp.read() msgpart.set_payload(strarr) fp.close() msgpart.add_header('Content-Disposition', 'attachment', filename="buildlog.txt.gz") Encoders.encode_base64(msgpart) msg.attach(msgpart) if code == "plain": msgpart = MIMEText(buildlog) msg.attach(msgpart) # copypaste from example on www.python.org s = smtplib.SMTP(self.__config.getSmtpServer()) #s.connect() fp = open(command.getBuildDir() + "/mail", "wb") fp.write(msg.as_string()) fp.close() s.sendmail(me, [you], msg.as_string()) s.close() myInstance = Slave() myInstance.run()