A few things. On 9 May 2016 at 12:06, Joshua Lock <joshua.g.l...@intel.com> wrote: > buildlogger will be started with the autobuilder and, when correctly > configured, monitor the AB's JSON API for newly started builds. When one is > detected information about the build will be posted to the wiki. > > Requires a ConfigParser (ini) style configuration file at > AB_BASE/etc/buildlogger.conf formatted as follows:
Can we get a buildlogger.conf.example in AB_BASE/etc? > > [wikiuser] > username = botuser > password = botuserpassword > > [wiki] > pagetitle = BuildLog > > Signed-off-by: Joshua Lock <joshua.g.l...@intel.com> > --- > .gitignore | 2 + > bin/buildlogger | 273 > ++++++++++++++++++++++++++++++++++++++++++++++++ > yocto-start-autobuilder | 8 ++ > yocto-stop-autobuilder | 45 ++++---- > 4 files changed, 309 insertions(+), 19 deletions(-) > create mode 100755 bin/buildlogger > > diff --git a/.gitignore b/.gitignore > index 3f9505b..48c8a85 100644 > --- a/.gitignore > +++ b/.gitignore > @@ -8,6 +8,7 @@ > ################################################### > buildset-config > config/autobuilder.conf > +etc/buildlogger.conf > > # Everything else # > ################### > @@ -25,6 +26,7 @@ yocto-controller/controller.cfg > yocto-controller/state.sqlite > yocto-controller/twistd.log* > yocto-controller/buildbot.tac > +yocto-controller/logger.log > yocto-worker/build-appliance/build(newcommits) > yocto-worker/buildbot.tac > yocto-worker/janitor.log > diff --git a/bin/buildlogger b/bin/buildlogger > new file mode 100755 > index 0000000..7b39f92 > --- /dev/null > +++ b/bin/buildlogger > @@ -0,0 +1,273 @@ > +#!/usr/bin/env python3 > +''' > +Created on May 5, 2016 > + > +__author__ = "Joshua Lock" > +__copyright__ = "Copyright 2016, Intel Corporation" > +__credits__ = ["Joshua Lock"] > +__license__ = "GPL" > +__version__ = "2.0" > +__maintainer__ = "Joshua Lock" > +__email__ = "joshua.g.l...@intel.com" > +''' > + > +# We'd probably benefit from using some caching, but first we'd need the AB > API > +# to include > +# > +# We can set repo url, branch & commit for a bunch of repositorys. > +# Do they all get built for nightly? > + > +try: > + import configparser > +except ImportError: > + import ConfigParser as configparser > +import json > +import os > +import requests > +import signal > +import sys > +import time > + > +abapi = > "https://autobuilder.yoctoproject.org/main/json/builders/nightly/builds/_all" > +# Wiki editing params > +un = '' > +pw = '' > +wikiapi = "https://wiki.yoctoproject.org/wiki/api.php" > +title = '' > + > +last_logged = '' > +# TODO: probably shouldn't write files in the same location as the script? > +cachefile = 'buildlogger.lastbuild' > +tmpfile = '/tmp/.buildlogger.pid' > + > + > +# Load configuration information from an ini > +def load_config(configfile): > + global un > + global pw > + global title > + success = False > + > + if os.path.exists(configfile): > + try: > + config = configparser.ConfigParser() > + config.read(configfile) > + un = config.get('wikiuser', 'username') > + pw = config.get('wikiuser', 'password') > + title = config.get('wiki', 'pagetitle') > + success = True > + except configparser.Error as ex: > + print('Failed to load buildlogger configuration with error: %s' > % str(ex)) > + else: > + print('Config file %s does not exist, please create and populate > it.' % configfile) > + > + return success > + > +# we can't rely on the built in JSON parser in the requests module because > +# the JSON we get from the wiki begins with a UTF-8 BOM which chokes > +# json.loads(). > +# Thus we decode the raw resonse content into a string and load that into a > +# JSON object ourselves. > +# > +# http://en.wikipedia.org/wiki/Byte_Order_Mark > +# http://bugs.python.org/issue18958 > +def parse_json(response): > + text = response.content.decode('utf-8-sig') > + > + return json.loads(text) > + > + > +# Get the current content of the BuildLog page -- to make the wiki page as > +# useful as possible the most recent log entry should be at the top, to > +# that end we need to edit the whole page so that we can insert the new entry > +# after the log but before the other entries. > +# This method fetches the current page content, splits out the blurb and > +# returns a pair: > +# 1) the blurb > +# 2) the current entries > +def wiki_get_content(): > + params = > '?format=json&action=query&prop=revisions&rvprop=content&titles=' > + req = requests.get(wikiapi+params+title) > + parsed = parse_json(req) > + pageid = sorted(parsed['query']['pages'].keys())[-1] > + content = parsed['query']['pages'][pageid]['revisions'][0]['*'] > + blurb, entries = content.split('==', 1) > + # ensure we keep only a single newline after the blurb > + blurb = blurb.strip() + "\n" > + entries = '=='+entries > + > + return blurb, entries > + > + > +# Login to the wiki and return cookies for the logged in session > +def wiki_login(): > + payload = { > + 'action': 'login', > + 'lgname': un, > + 'lgpassword': pw, > + 'utf8': '', > + 'format': 'json' > + } > + req1 = requests.post(wikiapi, data=payload) > + parsed = parse_json(req1) > + login_token = parsed['login']['token'] > + > + payload['lgtoken'] = login_token > + req2 = requests.post(wikiapi, data=payload, cookies=req1.cookies) > + > + return req2.cookies.copy() > + > + > +# Post the new page contents *content* with a summary of the action *summary* > +def wiki_post_page(content, summary, cookies): > + params = > '?format=json&action=query&prop=info|revisions&intoken=edit&rvprop=timestamp&titles=' > + req = requests.get(wikiapi+params+title, cookies=cookies) > + > + parsed = parse_json(req) > + pageid = sorted(parsed['query']['pages'].keys())[-1] > + edit_token = parsed['query']['pages'][pageid]['edittoken'] > + > + edit_cookie = cookies.copy() > + edit_cookie.update(req.cookies) > + > + payload = { > + 'action': 'edit', > + 'assert': 'user', > + 'title': title, > + 'summary': summary, > + 'text': content, > + 'token': edit_token, > + 'utf8': '', > + 'format': 'json' > + } > + > + req = requests.post(wikiapi, data=payload, cookies=edit_cookie) > + if not req.status_code == requests.codes.ok: > + print("Unexpected status code %s received when trying to post entry > to" > + "the wiki." % req.status_code) > + return False > + else: > + return True > + > + > +# Extract required info about the last build from the Autobuilder's JSON API > +# and format it for entry into the BuildLog, along with a summary of the edit > +def ab_last_build_to_entry(build_json, build_id): > + build_info = build_json[build_id] > + builder = build_info.get('builderName', 'Unknown builder') > + reason = build_info.get('reason', 'No reason given') > + buildid = build_info.get('number', '') > + buildbranch = '' > + chash = '' > + for prop in build_info.get('properties'): > + if prop[0] == 'branch': > + buildbranch = prop[1] > + # TODO: is it safe to assume we're building from the poky repo? Or at > + # least only to log the poky commit hash. > + if prop[0] == 'commit_poky': > + chash = prop[1] > + > + urlfmt = > 'https://autobuilder.yoctoproject.org/main/builders/%s/builds/%s/' > + url = urlfmt % (builder, buildid) > + sectionfmt = '==[%s %s %s - %s %s]==' > + section_title = sectionfmt % (url, builder, buildid, buildbranch, chash) > + summaryfmt = 'Adding new BuildLog entry for build %s (%s)' > + summary = summaryfmt % (buildid, chash) > + content = "* '''Build ID''' - %s\n" % chash > + content = content + '* ' + reason + '\n' > + new_entry = '%s\n%s\n' % (section_title, content) > + > + return new_entry, summary > + > + > +# Write the last logged build id to a file > +def write_last_build(buildid): > + with open(cachefile, 'w') as fi: > + fi.write(buildid) > + > + > +# Read last logged buildid from a file > +def read_last_build(): > + last_build = '' > + try: > + with open(cachefile, 'r') as fi: > + last_build = fi.readline() > + except FileNotFoundError as ex: > + # A build hasn't been logged yet > + pass > + except Exception as e: > + print('Error reading last build %s' % str(e)) > + > + return last_build > + > + > +def watch_for_builds(configfile): > + if not load_config(configfile): > + print('Failed to start buildlogger.') > + sys.exit(1) > + last_logged = read_last_build() > + > + while True: > + # wait a minute... > + time.sleep(60) > + > + builds = requests.get(abapi) > + > + if not builds: > + print("Failed to fetch Autobuilder data. Exiting.") > + continue > + try: > + build_json = builds.json() > + except Exception as e: > + print("Failed to decode JSON: %s" % str(e)) > + continue > + > + last_build = sorted(build_json.keys())[-1] > + # If a new build is detected, post a new entry to the BuildLog > + if last_build != last_logged: > + new_entry, summary = ab_last_build_to_entry(build_json, > last_build) > + blurb, entries = wiki_get_content() > + entries = new_entry+entries > + cookies = wiki_login() > + if wiki_post_page(blurb+entries, summary, cookies): > + write_last_build(last_build) > + last_logged = last_build > + print("Entry posted:\n%s\n" % new_entry) > + else: > + print("Failed to post new entry.") > + > + sys.exit(0) > + > + > +if __name__ == "__main__": > + if len(sys.argv) < 2: > + print('Please specify the path to the config file on the command > line as the first argument.') > + sys.exit(1) > + > + # Check to see if this is running already. If so, kill it and rerun > + if os.path.exists(tmpfile) and os.path.isfile(tmpfile): > + print("A prior PID file exists. Attempting to kill.") > + with open(tmpfile, 'r') as f: > + pid=f.readline() > + try: > + os.kill(int(pid), signal.SIGKILL) > + # We need to sleep for a second or two just to give the SIGKILL > time > + time.sleep(2) > + except OSError as ex: > + print("""We weren't able to kill the prior buildlogger. Trying > again.""") > + pass > + # Check if the process that we killed is alive. > + try: > + os.kill(int(pid), 0) > + except OSError as ex: > + pass > + elif os.path.exists(tmpfile) and not os.path.isfile(tmpfile): > + raise Exception("""/tmp/.buildlogger.pid is a directory, remove it > to continue.""") > + try: > + os.unlink(tmpfile) > + except: > + pass > + with open(tmpfile, 'w') as f: > + f.write(str(os.getpid())) > + > + watch_for_builds(sys.argv[1]) > diff --git a/yocto-start-autobuilder b/yocto-start-autobuilder > index 85b748d..f8154c1 100755 > --- a/yocto-start-autobuilder > +++ b/yocto-start-autobuilder > @@ -72,6 +72,14 @@ if os.path.isfile(os.path.join(AB_BASE, ".setupdone")): > os.chdir(os.path.join(AB_BASE, "yocto-controller")) > subprocess.call(["make", "start"]) > os.chdir(AB_BASE) This should be: a. Optional and defaulting to False (something in autobuilder.conf like PUSH_TO_WIKI) b. Probably only want to run this on controller/both. If you run it on workers you're going to have a lot of workers hitting the page. Realise, most autobuilder end users won't use this functionality, so yeah, let's make sure this is only run when we tell it to. > + logger_log = open('yocto-controller/buildlogger.log', 'a') > + logger_log.write('[ buildlogger started: %s ]\n' % > datetime.datetime.now()) > + subprocess.Popen('python bin/buildlogger ' + os.path.join(AB_BASE, > 'etc/buildlogger.conf'), > + shell=True, stdin=None, > + stdout=logger_log, > + stderr=logger_log, > + close_fds=True) > + logger_log.close() > > if sys.argv[1] == "worker" or sys.argv[1] == "both": > if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == > "localhost": > diff --git a/yocto-stop-autobuilder b/yocto-stop-autobuilder > index a313b27..df5fd34 100755 > --- a/yocto-stop-autobuilder > +++ b/yocto-stop-autobuilder > @@ -48,30 +48,18 @@ for section_name in parser.sections(): > for name, value in parser.items(section_name): > os.environ[name.upper()] = value.strip('"').strip("'") > > -if sys.argv[1] == "controller" or sys.argv[1] == "both": > - os.chdir(os.path.join(AB_BASE, "yocto-controller")) > - subprocess.call(["make", "stop"]) > - os.chdir(AB_BASE) > > -if sys.argv[1] == "worker" or sys.argv[1] == "both": > - if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == > "localhost": > - os.chdir(AB_BASE) > - subprocess.call([os.path.join(AB_BASE, "ab-prserv"), "stop"]) > - > - os.chdir(os.path.join(AB_BASE, "yocto-worker")) > - subprocess.call(["make", "stop"]) > - os.chdir(AB_BASE) > - tmpfile = '/tmp/.buildworker-janitor'+os.getcwd().replace('/', '-') > - if os.path.exists(tmpfile) and os.path.isfile(tmpfile): > +def killpid(pidfile): > + if os.path.exists(pidfile) and os.path.isfile(pidfile): > print("A prior PID file exists. Attempting to kill.") > - with open(tmpfile, 'r') as f: > + with open(pidfile, 'r') as f: > pid=f.readline() > try: > os.kill(int(pid), signal.SIGKILL) > # We need to sleep for a second or two just to give the SIGKILL > time > time.sleep(2) > except OSError as ex: > - print("""We weren't able to kill the prior buildworker-janitor. > Trying again.""") > + print("""We weren't able to kill the owner of %s, trying > again.""" % pidfile) > pass > # Check if the process that we killed is alive. > try: > @@ -80,10 +68,29 @@ if sys.argv[1] == "worker" or sys.argv[1] == "both": > HINT:use signal.SIGKILL or signal.SIGABORT""") > except OSError as ex: > pass > - elif os.path.exists(tmpfile) and not os.path.isfile(tmpfile): > - raise Exception(tmpfile + """ is a directory. Remove it to > continue.""") > + elif os.path.exists(pidfile) and not os.path.isfile(pidfile): > + raise Exception(pidfile + """ is a directory. Remove it to > continue.""") > try: > - os.unlink(tmpfile) > + os.unlink(pidfile) > except: > pass > > +if sys.argv[1] == "controller" or sys.argv[1] == "both": > + os.chdir(os.path.join(AB_BASE, "yocto-controller")) > + subprocess.call(["make", "stop"]) > + os.chdir(AB_BASE) > + tmpfile = '/tmp/.buildlogger.pid' > + killpid(tmpfile) > + > + > +if sys.argv[1] == "worker" or sys.argv[1] == "both": > + if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == > "localhost": > + os.chdir(AB_BASE) > + subprocess.call([os.path.join(AB_BASE, "ab-prserv"), "stop"]) > + > + os.chdir(os.path.join(AB_BASE, "yocto-worker")) > + subprocess.call(["make", "stop"]) > + os.chdir(AB_BASE) > + tmpfile = '/tmp/.buildworker-janitor'+os.getcwd().replace('/', '-') > + killpid(tmpfile) > + > -- > 2.5.5 > > -- > _______________________________________________ > yocto mailing list > yocto@yoctoproject.org > https://lists.yoctoproject.org/listinfo/yocto -- Elizabeth Flanagan Yocto Project Build and Release -- _______________________________________________ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto