The branch, master has been updated via 99b7f0c Simplify revision_details() call. via 720b006 Add upload_build for sqlite. via 081607d add initial code for sql backed build result data. from fdfaa5a Scan for builds then verify, rather than probing.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 99b7f0ce3b11bddac4580c9c0faec01085bad04c Author: Jelmer Vernooij <jel...@samba.org> Date: Fri Nov 12 23:12:53 2010 +0100 Simplify revision_details() call. commit 720b006124fd809bf38ca37d44367d107778e813 Author: Jelmer Vernooij <jel...@samba.org> Date: Fri Nov 12 23:08:22 2010 +0100 Add upload_build for sqlite. commit 081607dfd99532b299e19df82dfc36b543502ed8 Author: Jelmer Vernooij <jel...@samba.org> Date: Fri Nov 12 22:51:50 2010 +0100 add initial code for sql backed build result data. ----------------------------------------------------------------------- Summary of changes: buildfarm/__init__.py | 10 ++++ buildfarm/data.py | 89 ++++++++++++++++++------------------- buildfarm/hostdb.py | 9 +--- buildfarm/tests/test_buildfarm.py | 4 +- buildfarm/tests/test_data.py | 19 ++++++-- web/build.py | 12 +---- 6 files changed, 74 insertions(+), 69 deletions(-) Changeset truncated at 500 lines: diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py index 40522f7..822dc1f 100644 --- a/buildfarm/__init__.py +++ b/buildfarm/__init__.py @@ -185,3 +185,13 @@ class CachingBuildFarm(BuildFarm): if not self.readonly: util.FileSave(cachefile, perc) return perc + + +def setup_db(db): + db.executescript(""" + CREATE TABLE IF NOT EXISTS host (name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int); + CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name); + CREATE TABLE IF NOT EXISTS build (id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text); + CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum); + CREATE TABLE IF NOT EXISTS test_run (build int, test text, result text, output text); + """) diff --git a/buildfarm/data.py b/buildfarm/data.py index 0eaae74..888aa73 100644 --- a/buildfarm/data.py +++ b/buildfarm/data.py @@ -21,22 +21,23 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - +from buildfarm import setup_db from cStringIO import StringIO import hashlib import os import re +import sqlite3 import time import util class BuildSummary(object): - def __init__(self, host, tree, compiler, rev, status): + def __init__(self, host, tree, compiler, revision, status): self.host = host self.tree = tree self.compiler = compiler - self.rev = rev + self.revision = revision self.status = status @@ -180,11 +181,11 @@ class Build(object): self.tree = tree self.host = host self.compiler = compiler - self.rev = rev + self.revision = rev def __repr__(self): - if self.rev: - return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.rev, self.tree, self.host, self.compiler) + if self.revision: + return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler) else: return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler) @@ -228,9 +229,7 @@ class Build(object): f.close() def summary(self): - (revid, commit_revid, timestamp) = self.revision_details() - if commit_revid: - revid = commit_revid + (revid, timestamp) = self.revision_details() status = self.status() return BuildSummary(self.host, self.tree, self.compiler, revid, status) @@ -239,23 +238,19 @@ class Build(object): :return: Tuple with revision id and timestamp (if available) """ - revid = None - commit_revid = None timestamp = None f = self.read_log() try: for l in f: if l.startswith("BUILD COMMIT REVISION: "): - commit_revid = l.split(":", 1)[1].strip() - elif l.startswith("BUILD REVISION: "): revid = l.split(":", 1)[1].strip() elif l.startswith("BUILD COMMIT TIME"): timestamp = l.split(":", 1)[1].strip() finally: f.close() - return (revid, commit_revid, timestamp) + return (revid, timestamp) def status(self): """get status of build @@ -283,8 +278,8 @@ class CachingBuild(Build): to calculate.""" def revision_details(self): - if self.rev: - cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev) + if self.revision: + cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision) else: cachef = self._store.cache_fname(self.tree, self.host, self.compiler) st1 = os.stat("%s.log" % self.basename) @@ -298,21 +293,19 @@ class CachingBuild(Build): # the ctime/mtime asymmetry is needed so we don't get fooled by # the mtime update from rsync if st2 and st1.st_ctime <= st2.st_mtime: - (revid, commit_revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2) + (revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2) if timestamp == "": timestamp = None if revid == "": revid = None - if commit_revid == "": - commit_revid = None - return (revid, commit_revid, timestamp) - (revid, commit_revid, timestamp) = super(CachingBuild, self).revision_details() + return (revid, timestamp) + (revid, timestamp) = super(CachingBuild, self).revision_details() if not self._store.readonly: - util.FileSave("%s.revision" % cachef, "%s:%s:%s" % (revid, commit_revid or "", timestamp or "")) - return (revid, commit_revid, timestamp) + util.FileSave("%s.revision" % cachef, "%s:%s" % (revid, timestamp or "")) + return (revid, timestamp) def err_count(self): - cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev) + cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision) st1 = os.stat("%s.err" % self.basename) try: @@ -332,8 +325,8 @@ class CachingBuild(Build): return ret def status(self): - if self.rev: - cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)+".status" + if self.revsion: + cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)+".status" else: cachefile = self._store.cache_fname(self.tree, self.host, self.compiler)+".status" @@ -463,10 +456,7 @@ class BuildResultStore(object): return ret def upload_build(self, build): - (rev, commit_rev, rev_timestamp) = build.revision_details() - - if commit_rev is not None: - rev = commit_rev + (rev, rev_timestamp) = build.revision_details() if not rev: raise Exception("Unable to find revision in %r log" % build) @@ -476,25 +466,9 @@ class BuildResultStore(object): if os.path.exists(build.basename+".err"): os.link(build.basename+".err", new_basename+".err") - # FIXME: - # $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - # $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html) - def get_previous_revision(self, tree, host, compiler, revision): raise NoSuchBuildError(tree, host, compiler, revision) -""" - def get_previous_revision(self, tree, host, compiler, revision): - # Look up the database to find the previous status - $st = $dbh->prepare("SELECT status, revision, commit_revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND revision != ? AND commit_revision != ? ORDER BY id DESC LIMIT 1") - $st->execute( $tree, $host, $compiler, $rev, $commit) - - while ( my @row = $st->fetchrow_array ) { - $old_status_html = @row[0] - $old_rev = @row[1] - $old_commit = @row[2] - """ - class CachingBuildResultStore(BuildResultStore): @@ -515,3 +489,26 @@ class CachingBuildResultStore(BuildResultStore): def cache_fname(self, tree, host, compiler, rev): return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev)) + + +class SQLCachingBuildResultStore(BuildResultStore): + + def __init__(self, basedir, db=None): + super(SQLCachingBuildResultStore, self).__init__(basedir) + + if db is None: + db = sqlite3.connect(":memory:") + setup_db(db) + + self.db = db + + def get_previous_revision(self, tree, host, compiler, revision): + cursor = self.db.execute("SELECT revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND revision < ? ORDER BY id DESC LIMIT 1", (tree, host, compiler, revision)) + row = cursor.fetchone() + if row is None: + raise NoSuchBuildError(tree, host, compiler, revision) + return row[0] + + def upload_build(self, build): + super(SQLCachingBuildResultStore, self).upload_build(build) + self.db.execute("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (build.tree, build.revision, build.revision, build.host, build.compiler, build.log_checksum(), build.age_mtime(), repr(build.status()))) diff --git a/buildfarm/hostdb.py b/buildfarm/hostdb.py index 3c42757..2474505 100644 --- a/buildfarm/hostdb.py +++ b/buildfarm/hostdb.py @@ -18,6 +18,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # +from buildfarm import setup_db import sqlite3 import time @@ -68,13 +69,7 @@ class HostDatabase(object): else: self.db = sqlite3.connect(filename) self.filename = filename - self.db.executescript(""" - CREATE TABLE IF NOT EXISTS host ( name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int ); - CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name); - CREATE TABLE IF NOT EXISTS build ( id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text); - CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum); - CREATE TABLE IF NOT EXISTS test_run ( build int, test text, result text, output text); - """) + setup_db(self.db) self.db.commit() def createhost(self, name, platform=None, owner=None, owner_email=None, password=None, permission=None): diff --git a/buildfarm/tests/test_buildfarm.py b/buildfarm/tests/test_buildfarm.py index 86e249a..02eda60 100644 --- a/buildfarm/tests/test_buildfarm.py +++ b/buildfarm/tests/test_buildfarm.py @@ -100,7 +100,7 @@ class BuildFarmTestBase(object): self.assertEquals("tdb", build.tree) self.assertEquals("charis", build.host) self.assertEquals("cc", build.compiler) - self.assertEquals("12", build.rev) + self.assertEquals("12", build.revision) def test_get_build_no_rev(self): path = self.create_mock_logfile("tdb", "charis", "cc", @@ -109,7 +109,7 @@ class BuildFarmTestBase(object): self.assertEquals("tdb", build.tree) self.assertEquals("charis", build.host) self.assertEquals("cc", build.compiler) - self.assertIs(None, build.rev) + self.assertIs(None, build.revision) class BuildFarmTests(BuildFarmTestBase, BuildFarmTestCase): diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py index 9824c51..4b60ad8 100755 --- a/buildfarm/tests/test_data.py +++ b/buildfarm/tests/test_data.py @@ -94,11 +94,10 @@ class BuildResultStoreTestBase(object): self.create_mock_logfile("tdb", "charis", "cc", "12", contents=""" BUILD COMMIT REVISION: 43 bla -BUILD REVISION: 42 BUILD COMMIT TIME: 3 August 2010 """) build = self.x.get_build("tdb", "charis", "cc", "12") - self.assertEquals(("42", "43", "3 August 2010"), build.revision_details()) + self.assertEquals(("43", "3 August 2010"), build.revision_details()) def test_revision_details_no_timestamp(self): self.create_mock_logfile("tdb", "charis", "cc", rev="12", contents=""" @@ -107,7 +106,7 @@ BUILD REVISION: 42 BLA """) build = self.x.get_build("tdb", "charis", "cc", "12") - self.assertEquals(("42", "43", None), build.revision_details()) + self.assertEquals(("43", None), build.revision_details()) def test_err_count(self): self.create_mock_logfile("tdb", "charis", "cc", "12") @@ -132,6 +131,9 @@ BUILD COMMIT REVISION: myrev build = data.Build(None, path[:-4], "tdb", "charis", "cc") self.assertRaises(Exception, self.x.upload_build, build) + def test_get_previous_revision(self): + self.assertRaises(data.NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12") + class BuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase): @@ -158,6 +160,15 @@ class CachingBuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase): "%s/cache/build.mytree.myhost.cc-123" % self.path) +class SQLCachingBuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase): + + def setUp(self): + super(SQLCachingBuildResultStoreTests, self).setUp() + + self.x = data.SQLCachingBuildResultStore( + os.path.join(self.path, "data", "oldrevs")) + + class BuildStatusFromLogs(testtools.TestCase): def parse_logs(self, log, err): @@ -330,5 +341,3 @@ class CachingUploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFar self.assertEquals( self.x.cache_fname("mytree", "myhost", "cc"), "%s/cache/build.mytree.myhost.cc" % self.path) - - diff --git a/web/build.py b/web/build.py index 13700c1..43401de 100755 --- a/web/build.py +++ b/web/build.py @@ -280,9 +280,7 @@ def view_recent_builds(myself, tree, sort_by): else: age_mtime = build.age_mtime() age_ctime = build.age_ctime() - (revision, commit_revision, revision_time) = build.revision_details() - if commit_revision: - revision = commit_revision + (revision, revision_time) = build.revision_details() if revision: all_builds.append([age_ctime, host.platform.encode("utf-8"), @@ -393,9 +391,7 @@ def view_build(myself, tree, host, compiler, rev, plain_logs=False): config = "" build = buildfarm.get_build(tree, host, compiler, rev) age_mtime = build.age_mtime() - (revision, commit_revision, revision_time) = build.revision_details() - if commit_revision: - revision = commit_revision + (revision, revision_time) = build.revision_details() status = build_status_html(myself, build) if rev: @@ -524,9 +520,7 @@ def view_host(myself, output_type, *requested_hosts): except data.NoSuchBuildError: pass else: - (revision, commit_revision, revision_time) = build.revision_details() - if commit_revision: - revision = commit_revision + (revision, revision_time) = build.revision_details() age_mtime = build.age_mtime() age_ctime = build.age_ctime() warnings = build.err_count() -- build.samba.org