Author: wade
Date: 2006-06-15 18:22:18 -0400 (Thu, 15 Jun 2006)
New Revision: 61781
Modified:
trunk/release/monobuild/sync-daemon.py
trunk/release/monobuild/tarball-builder-daemon.py
trunk/release/monobuild/www/python/monobuild.py
trunk/release/packaging/do-zip-build
trunk/release/pyutils/build.py
trunk/release/pyutils/datastore.py
Log:
do-zip-build:
-require parameter in def file for patches to be applied
monobuild.py:
-add rows for mktarball states and links
-get rid of scheduling cruft since architecture changed
sync-daemon.py:
-sync tarball_map and latest n tarball logs
tarball-builder-daemon.py
-send me an email if mktarball fails
build.py:
-get rid of old scheduling cruft
datastore.py:
-add helper methods to get status and latest revisions for monobuild webview
Modified: trunk/release/monobuild/sync-daemon.py
===================================================================
--- trunk/release/monobuild/sync-daemon.py 2006-06-15 22:05:25 UTC (rev
61780)
+++ trunk/release/monobuild/sync-daemon.py 2006-06-15 22:22:18 UTC (rev
61781)
@@ -32,6 +32,10 @@
def sync_dirs():
dirs = []
+
+ # Add tarball_map
+ dirs += ['../../../packaging/tarball_map']
+
# Gather dirs to synchronize
for i in ['HEAD', 'RELEASE']:
for distro in os.listdir(config.build_info_dir + os.sep + i):
@@ -41,12 +45,20 @@
for j in versions:
dirs.append(os.path.join(i, distro,
component, j))
+ # Grab latest num_builds for tarball log files as well
+ tarball_path = os.path.join(config.build_info_dir, '..',
'tarball_logs', i)
+ for component in os.listdir(tarball_path):
+ versions = os.listdir(tarball_path + os.sep + component)
+ versions.sort()
+ for j in versions[-num_builds:]:
+ dirs.append(os.path.join('..', 'tarball_logs',
i, component, j))
+
os.chdir(config.build_info_dir)
# For some reason the --delete option crashes when running the second
time to go-mono.com and mono.ximian.com ... ?
# rsync all files over, and don't include the builds... just logs and
info.xml
- status, output = utils.launch_process('rsync -avzR -e ssh --copy-links
--exclude=files %s %s:%s' % (" ".join(dirs), host, target_dir))
+ status, output = utils.launch_process('rsync -avzR -e ssh
--exclude=files %s %s:%s' % (" ".join(dirs), host, target_dir))
while(1):
Modified: trunk/release/monobuild/tarball-builder-daemon.py
===================================================================
--- trunk/release/monobuild/tarball-builder-daemon.py 2006-06-15 22:05:25 UTC
(rev 61780)
+++ trunk/release/monobuild/tarball-builder-daemon.py 2006-06-15 22:22:18 UTC
(rev 61781)
@@ -87,7 +87,9 @@
log.log("Tarball
creation failed...\n")
distfiles.add_file('HEAD', pack_name, str(latest_for_package),
"tarball_creation_failed")
- utils.send_mail('[EMAIL
PROTECTED]', '[EMAIL PROTECTED]', 'mktarball failed (%s %d)' % (pack_name,
latest_for_package), "mktarball has failed for package %s revision %d" %
(pack_name, latest_for_package))
+ # Send out the log with
the tarball, or at least a link... ?
+ link =
"http://monobuild1.boston.ximian.com/tarball_logs/HEAD/%s/%d.log"; % (pack_name,
latest_for_package)
+ utils.send_mail('[EMAIL
PROTECTED]', '[EMAIL PROTECTED]', 'mktarball failed (%s %d)' % (pack_name,
latest_for_package), "mktarball has failed for package %s revision %d\n\n%s" %
(pack_name, latest_for_package, link))
# TODO: Don't sleep if the above loop took longer than
max_poll_interval
if not sigint_event.isSet():
Modified: trunk/release/monobuild/www/python/monobuild.py
===================================================================
--- trunk/release/monobuild/www/python/monobuild.py 2006-06-15 22:05:25 UTC
(rev 61780)
+++ trunk/release/monobuild/www/python/monobuild.py 2006-06-15 22:22:18 UTC
(rev 61781)
@@ -35,11 +35,8 @@
plat_objs = build.get_platform_objs()
pack_objs = build.get_package_objs()
- schedule_flag = ""
- if vars.has_key("schedule"): schedule_flag = vars["schedule"]
+ req.content_type = "text/html"
- req.content_type = "text/html"
-
req.write( """
<HTML>
<HEAD>
@@ -51,19 +48,8 @@
<H1>Mono Build Status - %s</H1>
- <FORM name=buildform action="%s/schedulebuild" method=get
enctype="multipart/form-data">
+ """ % (HEAD_or_RELEASE, config.web_root_url, config.web_root_url,
HEAD_or_RELEASE))
- """ % (HEAD_or_RELEASE, config.web_root_url, config.web_root_url,
HEAD_or_RELEASE, scriptname))
-
- if schedule_flag:
- req.write("""<p><INPUT type=submit Value="Build Selected
Packages"></p>""")
-
-
- checkbox_html = ""
-
- if schedule_flag:
- checkbox_html = """Select All <INPUT type=checkbox name=linux
onClick=toggleCheckBoxes(document.buildform.build)>"""
-
req.write("""
<h3>Build Matrix</h3>
@@ -101,16 +87,57 @@
count = 0
+ # Only read once here
+ src = datastore.source_file_repo()
+ src.load_info()
+
# Iterate through arch packages, and then noarch packages
for (platforms, packs) in [ (plat_objs, pack_objs), (['noarch'],
noarch_pack_objs) ]:
- req.write("<thead><tr><td>%s</td>\n" % checkbox_html)
+ req.write("<thead><tr><td></td>\n")
for obj in packs:
req.write("<th>%s</th>\n" % obj.name)
req.write("</tr></thead>\n<tbody>\n")
+ # Add row for mktarball status
+ req.write("<tr><th>mktarball</th>\n")
+ for package in packs:
+ info = src.get_tarball_state_info(HEAD_or_RELEASE,
package.name, read_info=False)
+
+ revision = ""
+ revision2 = ""
+ state = ""
+ state2 = ""
+ if len(info['revisions']) > 0:
+ revision = info['revisions'][0]
+ state = info['state'][revision]
+
+ link = "<a
href=%s/tarball_logs/%s/%s/%s.log>%s</a>" % (config.web_root_url,
HEAD_or_RELEASE, package.name, revision, revision)
+
+ if len(info['revisions']) > 1:
+ revision2 = info['revisions'][1]
+ state2 = info['state'][revision2]
+
+ if revision2:
+ req.write("<td class=%s>" % state2)
+ req.write("<center><table
class=%s><td>" % (state))
+ req.write(link)
+ req.write("</td></table></center>")
+ req.write("</td>\n")
+ else:
+ req.write("<td class=%s>" % state)
+ req.write(link)
+ req.write("</td>\n")
+
+ else:
+ req.write("<td class=new>")
+ req.write("</td>\n")
+
+ req.write("</tr>\n")
+ # End of mktarball row
+
# list out non-noarch packages
for platform in platforms:
if platform == 'noarch':
@@ -125,7 +152,7 @@
# if this is a valid package for this
platform...
if
package.info['BUILD_HOSTS'].count(platform_name):
- # todo: show the last two builds
(otherwise, it's going to mostly be showing yellow)
+ # show the last two builds (otherwise,
it's going to mostly be showing yellow)
versions =
build.get_versions(HEAD_or_RELEASE, platform_name, package.name) # in <num>
format
version = ""
@@ -154,9 +181,9 @@
# if there's a previous version, print
a container table
if version2:
req.write("<td class=%s>" %
state2)
- req.write("<table
class=%s><td>" % (state))
+ req.write("<center><table
class=%s><td>" % (state))
req.write(link)
- req.write("</td></table>")
+
req.write("</td></table></center>")
req.write("</td>\n")
else:
req.write("<td class=%s>" %
state)
@@ -180,69 +207,38 @@
req.write("</table>\n</p>")
- # Old Legend
- if 0:
- req.write("""
+ # Legend
+ req.write("""
- <h3>Legend</h3>
- <p>
- <table class=legend>
+ <h3>Legend</h3>
+ <p>
+ <table class=legend>
- <tbody>
- <tr><th>In Progress</th><td class=inprogress></td></tr>
- <tr><th>Success</td><td class=success></td></tr>
- <tr><th>Failed</td><td class=failure></td></tr>
- <tr><th>Tests Failed</td><td class=testfailure></td></tr>
- <tr><th>Timed Out</td><td class=timeout></td></tr>
- <tr><th>Queued</td><td class=queued></td></tr>
- <tr><th>New</td><td class=new></td></tr>
+ <tbody>
+ <tr>
+ <th>In Progress</th>
+ <th>Success</th>
+ <th>Failed</th>
+ <th>Tests Failed</th>
+ <th>Timed Out</td>
+ <th>Queued</th>
+ <th>New</th>
+ </tr>
- </tbody>
- </table>
- </p>
+ <td class=inprogress></td>
+ <td class=success></td>
+ <td class=failure></td>
+ <td class=testfailure></td>
+ <td class=timeout></td>
+ <td class=queued></td>
+ <td class=new></td>
- """)
+ </tbody>
+ </table>
+ </p>
- # New Legend
- else:
- req.write("""
+ """)
- <h3>Legend</h3>
- <p>
- <table class=legend>
-
- <tbody>
- <tr>
- <th>In Progress</th>
- <th>Success</th>
- <th>Failed</th>
- <th>Tests Failed</th>
- <th>Timed Out</td>
- <th>Queued</th>
- <th>New</th>
- </tr>
-
- <td class=inprogress></td>
- <td class=success></td>
- <td class=failure></td>
- <td class=testfailure></td>
- <td class=timeout></td>
- <td class=queued></td>
- <td class=new></td>
-
- </tbody>
- </table>
- </p>
-
- """)
-
-
- if schedule_flag:
- req.write("""<p><a href="%s">Build Status</a></p>""" %
scriptname)
- else:
- #req.write("""<p><a href="%s?schedule=true">Schedule
Builds</a></p>""" % scriptname)
- pass
-
# Link to toggle between HEAD and RELEASE
if HEAD_or_RELEASE == "RELEASE":
toggle_link = "HEAD"
@@ -407,58 +403,3 @@
</html>""" % (config.web_root_url, html))
-
-def schedulebuild(req, **vars):
-
- builds = ""
- scriptname = os.path.basename(req.canonical_filename)
-
- # Only select keys if they exist, and make sure the args are always
treated as lists
- if vars.has_key('build'):
- builds = vars['build']
- if builds.__class__ != list:
- builds = [ builds ]
-
-
- html = """
- <HTML>
- <HEAD>
- <TITLE>Mono Build Control</TITLE>
- <link rel="stylesheet" href="%s/build.css"
type="text/css">
- </HEAD>
- <BODY>
-
- <H1>Mono Build Control</H1>
-
- <H3>Scheduled the following builds</H3>
-
- <p>""" % config.web_root_url
-
- #latest_rev = Mono.Build.getLatestTreeRevision()
- latest_rev = "57777"
- latest_rev = "57635"
-
- #if latest_rev:
- # html += "Error scheduling build<br>"
-
- html += "Tree revision: %s<br>" % latest_rev
-
- for build in builds:
- #html += "Build: " + build + "<br>"
-
- (platform, package) = build.split(":")
- html += "Platform: %s, package: %s<br>" % (platform, package)
- Mono.Build.scheduleBuild(platform, package, latest_rev)
-
- html += """
-
- <a href=../%s>Back</a>
-
- </p>
-
- </BODY>
- </HTML>""" % scriptname
-
- req.content_type = "text/html"
- req.write(html)
-
Modified: trunk/release/packaging/do-zip-build
===================================================================
--- trunk/release/packaging/do-zip-build 2006-06-15 22:05:25 UTC (rev
61780)
+++ trunk/release/packaging/do-zip-build 2006-06-15 22:22:18 UTC (rev
61781)
@@ -51,16 +51,19 @@
cd $component_name_ver
-# Apply patches
-for i in `ls ../*.patch | sort -n` ; do
- patch -p1 < $i || exit 1
-done
-
# Load environment for deps
. /tmp/build_deps/env.sh
# Call the function for this destroot
. /tmp/scratch/$package
+
+# Apply patches (if the def file has USE_PATCHES in it)
+if [ x$USE_PATCHES == "x1" ] ; then
+ for i in `ls ../*.patch | sort -n` ; do
+ patch -p1 < $i || exit 1
+ done
+fi
+
DISTRO_UNDERSCORES=${distro//-/_}
${DISTRO_UNDERSCORES}_ZIP_BUILD || exit 1
# TODO: For some reason the above exit isn't being called on failures
Modified: trunk/release/pyutils/build.py
===================================================================
--- trunk/release/pyutils/build.py 2006-06-15 22:05:25 UTC (rev 61780)
+++ trunk/release/pyutils/build.py 2006-06-15 22:22:18 UTC (rev 61781)
@@ -115,86 +115,4 @@
return versions
-# Args: platform, package
-# Returns: success or fail
-def scheduleBuild(platform, package, rev):
- #if debug:
- # sys.stderr.write("scheduleBuild: Latest rev: %s\n" % rev)
-
- # Check to see if this isn't here already...
-
- # Start outputting the structure
- #if debug: sys.stderr.write(Mono.Build.Config.buildsDir)
- dir = os.path.join(Mono.Build.Config.buildsDir, platform, package, rev)
-
- # If it hasn't already been scheduled
- if not os.path.exists(dir):
-
- if debug: sys.stderr.write("Creating structure (%s)\n" % dir)
- #try:
- distutils.dir_util.mkpath(dir)
- distutils.dir_util.mkpath(dir + os.sep + "files")
- distutils.dir_util.mkpath(dir + os.sep + "logs")
-
- os.chmod(dir, all_rwx)
- os.chmod(dir + os.sep + "files", all_rwx)
- os.chmod(dir + os.sep + "logs", all_rwx)
-
- # Make each one world writable
-
- #except:
- # sys.stderr.write("Failed to create directory structure
(%s)\n" % dir)
- # return "create_dir error"
-
- # TODO: new xml data store
-
- return ""
-
- else:
-
- # Either the data is bogus, or it's already been scheduled...
-
- # TODO What to do when the build exists? Probably want to
schedule it again if it's finished
- # If it's in the finished state, queue it again
- return "already scheduled"
-
-
-# Hmm... does this method make sense anymore... ?
-def get_queued_packages(self):
-
- queuedPackages = []
-
- distros = []
- packages = []
- revisions = []
- latestRev = ""
- state = ""
-
- platforms = glob.glob(Mono.Build.Config.buildsDir + "/*")
-
- for platform in platforms:
-
- platform = os.path.basename(platform)
-
- packages = glob.glob(Mono.Build.Config.buildsDir + os.sep +
platform + os.sep + "*");
-
- for package in packages:
-
- package = os.path.basename(package)
-
- revisions = glob.glob(Mono.Build.Config.buildsDir +
os.sep + platform + os.sep + package + os.sep + "*")
- revisions.sort()
-
- latestRev = revisions.pop()
-
- latestRev = os.path.basename(latestRev)
-
- state = getState(platform, package, latestRev)
-
- if state == "queued":
- queuedPackages.append(":".join([platform,
package, latestRev]))
-
- return queuedPackages
-
-
Modified: trunk/release/pyutils/datastore.py
===================================================================
--- trunk/release/pyutils/datastore.py 2006-06-15 22:05:25 UTC (rev 61780)
+++ trunk/release/pyutils/datastore.py 2006-06-15 22:22:18 UTC (rev 61781)
@@ -102,6 +102,7 @@
fd.close()
def get_log_file(self, source_file, package_name_alias=""):
+ """Find a key (mktarball parameters) based on a source
filename."""
self.load_info()
key = ""
@@ -127,6 +128,7 @@
return config.mktarball_logs_release_relpath + os.sep +
key.replace(":", "/") + ".log"
def get_latest_tarball(self, HEAD_or_RELEASE, package_name):
+ """Find latest tarball filename for a component."""
self.load_info()
versions = []
@@ -149,6 +151,7 @@
return latest_filename
def get_tarball_version(self, HEAD_or_RELEASE, package_name, rev):
+ """Grab version from tarball filename."""
self.load_info()
source_path = self.info[":".join([HEAD_or_RELEASE,
package_name, rev])]
@@ -161,8 +164,38 @@
return version
+ def get_tarball_state_info(self, HEAD_or_RELEASE, component,
read_info=True):
+ """Get info map for the monobuild web view.
+ Returns a map with the following keys: name, revisions, state.
revisions is a reverse
+ sorted array. Status is another map with revisions as keys and
successs or failure as values
+ You can set read_info=False if you don't want to read in the
data again.
+ """
+ if read_info:
+ self.load_info()
+
+ info = {}
+ info['name'] = component
+ info['revisions'] = []
+ info['state'] = {}
+
+ for key in self.info.iterkeys():
+ H_or_R, name, revision = key.split(":")
+ if H_or_R == HEAD_or_RELEASE and name == component:
+ if self.info[key] == "tarball_creation_failed":
+ state = "failure"
+ else: state = "success"
+
+ info['revisions'].append(revision)
+ info['state'][revision] = state
+
+ info['revisions'].sort()
+ info['revisions'].reverse()
+
+ return info
+
+
# XML data store
class build_info:
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches
