Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py: Add some basic logging
On Wed, 2016-06-29 at 16:08 -0700, Graydon, Tracy wrote: > This patch adds some basic logging to help find failure point should > the script > barf due to lost ssh session, etc. Without it, finding where to > resume is not > particularly entertaining. > > Signed-off-by: Graydon, Tracy > --- > bin/release_scripts/release.py | 43 > +++--- > 1 file changed, 40 insertions(+), 3 deletions(-) > > diff --git a/bin/release_scripts/release.py > b/bin/release_scripts/release.py > index 89f68be..b62b48c 100755 > --- a/bin/release_scripts/release.py > +++ b/bin/release_scripts/release.py > @@ -10,6 +10,7 @@ __maintainer__ = "Tracy Graydon" > __email__ = "tracy.gray...@intel.com" > ''' > @@ -348,9 +355,18 @@ if __name__ == '__main__': > os.system("clear") > print > > +logfile = 'staging.log' > +try: > +os.remove(logfile) > +except OSError: > +pass > + > +logging.basicConfig(format='%(levelname)s:%(message)s',filename= > logfile,level=logging.INFO) > + > VHOSTS = "/srv/www/vhosts" > AB_BASE = os.path.join(VHOSTS, > "autobuilder.yoctoproject.org/pub/releases") > DL_DIR = os.path.join(VHOSTS, > "downloads.yoctoproject.org/releases") > +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") This will result in DL_BASE being /releases/yocto You shouldn't include a path separator in any of the components after the first. From the os.path.join() docs: "If a component is an absolute path, all previous components are thrown away and joining continues from the absolute path component." i.e. >>> import os >>> os.path.join('/some', 'path', 'foo', 'bar') '/some/path/foo/bar' >>> os.path.join('/some', 'path', '/foo', 'bar') '/foo/bar' Cheers, Joshua -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py: Add some basic logging
This patch adds some basic logging to help find failure point should the script barf due to lost ssh session, etc. Without it, finding where to resume is not particularly entertaining. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 43 +++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 89f68be..b62b48c 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -10,6 +10,7 @@ __maintainer__ = "Tracy Graydon" __email__ = "tracy.gray...@intel.com" ''' +import logging import os import optparse import sys @@ -82,6 +83,7 @@ def rejoin_thing(thing, marker): def fix_tarballs(): print print "Repackaging poky and eclipse tarballs" +logging.info('Repackaging poky and eclipse tarballs.') os.chdir(RELEASE_DIR) os.mkdir(TARBALL_DIR) os.system("mv %s/*.tar.bz2 %s" %(RELEASE_DIR, TARBALL_DIR)) @@ -90,6 +92,7 @@ def fix_tarballs(): dirlist = get_list(TARBALL_DIR) for blob in dirlist: print "Original Tarball: %s" %blob +logging.info('Repackaging %s' %blob) chunks = split_thing(blob, ".") filename = chunks[0] basename = split_thing(filename, "-") @@ -99,6 +102,7 @@ def fix_tarballs(): chunks[0] = new_name new_blob = rejoin_thing(chunks, ".") print "New Tarball: %s" %new_blob +logging.info('New blob is %s' %new_blob) os.system("tar jxf %s" %blob) os.system("mv %s %s" %(filename, new_name)) os.system("rm -rf %s/.git*" %new_name) @@ -107,10 +111,13 @@ def fix_tarballs(): rmtree(new_name) os.symlink(new_blob, blob) os.system("md5sum %s > %s.md5sum" %(new_blob, new_blob)) +logging.info('Successful.') print +logging.info('Moving new blobs to release diri and cleaning up.') os.system("mv * %s" %RELEASE_DIR) os.chdir(RELEASE_DIR) os.rmdir(TARBALL_DIR) +logging.info('Successful.') print return @@ -348,9 +355,18 @@ if __name__ == '__main__': os.system("clear") print +logfile = 'staging.log' +try: +os.remove(logfile) +except OSError: +pass + + logging.basicConfig(format='%(levelname)s:%(message)s',filename=logfile,level=logging.INFO) + VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases @@ -453,40 +469,59 @@ if __name__ == '__main__': # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens +logging.info('Start rsync.') +print "Doing the rsync for the staging directory." sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED) - +logging.info('Successful.') + # 2) Convert the symlinks in build-appliance dir. print "Converting the build-appliance symlink." +logging.info('Converting build-appliance symlink.') convert_symlinks(BUILD_APP_DIR) +logging.info('Successful.') # 3) In machines dir, convert the symlinks, delete the cruft print "Cleaning up the machines dirs, converting symlinks." +logging.info('Machines dir cleanup started.') dirlist = get_list(MACHINES) for dirname in dirlist: dirname = os.path.join(MACHINES, dirname) +logging.info('Converting symlinks in %s' %dirname) convert_symlinks(dirname) +logging.info('Successful.') +logging.info('Nuking cruft in %s' %dirname) nuke_cruft(dirname, CRUFT_LIST) +logging.info('Successful.') print "Generating fresh md5sums." +logging.info('Generating fresh md5sums.') gen_md5sum(MACHINES) +logging.info('Successful.') # For major and point releases if REL_TYPE == "major" or REL_TYPE == "point": # 4) Fix up the eclipse and poky tarballs print "Cleaning up the eclipse, poky and other tarballs." +logging.info('Fixing tarballs.') fix_tarballs() - + # 5) Publish the eclipse stuff print "Publishing the eclipse plugins." +logging.info('Publishing eclipse plugins.') pub_eclipse(ECLIPSE_DIR, PLUGIN_DIR) +logging.info('Successful.') # 6) Make the bsps print "Generating the BSP tarballs." +logging.info('Generating BSP tarballs.') make_bsps(BSP_LIST, BSP_DIR) +logging.info('Successful.') # 7) Generate the master md5sum file for the release (for all releases) print "Generating the master md5sum table." +logging.info('Generating the master md5sum table.') gen_rel_md5(RELEASE_DIR, REL_MD5_F
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py: Add some basic logging
This patch adds some basic logging to help find failure point should the script barf due to lost ssh session, etc. Without it, finding where to resume is not particularly entertaining. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 43 +++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 89f68be..b62b48c 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -10,6 +10,7 @@ __maintainer__ = "Tracy Graydon" __email__ = "tracy.gray...@intel.com" ''' +import logging import os import optparse import sys @@ -82,6 +83,7 @@ def rejoin_thing(thing, marker): def fix_tarballs(): print print "Repackaging poky and eclipse tarballs" +logging.info('Repackaging poky and eclipse tarballs.') os.chdir(RELEASE_DIR) os.mkdir(TARBALL_DIR) os.system("mv %s/*.tar.bz2 %s" %(RELEASE_DIR, TARBALL_DIR)) @@ -90,6 +92,7 @@ def fix_tarballs(): dirlist = get_list(TARBALL_DIR) for blob in dirlist: print "Original Tarball: %s" %blob +logging.info('Repackaging %s' %blob) chunks = split_thing(blob, ".") filename = chunks[0] basename = split_thing(filename, "-") @@ -99,6 +102,7 @@ def fix_tarballs(): chunks[0] = new_name new_blob = rejoin_thing(chunks, ".") print "New Tarball: %s" %new_blob +logging.info('New blob is %s' %new_blob) os.system("tar jxf %s" %blob) os.system("mv %s %s" %(filename, new_name)) os.system("rm -rf %s/.git*" %new_name) @@ -107,10 +111,13 @@ def fix_tarballs(): rmtree(new_name) os.symlink(new_blob, blob) os.system("md5sum %s > %s.md5sum" %(new_blob, new_blob)) +logging.info('Successful.') print +logging.info('Moving new blobs to release diri and cleaning up.') os.system("mv * %s" %RELEASE_DIR) os.chdir(RELEASE_DIR) os.rmdir(TARBALL_DIR) +logging.info('Successful.') print return @@ -348,9 +355,18 @@ if __name__ == '__main__': os.system("clear") print +logfile = 'staging.log' +try: +os.remove(logfile) +except OSError: +pass + + logging.basicConfig(format='%(levelname)s:%(message)s',filename=logfile,level=logging.INFO) + VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases @@ -453,40 +469,59 @@ if __name__ == '__main__': # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens +logging.info('Start rsync.') +print "Doing the rsync for the staging directory." sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED) - +logging.info('Successful.') + # 2) Convert the symlinks in build-appliance dir. print "Converting the build-appliance symlink." +logging.info('Converting build-appliance symlink.') convert_symlinks(BUILD_APP_DIR) +logging.info('Successful.') # 3) In machines dir, convert the symlinks, delete the cruft print "Cleaning up the machines dirs, converting symlinks." +logging.info('Machines dir cleanup started.') dirlist = get_list(MACHINES) for dirname in dirlist: dirname = os.path.join(MACHINES, dirname) +logging.info('Converting symlinks in %s' %dirname) convert_symlinks(dirname) +logging.info('Successful.') +logging.info('Nuking cruft in %s' %dirname) nuke_cruft(dirname, CRUFT_LIST) +logging.info('Successful.') print "Generating fresh md5sums." +logging.info('Generating fresh md5sums.') gen_md5sum(MACHINES) +logging.info('Successful.') # For major and point releases if REL_TYPE == "major" or REL_TYPE == "point": # 4) Fix up the eclipse and poky tarballs print "Cleaning up the eclipse, poky and other tarballs." +logging.info('Fixing tarballs.') fix_tarballs() - + # 5) Publish the eclipse stuff print "Publishing the eclipse plugins." +logging.info('Publishing eclipse plugins.') pub_eclipse(ECLIPSE_DIR, PLUGIN_DIR) +logging.info('Successful.') # 6) Make the bsps print "Generating the BSP tarballs." +logging.info('Generating BSP tarballs.') make_bsps(BSP_LIST, BSP_DIR) +logging.info('Successful.') # 7) Generate the master md5sum file for the release (for all releases) print "Generating the master md5sum table." +logging.info('Generating the master md5sum table.') gen_rel_md5(RELEASE_DIR, REL_MD5_F
Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
On 05/24/16, 4:30 PM, "Randle, William C" wrote: >On Tue, 2016-05-24 at 16:06 -0700, Graydon, Tracy wrote: >> Get rid of the extraneous DL_BASE line. It's not used and should have >> been removed when I fixed the eclipse-plugin release path. >> >> Signed-off-by: Graydon, Tracy >> --- >> bin/release_scripts/release.py | 1 - >> 1 file changed, 1 deletion(-) >> >> diff --git a/bin/release_scripts/release.py >>b/bin/release_scripts/release.py >> index 2b4d21c..89f68be 100755 >> --- a/bin/release_scripts/release.py >> +++ b/bin/release_scripts/release.py >> @@ -351,7 +351,6 @@ if __name__ == '__main__': >> VHOSTS = "/srv/www/vhosts" >> AB_BASE = os.path.join(VHOSTS, >> "autobuilder.yoctoproject.org/pub/releases") >> DL_DIR = os.path.join(VHOSTS, >>"downloads.yoctoproject.org/releases") >> -DL_BASE = os.path.join(DL_DIR, "/releases/yocto") >> ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") >> >> # List of the directories we delete from all releases > >Since your previous version of this patch has not yet been applied to >master, I >would think it would be better to incorporate this as part of a V2 of your >previous patch, rather than a new patch on top of an as yet unapplied >patch. Ah, you are right. I thought it had been pulled in already. I’ll rebase. > >-Bill -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
On Tue, 2016-05-24 at 16:06 -0700, Graydon, Tracy wrote: > Get rid of the extraneous DL_BASE line. It's not used and should have > been removed when I fixed the eclipse-plugin release path. > > Signed-off-by: Graydon, Tracy > --- > bin/release_scripts/release.py | 1 - > 1 file changed, 1 deletion(-) > > diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py > index 2b4d21c..89f68be 100755 > --- a/bin/release_scripts/release.py > +++ b/bin/release_scripts/release.py > @@ -351,7 +351,6 @@ if __name__ == '__main__': > VHOSTS = "/srv/www/vhosts" > AB_BASE = os.path.join(VHOSTS, > "autobuilder.yoctoproject.org/pub/releases") > DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") > -DL_BASE = os.path.join(DL_DIR, "/releases/yocto") > ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") > > # List of the directories we delete from all releases Since your previous version of this patch has not yet been applied to master, I would think it would be better to incorporate this as part of a V2 of your previous patch, rather than a new patch on top of an as yet unapplied patch. -Bill -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
Get rid of the extraneous DL_BASE line. It's not used and should have been removed when I fixed the eclipse-plugin release path. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 2b4d21c..89f68be 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -351,7 +351,6 @@ if __name__ == '__main__': VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") -DL_BASE = os.path.join(DL_DIR, "/releases/yocto") ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases -- 2.7.0 -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
On 05/24/16, 8:46 AM, "Randle, William C" wrote: >On Mon, 2016-05-23 at 19:07 -0700, Graydon, Tracy wrote: >> Fixed the path for the publishing of the eclipse plugins. They were >> going to the wrong subdirectory under downloads. >> >> Signed-off-by: Graydon, Tracy >> --- >> bin/release_scripts/release.py | 22 +++--- >> 1 file changed, 11 insertions(+), 11 deletions(-) >> >> diff --git a/bin/release_scripts/release.py >>b/bin/release_scripts/release.py >> index 51cd607..3d4ce39 100755 >> --- a/bin/release_scripts/release.py >> +++ b/bin/release_scripts/release.py >> @@ -340,18 +340,18 @@ def publish_adt(rel_id, rel_type, opts): >> QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname) >> print "QEMU_SRC: %s" %QEMU_SRC >> sync_it(QEMU_SRC, QEMU_TARGET, "") >> - >> sync_it(IPK_DIR, ADT_IPK, "") >> return >> >> if __name__ == '__main__': >> - >> + >> os.system("clear") >> print >> - >> + >> VHOSTS = "/srv/www/vhosts" >> AB_BASE = os.path.join(VHOSTS, >> "autobuilder.yoctoproject.org/pub/releases") >> -DL_BASE = os.path.join(VHOSTS, >> "downloads.yoctoproject.org/releases/yocto") >> +DL_DIR = os.path.join(VHOSTS, >>"downloads.yoctoproject.org/releases") >> +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") >> ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") >> >So is the intent is to create a new DL_BASE >directory downloads.yoctoproject.org/releases/releases/yocto or should >that be: >DL_BASE = os.path.join(DL_DIR, "/yocto") > >-Bill > Argh. No, that DL_BASE line shouldn’t be there. It’s cruft. I thought I had deleted it. Thanks for catching. I will resubmit the patch with that fix. -t -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
On Mon, 2016-05-23 at 19:07 -0700, Graydon, Tracy wrote: > Fixed the path for the publishing of the eclipse plugins. They were > going to the wrong subdirectory under downloads. > > Signed-off-by: Graydon, Tracy > --- > bin/release_scripts/release.py | 22 +++--- > 1 file changed, 11 insertions(+), 11 deletions(-) > > diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py > index 51cd607..3d4ce39 100755 > --- a/bin/release_scripts/release.py > +++ b/bin/release_scripts/release.py > @@ -340,18 +340,18 @@ def publish_adt(rel_id, rel_type, opts): > QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname) > print "QEMU_SRC: %s" %QEMU_SRC > sync_it(QEMU_SRC, QEMU_TARGET, "") > - > sync_it(IPK_DIR, ADT_IPK, "") > return > > if __name__ == '__main__': > - > + > os.system("clear") > print > - > + > VHOSTS = "/srv/www/vhosts" > AB_BASE = os.path.join(VHOSTS, > "autobuilder.yoctoproject.org/pub/releases") > -DL_BASE = os.path.join(VHOSTS, > "downloads.yoctoproject.org/releases/yocto") > +DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") > +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") > ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") > So is the intent is to create a new DL_BASE directory downloads.yoctoproject.org/releases/releases/yocto or should that be: DL_BASE = os.path.join(DL_DIR, "/yocto") -Bill -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
Re: [yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
pulled into master. thanks! -b On 24 May 2016 at 03:12, Graydon, Tracy wrote: > Remove the step for syncing the staged release to downloads directory. We > pretty much NEVER do the sync to downloads at the time we do staging. Sync > happens much later, as a separate process. We don't need this in the release > script. > > Signed-off-by: Graydon, Tracy > --- > bin/release_scripts/release.py | 12 +--- > 1 file changed, 1 insertion(+), 11 deletions(-) > > diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py > index 3d4ce39..2b4d21c 100755 > --- a/bin/release_scripts/release.py > +++ b/bin/release_scripts/release.py > @@ -488,17 +488,7 @@ if __name__ == '__main__': > print "Generating the master md5sum table." > gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) > > -# 8) sync to downloads > -if REL_TYPE == "milestone": > -DL_DIR = os.path.join(DL_BASE, "milestones", RELEASE) > -print "DL_DIR for milestones: %s" %DL_DIR > -else: > -DL_DIR = os.path.join(DL_BASE, RELEASE) > -print "DL_DIR for point/major: %s" %DL_DIR > -print "Publishing release to downloads." > -sync_it(RELEASE_DIR, DL_DIR, "") > - > -# 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT > +# 8) Publish the ADT repo. The default is NOT to publish the ADT. The ADT > # is deprecated as of 2.1_M1. However, we need to retain backward > # compatability for point releases, etc. We do this step after all the > other > # stuff because we want the symlinks to have been converted, extraneous > -- > 2.7.0 > -- Elizabeth Flanagan Yocto Project Build and Release -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
Remove the step for syncing the staged release to downloads directory. We pretty much NEVER do the sync to downloads at the time we do staging. Sync happens much later, as a separate process. We don't need this in the release script. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 12 +--- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 3d4ce39..2b4d21c 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -488,17 +488,7 @@ if __name__ == '__main__': print "Generating the master md5sum table." gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) -# 8) sync to downloads -if REL_TYPE == "milestone": -DL_DIR = os.path.join(DL_BASE, "milestones", RELEASE) -print "DL_DIR for milestones: %s" %DL_DIR -else: -DL_DIR = os.path.join(DL_BASE, RELEASE) -print "DL_DIR for point/major: %s" %DL_DIR -print "Publishing release to downloads." -sync_it(RELEASE_DIR, DL_DIR, "") - -# 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT +# 8) Publish the ADT repo. The default is NOT to publish the ADT. The ADT # is deprecated as of 2.1_M1. However, we need to retain backward # compatability for point releases, etc. We do this step after all the other # stuff because we want the symlinks to have been converted, extraneous -- 2.7.0 -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
Fixed the path for the publishing of the eclipse plugins. They were going to the wrong subdirectory under downloads. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 22 +++--- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 51cd607..3d4ce39 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -340,18 +340,18 @@ def publish_adt(rel_id, rel_type, opts): QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname) print "QEMU_SRC: %s" %QEMU_SRC sync_it(QEMU_SRC, QEMU_TARGET, "") - sync_it(IPK_DIR, ADT_IPK, "") return if __name__ == '__main__': - + os.system("clear") print - + VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") -DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases/yocto") +DL_DIR = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") +DL_BASE = os.path.join(DL_DIR, "/releases/yocto") ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases @@ -380,7 +380,7 @@ if __name__ == '__main__': help="Use when you need to publish the ADT repo to a custom location. i.e. python adtcopy -b yocto-2.0_M1.rc1 -a 1.8+snaphot") (options, args) = parser.parse_args() - + REL_TYPE = "" MILESTONE = "" if options.poky: @@ -427,12 +427,12 @@ if __name__ == '__main__': print "Build ID is a required argument." print "Please use -h or --help for options." sys.exit() - + if not (RELEASE and RC and REL_ID and REL_TYPE): print "Can't determine the release type. Check your args." print "You gave me: %s" %options.build sys.exit() - + print "RC_DIR: %s" %RC_DIR print "RELEASE: %s" %RELEASE print "RC: %s" %RC @@ -442,7 +442,7 @@ if __name__ == '__main__': print "MILESTONE: %s" %MILESTONE print -PLUGIN_DIR = os.path.join(DL_BASE, "eclipse-plugin", REL_ID) +PLUGIN_DIR = os.path.join(DL_DIR, "eclipse-plugin", REL_ID) RELEASE_DIR = os.path.join(AB_BASE, RELEASE) MACHINES = os.path.join(RELEASE_DIR, "machines") BSP_DIR = os.path.join(RELEASE_DIR, 'bsptarballs') @@ -455,7 +455,7 @@ if __name__ == '__main__': # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED) - + # 2) Convert the symlinks in build-appliance dir. print "Converting the build-appliance symlink." convert_symlinks(BUILD_APP_DIR) @@ -469,7 +469,7 @@ if __name__ == '__main__': nuke_cruft(dirname, CRUFT_LIST) print "Generating fresh md5sums." gen_md5sum(MACHINES) - + # For major and point releases if REL_TYPE == "major" or REL_TYPE == "point": # 4) Fix up the eclipse and poky tarballs @@ -487,7 +487,7 @@ if __name__ == '__main__': # 7) Generate the master md5sum file for the release (for all releases) print "Generating the master md5sum table." gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) - + # 8) sync to downloads if REL_TYPE == "milestone": DL_DIR = os.path.join(DL_BASE, "milestones", RELEASE) -- 2.7.0 -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
This patch removes the BSP creation for the p1022ds and the mpc8315e-rdb platforms. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 17 - 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 158c44a..be67f9d 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -244,18 +244,9 @@ def nuke_cruft(dirname, ext_list): qemu_dir = os.path.join(MACHINES, dirname, dir) nuke_cruft(qemu_dir, CRUFT_LIST) else: -foo = dirname.find("p1022") -if foo == -1: -# NOT P1022ds -for ext in ext_list: -print "Deleting %s files" %ext -os.system("rm -f %s/%s" %(dirname, ext)) -else: -# IS P1022ds -for ext in ext_list: -if ext != "*.tar.gz": -print "Deleting %s files" %ext -os.system("rm -f %s/%s" %(dirname, ext)) +for ext in ext_list: +print "Deleting %s files" %ext +os.system("rm -f %s/%s" %(dirname, ext)) print return @@ -368,7 +359,7 @@ if __name__ == '__main__': # List of the files in machines directories that we delete from all releases CRUFT_LIST = ['*.md5sum', '*.tar.gz', '*.iso'] # List of the platforms for which we want to generate BSP tarballs. Major and point releases. -BSP_LIST = ['beaglebone', 'edgerouter', 'genericx86', 'genericx86-64', 'mpc8315e-rdb', 'p1022ds'] +BSP_LIST = ['beaglebone', 'edgerouter', 'genericx86', 'genericx86-64'] # List of files we do not want to include in the BSP tarballs. BSP_JUNK = ['*.manifest', '*.tar.bz2', '*.tgz', '*.iso', '*.md5sum', '*.tar.gz', '*-dev-*', '*-sdk-*'] -- 2.7.0 -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py: fix milestone staging
From: Tracy Graydon Milestones were going into the top level of the download release directory instead of the milestones subdirectory. Added a check for milestone releases and now adjusts the path accordingly. Also fixed an indent issue which was causing the master md5sum table to be generated for milestones. Signed-off-by: Tracy Graydon --- bin/release_scripts/release.py | 19 --- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 4c71e31..158c44a 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -453,7 +453,6 @@ if __name__ == '__main__': PLUGIN_DIR = os.path.join(DL_BASE, "eclipse-plugin", REL_ID) RELEASE_DIR = os.path.join(AB_BASE, RELEASE) -DL_DIR = os.path.join(DL_BASE, RELEASE) MACHINES = os.path.join(RELEASE_DIR, "machines") BSP_DIR = os.path.join(RELEASE_DIR, 'bsptarballs') TARBALL_DIR = os.path.join(RELEASE_DIR, "tarballs") @@ -494,18 +493,24 @@ if __name__ == '__main__': print "Generating the BSP tarballs." make_bsps(BSP_LIST, BSP_DIR) -# 7) Generate the master md5sum file for the release (for all releases) -print "Generating the master md5sum table." -gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) +# 7) Generate the master md5sum file for the release (for all releases) +print "Generating the master md5sum table." +gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) # 8) sync to downloads +if REL_TYPE == "milestone": +DL_DIR = os.path.join(DL_BASE, "milestones", RELEASE) +print "DL_DIR for milestones: %s" %DL_DIR +else: +DL_DIR = os.path.join(DL_BASE, RELEASE) +print "DL_DIR for point/major: %s" %DL_DIR print "Publishing release to downloads." sync_it(RELEASE_DIR, DL_DIR, "") -# 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT -# is deprecated as of 2.1_M1. However, we need to retain backward +# 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT +# is deprecated as of 2.1_M1. However, we need to retain backward # compatability for point releases, etc. We do this step after all the other -# stuff because we want the symlinks to have been converted, extraneous +# stuff because we want the symlinks to have been converted, extraneous # files deleted, and md5sums generated. # if options.pub_adt: -- 2.4.3 -- ___ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto
[yocto] [yocto-autobuilder][PATCH] bin/release_scripts/release.py
This is part of the release automation work, related to Yocto bug #8942. This patch includes: - Adding a check to see if the rc candidate passed in actually exists first. - Adds the ADT repo publishing step of the release process. - Removes the print_vars function that was largely for script testing purposes and is essentially otherwise useless. - Removes some variable definitions that were used for testing. Signed-off-by: Graydon, Tracy --- bin/release_scripts/release.py | 134 ++--- 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/bin/release_scripts/release.py b/bin/release_scripts/release.py index 27ee409..d67a7ec 100755 --- a/bin/release_scripts/release.py +++ b/bin/release_scripts/release.py @@ -20,33 +20,6 @@ import shutil from shutil import rmtree, copyfile from subprocess import call -def print_vars(): -print "RELEASE: %s" %RELEASE -print "REL_TYPE: %s" %REL_TYPE -print "RC_DIR: %s" %RC_DIR -print "REL_ID: %s" %REL_ID -print "RC: %s" %RC -if MILESTONE != "": -print "Milestone: %s" %MILESTONE -if POKY_VER != "": -print "POKY_VER: %s" %POKY_VER -else: - print "POKY_VER: undefined!" -if BRANCH: -print "BRANCH: %s" %BRANCH -else: -print "BRANCH: undefined!" - -print "DL_BASE: %s" %DL_BASE -if RC_SOURCE != "": -print "RC_SOURCE: %s" %RC_SOURCE -print "RELEASE_DIR: %s" %RELEASE_DIR -print "ECLIPSE_DIR: %s" %ECLIPSE_DIR -print "PLUGIN_DIR: %s" %PLUGIN_DIR -print "DL_DIR: %s" %DL_DIR -print -return - def sanity_check(source, target): if not os.path.exists(source): print @@ -64,7 +37,6 @@ def sanity_check(source, target): sys.exit() return - def sync_it(source, target, exclude_list): print "Syncing %s to %s" %(source, target) sanity_check(source, target) @@ -346,21 +318,48 @@ def gen_rel_md5(dirname, md5_file): f.close() return +def publish_adt(rel_id, rel_type, opts): +if opts: +ADT_DIR = os.path.join(ADT_BASE, opts) +else: +if rel_type == "milestone": +chunks = split_thing(rel_id, "_") +id_thing = float(chunks[0]) +id_thing = id_thing - 0.1 +rel_id = str(id_thing) + "+" + "snapshot" +ADT_DIR = os.path.join(ADT_BASE, rel_id) +print "ADT_DIR: %s" %ADT_DIR +if os.path.exists(ADT_DIR): +print "ADT_DIR %s EXISTS! Refusing to clobber!" %ADT_DIR +sys.exit() +else: +ADT_ROOTFS = os.path.join(ADT_DIR, "rootfs") +ADT_IPK = os.path.join(ADT_DIR, "adt-ipk") +QEMU_DIR = os.path.join(MACHINES, "qemu") +IPK_DIR = os.path.join(RELEASE_DIR, "ipk") +os.mkdir(ADT_DIR) +os.mkdir(ADT_ROOTFS) +dirlist = get_list(QEMU_DIR) + +for dirname in dirlist: +QEMU_SRC = os.path.join(QEMU_DIR, dirname) +QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname) +print "QEMU_SRC: %s" %QEMU_SRC +sync_it(QEMU_SRC, QEMU_TARGET, "") + +sync_it(IPK_DIR, ADT_IPK, "") +return if __name__ == '__main__': os.system("clear") print -# This is for testing convenience -#HOME_BASE = "/home/tgraydon/work/release" -#AB_BASE = HOME_BASE -#DL_BASE = os.path.join(HOME_BASE, "downloads") - -# This is the legit set of vars used for production release VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") +ADT_DEV = os.path.join(VHOSTS, "adtrepo-dev") +ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases UNLOVED = ['rpm', 'deb', 'ptest', 'adt-installer-QA'] @@ -381,7 +380,10 @@ if __name__ == '__main__': parser.add_option("-p", "--poky-ver", type="string", dest="poky", help="Required for Major and Point releases. i.e. 14.0.0") - +parser.add_option("-a", "--adt-dir", + type="string", dest="adt", + help="Use when you need to publish the ADT repo to a custom location. i.e. python adtcopy -b yocto-2.0_M1.rc1 -a 1.8+snaphot") + (options, args) = parser.parse_args() REL_TYPE = "" @@ -395,19 +397,23 @@ if __name__ == '__main__': else: BRANCH = "" - if options.build: # Figure out the release name, type of release, and generate some vars, do some basic validation -chunks = split_thing(options.build, ".") +options.build = options.build.lower() +RC = split_thing(options.build, ".")[-1] +chunks = split_thing(options.build, ".") # i.e. split yocto-2.1_m1.rc1 chunks.pop() -RELEASE = rejoin_thing(chunks, ".") -rel_thing = split_thing(options.build, "-"