ajack 2004/03/12 16:17:40 Modified: python/gump/model project.py stats.py repository.py python/gump/document forrest.py documenter.py python/gump/utils tools.py sync.py python/gump/output statsdb.py python/gump engine.py Log: 1) Made sure not duplicate FOG calculations.
2) Do forrest to a staging area, then sync to public [minimal missing pages, but cleanest] 3) Clean up forrest sites/work once published. Revision Changes Path 1.63 +4 -8 gump/python/gump/model/project.py Index: project.py =================================================================== RCS file: /home/cvs/gump/python/gump/model/project.py,v retrieving revision 1.62 retrieving revision 1.63 diff -u -r1.62 -r1.63 --- project.py 9 Mar 2004 19:57:06 -0000 1.62 +++ project.py 13 Mar 2004 00:17:39 -0000 1.63 @@ -304,6 +304,7 @@ def getReports(self): return self.reports + def getFOGFactor(self): return self.getStats().getFOGFactor() @@ -1222,11 +1223,6 @@ """Statistics Holder""" def __init__(self,projectName): Statistics.__init__(self,projectName) - - def getFOGFactor(self): - good=self.successes - bad=(self.failures+self.prereqs) or 1 - return good/bad def getKeyBase(self): return 'project:'+ self.name 1.13 +6 -4 gump/python/gump/model/stats.py Index: stats.py =================================================================== RCS file: /home/cvs/gump/python/gump/model/stats.py,v retrieving revision 1.12 retrieving revision 1.13 diff -u -r1.12 -r1.13 --- stats.py 12 Mar 2004 16:10:39 -0000 1.12 +++ stats.py 13 Mar 2004 00:17:39 -0000 1.13 @@ -94,7 +94,9 @@ self.lastUpdated=0 def getFOGFactor(self): - return (float(self.successes) / (float(self.failures) + float(self.prereqs))) + good=self.successes + bad=(self.failures+self.prereqs) or 1 + return float(good)/float(bad) def getLastUpdated(self): return (self.lastUpdated) 1.9 +3 -6 gump/python/gump/model/repository.py Index: repository.py =================================================================== RCS file: /home/cvs/gump/python/gump/model/repository.py,v retrieving revision 1.8 retrieving revision 1.9 diff -u -r1.8 -r1.9 --- repository.py 20 Jan 2004 21:55:23 -0000 1.8 +++ repository.py 13 Mar 2004 00:17:39 -0000 1.9 @@ -171,9 +171,6 @@ """Statistics Holder""" def __init__(self,repositoryName): Statistics.__init__(self,repositoryName) - - def getFOGFactor(self): - return (self.successes / (self.failures - self.prereqs)) def getKeyBase(self): return 'repository:'+ self.name 1.105 +66 -63 gump/python/gump/document/forrest.py Index: forrest.py =================================================================== RCS file: /home/cvs/gump/python/gump/document/forrest.py,v retrieving revision 1.104 retrieving revision 1.105 diff -u -r1.104 -r1.105 --- forrest.py 12 Mar 2004 18:55:24 -0000 1.104 +++ forrest.py 13 Mar 2004 00:17:40 -0000 1.105 @@ -67,7 +67,6 @@ import os import sys import logging -import shutil from string import lower,replace from xml.sax.saxutils import escape @@ -79,7 +78,7 @@ from gump.document.resolver import * from gump.utils import * from gump.utils.xmlutils import xmlize -from gump.utils.tools import syncDirectories,copyDirectories +from gump.utils.tools import syncDirectories,copyDirectories,wipeDirectoryTree from gump.model import * from gump.model.stats import * from gump.model.project import AnnotatedPath, ProjectStatistics @@ -104,24 +103,31 @@ def getResolverForRun(self,run): return self.resolver - def documentRun(self, run): + def prepareRun(self, run): - log.debug('--- Documenting Results') + log.debug('--- Prepare for Documenting Results') workspace=run.getWorkspace() gumpSet=run.getGumpSet() # Seed with default/site skins/etc. self.prepareForrest(workspace) - - # Document... + + def documentRun(self, run): + + log.debug('--- Documenting Results') + + workspace=run.getWorkspace() + gumpSet=run.getGumpSet() + + # Document... self.documentWorkspace(run,workspace,gumpSet) if gumpSet.isFull(): self.documentStatistics(run,workspace,gumpSet) self.documentXRef(run,workspace,gumpSet) # Launch Forrest... - self.executeForrest(workspace) + return self.executeForrest(workspace) ##################################################################### # @@ -133,11 +139,18 @@ fdir=os.path.abspath(os.path.join(workspace.getBaseDirectory(),'forrest')) return fdir + def getForrestOutputDirectory(self,workspace): + """ Staging Area for built output """ + fdir=os.path.abspath(os.path.join(workspace.getBaseDirectory(),'forrest-site')) + return fdir + def getForrestTemplateDirectory(self): + """ Template (forrest skin/config) """ fdir=os.path.abspath(os.path.join(dir.template,'forrest')) return fdir def getForrestSiteTemplateDirectory(self): + """ Site Template (forrest skin/config tweaks) """ fdir=os.path.abspath(os.path.join(dir.template,'site-forrest')) return fdir @@ -148,75 +161,47 @@ """ # - # First deleted the tree (if exists), then ensure created + # First deleted the work tree (if exists), then ensure created # - forrestDir=self.getForrestDirectory(workspace) - if os.path.exists(forrestDir): - shutil.rmtree(forrestDir) - os.makedirs(forrestDir) - + forrestWorkDir=self.getForrestDirectory(workspace) + wipeDirectoryTree(forrestWorkDir) + # Copy in the defaults - forrestTemplate=self.getForrestTemplateDirectory() - # :TODO: We need to sync, but we write to contents... - copyDirectories( forrestTemplate, \ - forrestDir, \ + forrestTemplate=self.getForrestTemplateDirectory() + syncDirectories( forrestTemplate, \ + forrestWorkDir, \ workspace) # Copy over the local site defaults (if any) forrestSiteTemplate=self.getForrestSiteTemplateDirectory() if os.path.exists(forrestSiteTemplate): copyDirectories(forrestSiteTemplate, \ - forrestDir, \ - workspace) + forrestWorkDir, \ + workspace) + + # + # Delete the output tree + # + outputDirectory=self.getForrestOutputDirectory(workspace) + wipeDirectoryTree(outputDirectory) + def executeForrest(self,workspace): # The project tree xdocs=self.resolver.getDirectory(workspace) - forrestDir=self.getForrestDirectory(workspace) - # - # :TODO: - # 1) We need to do this sooner ('cos things write into content before this module) - # 2) We need a staging area that we write to, then we Sync to the real output - # so we have less of a window of missing public files. - # - # - # # - # # First deleted the tree (if exists), then ensure created - # # + # The three dirs, work, output (staging), public + forrestWorkDir=self.getForrestDirectory(workspace) + outputDirectory=self.getForrestOutputDirectory(workspace) logDirectory=workspace.getLogDirectory() - # if os.path.exists(logDirectory): - # try: - # shutil.rmtree(logDirectory) - # except: pass - # if not os.path.exists(logDirectory): - # os.makedirs(logDirectory) - # Then generate... - forrest=Cmd('forrest','forrest',forrestDir) + # Generate... + forrest=Cmd('forrest','forrest',forrestWorkDir) forrest.addPrefixedParameter('-D','java.awt.headless','true','=') - #forrest.addPrefixedParameter('-D','project.content-dir', \ - # content, '=') - #forrest.addPrefixedParameter('-D','project.xdocs-dir', \ - # xdocs, '=') - forrest.addPrefixedParameter('-D','project.site-dir', \ - logDirectory, '=') - - # - # Do we just tweak forrest.properties? - # - #forrest.addPrefixedParameter('-D','project.sitemap-dir', \ - # docroot, '=') - #forrest.addPrefixedParameter('-D','project.stylesheets-dir', \ - # docroot, '=') - #forrest.addPrefixedParameter('-D','project.images-dir', \ - # docroot, '=') - - #forrest.addPrefixedParameter('-D','project.skinconf', \ - # getWorkspaceSiteDirectory(workspace), '=' ) - + outputDirectory, '=') + # Temporary # Too verbose ... forrest.addParameter('-debug') #forrest.addParameter('-verbose') @@ -233,6 +218,18 @@ # Update Context work=CommandWorkItem(WORK_TYPE_DOCUMENT,forrest,forrestResult) workspace.performedWork(work) + + # If ok move from staging to publish + if forrestResult.state==CMD_STATE_SUCCESS: + # + # Sync over public pages... + # + syncDirectories(outputDirectory,logDirectory) + # + # Clean up + wipeDirectoryTree(outputDirectory) + wipeDirectoryTree(forrestWorkDir) + ##################################################################### # @@ -2236,12 +2233,18 @@ fileName='module_updated' file=self.resolver.getFile(stats,fileName) document=XDocDocument('Modules By Last Updated', file) - updTable=document.createTable(['Module','Last Updated']) + updTable=document.createTable(['Module','Last Updated Date','Last Updated']) + modules=0 for module in stats.modulesByLastUpdated: - if not gumpSet.inModuleSequence(module): continue + if not gumpSet.inModuleSequence(module): continue + if module.isPackaged(): continue updRow=updTable.createRow() self.insertLink( module, stats, updRow.createData()) updRow.createData(secsToDate(module.getLastUpdated())) + updRow.createData( \ + getGeneralSinceDescription(module.getLastUpdated())) + modules+=1 + if not modules: updTable.createLine('None') document.serialize() 1.10 +13 -4 gump/python/gump/document/documenter.py Index: documenter.py =================================================================== RCS file: /home/cvs/gump/python/gump/document/documenter.py,v retrieving revision 1.9 retrieving revision 1.10 diff -u -r1.9 -r1.10 --- documenter.py 17 Feb 2004 21:54:21 -0000 1.9 +++ documenter.py 13 Mar 2004 00:17:40 -0000 1.10 @@ -74,7 +74,16 @@ def __init__(self): pass # - # Populate a method called 'document(run)' + # Call a method called 'prepareRun(run)', if needed + # + def prepare(self,run): + if not hasattr(self,'prepareRun'): return + if not callable(self.prepareRun): return + log.info('Prepare to document run using [' + `self` + ']') + self.prepareRun(run) + + # + # Call a method called 'documentRun(run)' # def document(self,run): if not hasattr(self,'documentRun'): 1.19 +13 -4 gump/python/gump/utils/tools.py Index: tools.py =================================================================== RCS file: /home/cvs/gump/python/gump/utils/tools.py,v retrieving revision 1.18 retrieving revision 1.19 diff -u -r1.18 -r1.19 --- tools.py 12 Mar 2004 02:57:50 -0000 1.18 +++ tools.py 13 Mar 2004 00:17:40 -0000 1.19 @@ -64,6 +64,7 @@ import logging import types, StringIO +import shutil from gump import log from gump.utils.note import * @@ -206,7 +207,15 @@ finally: if annotatable: transferAnnotations(sync, annotatable) - + +def wipeDirectoryTree(dir): + if os.path.exists(dir): + try: + shutil.rmtree(dir) + except: pass + if not os.path.exists(dir): + os.makedirs(dir) + def tailFile(file,lines,wrapLen=0,eol=None,marker=None): """ Return the last N lines of a file as a list """ taillines=[] 1.6 +5 -5 gump/python/gump/utils/sync.py Index: sync.py =================================================================== RCS file: /home/cvs/gump/python/gump/utils/sync.py,v retrieving revision 1.5 retrieving revision 1.6 diff -u -r1.5 -r1.6 --- sync.py 11 Mar 2004 20:52:05 -0000 1.5 +++ sync.py 13 Mar 2004 00:17:40 -0000 1.6 @@ -138,8 +138,8 @@ elif os.path.isdir(srcname): if not name in ['CVS','.svn']: self.copytree(srcname, dstname, symlinks) - else: - log.debug('Skip SVN or CVS directory ' + str(srcname)) + #else: + # log.debug('Skip SVN or CVS directory ' + str(srcname)) else: self.maybecopy(srcname, dstname) except (IOError, os.error), why: 1.15 +14 -13 gump/python/gump/output/statsdb.py Index: statsdb.py =================================================================== RCS file: /home/cvs/gump/python/gump/output/statsdb.py,v retrieving revision 1.14 retrieving revision 1.15 diff -u -r1.14 -r1.15 --- statsdb.py 23 Feb 2004 20:55:08 -0000 1.14 +++ statsdb.py 13 Mar 2004 00:17:40 -0000 1.15 @@ -417,22 +417,22 @@ def compareModulesByFOGFactor(module1,module2): fog1=module1.getFOGFactor() fog2=module2.getFOGFactor() - c= int(round(fog2 - fog1,2)) + # Allow comparison to 2 decimal places, by *100 + c= int(round((fog2 - fog1)*100,0)) if not c: c=cmp(module1,module2) return c def compareModulesByLastUpdated(module1,module2): - fog1=module1.getLastUpdated() - fog2=module2.getLastUpdated() - c= int(round(fog2 - fog1,0)) + lu1=module1.getLastUpdated() + lu2=module2.getLastUpdated() + c= int(round((lu2 - lu1)*100,0)) if not c: c=cmp(module1,module2) return c # # Project Comparisons # - - + def compareProjectsByElapsed(project1,project2): elapsed1=project1.getElapsedSecs() elapsed2=project2.getElapsedSecs() @@ -459,14 +459,15 @@ def compareProjectsByFOGFactor(project1,project2): fog1=project1.getFOGFactor() fog2=project2.getFOGFactor() - c= int(round(fog2 - fog1,0)) + # Allow comparison to 2 decimal places, by *100 + c= int(round((fog2 - fog1)*100,0)) if not c: c=cmp(project1,project2) return c def compareProjectsByLastUpdated(project1,project2): - fog1=project1.getLastUpdated() - fog2=project2.getLastUpdated() - c= int(round(fog2 - fog1,0)) + lu1=project1.getLastUpdated() + lu2=project2.getLastUpdated() + c= int(round((lu2 - lu1)*100,0)) if not c: c=cmp(project1,project2) return c 1.85 +14 -7 gump/python/gump/engine.py Index: engine.py =================================================================== RCS file: /home/cvs/gump/python/gump/engine.py,v retrieving revision 1.84 retrieving revision 1.85 diff -u -r1.84 -r1.85 --- engine.py 12 Mar 2004 16:51:56 -0000 1.84 +++ engine.py 13 Mar 2004 00:17:40 -0000 1.85 @@ -102,7 +102,6 @@ def continuous(self): - # :TODO: WORK IN PROGRESS NOT COMPLETE!!! while 0: @@ -147,7 +146,12 @@ # logResourceUtilization('Before generate results') gatherResults(run) - + + # Prepare for documentation + documenter=run.getOptions().getDocumenter() + if documenter : + documenter.prepare(run) + # Update Statistics/Results on full runs if run.getGumpSet().isFull(): @@ -170,7 +174,6 @@ # Build HTML Result (via Forrest or ...) # logResourceUtilization('Before document') - documenter=run.getOptions().getDocumenter() if documenter : documenter.document(run) @@ -225,11 +228,15 @@ # Check the metadata # self.checkWorkspace(run) - + + # Prepare for documentation + documenter=run.getOptions().getDocumenter() + if documenter : + documenter.prepare(run) + # # Build HTML Result (via Forrest or ...) # - documenter=run.getOptions().getDocumenter() if documenter : documenter.document(run) @@ -474,10 +481,10 @@ if (not STATE_SUCCESS == stats.currentState) and \ not project.isVerboseOrDebug(): if stats.sequenceInState > INSIGNIFICANT_DURATION: - project.addInfo('Enable "debug" output, due to error.') + project.addInfo('Enable "debug" output, due to a sequence of %s previous errors.' % stats.sequenceInState) project.setDebug(1) else: - project.addInfo('Enable "verbose" output, due to error.') + project.addInfo('Enable "verbose" output, due to %s previous error(s).' % stats.sequenceInState) project.setVerbose(1) # --------------------------------------------------------------------- To unsubscribe, e-mail: [EMAIL PROTECTED] For additional commands, e-mail: [EMAIL PROTECTED]