On Tue, 2011-02-22 at 13:01 -0800, Jean-Marc Eurin wrote: > Packager.py should take advantage of the parallelism from the package manager. > Create the repo directory if it doesn't exists.
Applied, thanks! http://autotest.kernel.org/changeset/5273 > Signed-off-by: Jean-Marc Eurin <[email protected]> > > --- autotest/client/common_lib/base_packages.py 2011-02-09 > 14:25:38.000000000 -0800 > +++ autotest/client/common_lib/base_packages.py 2011-02-22 > 12:51:10.000000000 -0800 > @@ -28,27 +28,36 @@ > "Incorrect SSH path in global_config: %s" % repo) > > > -def repo_run_command(repo, cmd, ignore_status=False): > +def repo_run_command(repo, cmd, ignore_status=False, cd=True): > """Run a command relative to the repos path""" > repo = repo.strip() > run_cmd = None > + cd_str = '' > if repo.startswith('ssh://'): > username = None > hostline, remote_path = parse_ssh_path(repo) > + if cd: > + cd_str = 'cd %s && ' % remote_path > if '@' in hostline: > username, host = hostline.split('@') > - run_cmd = 'ssh %s@%s "cd %s && %s"' % (username, host, > - remote_path, cmd) > + run_cmd = 'ssh %s@%s "%s%s"' % (username, host, cd_str, cmd) > else: > - run_cmd = 'ssh %s "cd %s && %s"' % (host, remote_path, cmd) > + run_cmd = 'ssh %s "%s%s"' % (host, cd_str, cmd) > > else: > - run_cmd = "cd %s && %s" % (repo, cmd) > + if cd: > + cd_str = 'cd %s && ' % repo > + run_cmd = "%s%s" % (cd_str, cmd) > > if run_cmd: > return utils.run(run_cmd, ignore_status=ignore_status) > > > +def create_directory(repo): > + _, remote_path = parse_ssh_path(repo) > + repo_run_command(repo, 'mkdir -p %s' % remote_path, cd=False) > + > + > def check_diskspace(repo, min_free=None): > # Note: 1 GB = 10**9 bytes (SI unit). > if not min_free: > @@ -272,6 +281,7 @@ > if not repo.startswith('/') and not repo.startswith('ssh:'): > return > try: > + create_directory(repo) > check_diskspace(repo) > check_write(repo) > except (error.RepoWriteError, error.RepoUnknownError, > --- autotest/utils/packager.py 2009-10-13 17:18:06.000000000 -0700 > +++ autotest/utils/packager.py 2011-02-22 12:51:10.000000000 -0800 > @@ -71,7 +71,7 @@ > > > # Method to upload or remove package depending on the flag passed to it. > -def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, repo_url, > +def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, > remove=False): > exclude_string = ' .' > names = [p.strip() for p in pkg_names.split(',')] > @@ -101,12 +101,12 @@ > raise error.RepoDiskFullError(msg) > tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, > temp_dir, exclude_string) > - pkgmgr.upload_pkg(tarball_path, repo_url, > update_checksum=True) > + pkgmgr.upload_pkg(tarball_path, update_checksum=True) > finally: > # remove the temporary directory > shutil.rmtree(temp_dir) > else: > - pkgmgr.remove_pkg(pkg_name, repo_url, remove_checksum=True) > + pkgmgr.remove_pkg(pkg_name, remove_checksum=True) > print "Done." > > > @@ -137,7 +137,7 @@ > return tarballs > > > -def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False): > +def process_all_packages(pkgmgr, client_dir, remove=False): > """Process a full upload of packages as a directory upload.""" > test_dir = os.path.join(client_dir, "tests") > site_test_dir = os.path.join(client_dir, "site_tests") > @@ -185,22 +185,16 @@ > os.chdir(temp_dir) > client_utils.system('md5sum * > packages.checksum') > os.chdir(cwd) > - for path in upload_paths: > - print "Uploading to: " + path > - pkgmgr.upload_pkg(temp_dir, path) > + pkgmgr.upload_pkg(temp_dir) > client_utils.run('rm -rf ' + temp_dir) > else: > - for repo_url in upload_paths: > - process_packages(pkgmgr, 'test', tests, client_dir, repo_url, > - remove=remove) > - process_packages(pkgmgr, 'test', site_tests, client_dir, > repo_url, > - remove=remove) > - process_packages(pkgmgr, 'client', 'autotest', client_dir, > repo_url, > - remove=remove) > - process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url, > - remove=remove) > - process_packages(pkgmgr, 'profiler', profilers, prof_dir, > repo_url, > - remove=remove) > + process_packages(pkgmgr, 'test', tests, client_dir,remove=remove) > + process_packages(pkgmgr, 'test', site_tests, client_dir, > remove=remove) > + process_packages(pkgmgr, 'client', 'autotest', client_dir, > + remove=remove) > + process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove) > + process_packages(pkgmgr, 'profiler', profilers, prof_dir, > + remove=remove) > > > # Get the list of sub directories present in a directory > @@ -241,10 +235,6 @@ > if len(upload_paths) == 0: > return > > - pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, > - upload_paths=upload_paths, > - run_function_dargs={'timeout':600}) > - > client_dir = os.path.join(autotest_dir, "client") > > # Bail out if the client directory does not exist > @@ -267,35 +257,39 @@ > # we should not be getting here > assert(False) > > + if options.repo: > + upload_path_list = [options.repo] > + else: > + upload_path_list = upload_paths > + > + pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, > + upload_paths=upload_path_list, > + run_function_dargs={'timeout':600}) > + > if options.all: > - if options.repo: > - upload_path_list = [options.repo] > - else: > - upload_path_list = upload_paths > - process_all_packages(pkgmgr, client_dir, upload_path_list, > - remove=remove_flag) > + process_all_packages(pkgmgr, client_dir, remove=remove_flag) > > if options.client: > process_packages(pkgmgr, 'client', 'autotest', client_dir, > - options.repo, remove=remove_flag) > + remove=remove_flag) > > if options.dep: > process_packages(pkgmgr, 'dep', options.dep, dep_dir, > - options.repo, remove=remove_flag) > + remove=remove_flag) > > if options.test: > process_packages(pkgmgr, 'test', options.test, client_dir, > - options.repo, remove=remove_flag) > + remove=remove_flag) > > if options.prof: > process_packages(pkgmgr, 'profiler', options.prof, prof_dir, > - options.repo, remove=remove_flag) > + remove=remove_flag) > > if options.file: > if remove_flag: > - pkgmgr.remove_pkg(options.file, options.repo, > remove_checksum=True) > + pkgmgr.remove_pkg(options.file, remove_checksum=True) > else: > - pkgmgr.upload_pkg(options.file, options.repo, > update_checksum=True) > + pkgmgr.upload_pkg(options.file, update_checksum=True) > > > if __name__ == "__main__": > _______________________________________________ > Autotest mailing list > [email protected] > http://test.kernel.org/cgi-bin/mailman/listinfo/autotest _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
