Packager.py should take advantage of the parallelism from the package manager. Create the repo directory if it doesn't exists.
Signed-off-by: Jean-Marc Eurin <[email protected]> --- autotest/client/common_lib/base_packages.py 2011-02-09 14:25:38.000000000 -0800 +++ autotest/client/common_lib/base_packages.py 2011-02-22 12:51:10.000000000 -0800 @@ -28,27 +28,36 @@ "Incorrect SSH path in global_config: %s" % repo) -def repo_run_command(repo, cmd, ignore_status=False): +def repo_run_command(repo, cmd, ignore_status=False, cd=True): """Run a command relative to the repos path""" repo = repo.strip() run_cmd = None + cd_str = '' if repo.startswith('ssh://'): username = None hostline, remote_path = parse_ssh_path(repo) + if cd: + cd_str = 'cd %s && ' % remote_path if '@' in hostline: username, host = hostline.split('@') - run_cmd = 'ssh %s@%s "cd %s && %s"' % (username, host, - remote_path, cmd) + run_cmd = 'ssh %s@%s "%s%s"' % (username, host, cd_str, cmd) else: - run_cmd = 'ssh %s "cd %s && %s"' % (host, remote_path, cmd) + run_cmd = 'ssh %s "%s%s"' % (host, cd_str, cmd) else: - run_cmd = "cd %s && %s" % (repo, cmd) + if cd: + cd_str = 'cd %s && ' % repo + run_cmd = "%s%s" % (cd_str, cmd) if run_cmd: return utils.run(run_cmd, ignore_status=ignore_status) +def create_directory(repo): + _, remote_path = parse_ssh_path(repo) + repo_run_command(repo, 'mkdir -p %s' % remote_path, cd=False) + + def check_diskspace(repo, min_free=None): # Note: 1 GB = 10**9 bytes (SI unit). if not min_free: @@ -272,6 +281,7 @@ if not repo.startswith('/') and not repo.startswith('ssh:'): return try: + create_directory(repo) check_diskspace(repo) check_write(repo) except (error.RepoWriteError, error.RepoUnknownError, --- autotest/utils/packager.py 2009-10-13 17:18:06.000000000 -0700 +++ autotest/utils/packager.py 2011-02-22 12:51:10.000000000 -0800 @@ -71,7 +71,7 @@ # Method to upload or remove package depending on the flag passed to it. -def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, repo_url, +def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, remove=False): exclude_string = ' .' names = [p.strip() for p in pkg_names.split(',')] @@ -101,12 +101,12 @@ raise error.RepoDiskFullError(msg) tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, temp_dir, exclude_string) - pkgmgr.upload_pkg(tarball_path, repo_url, update_checksum=True) + pkgmgr.upload_pkg(tarball_path, update_checksum=True) finally: # remove the temporary directory shutil.rmtree(temp_dir) else: - pkgmgr.remove_pkg(pkg_name, repo_url, remove_checksum=True) + pkgmgr.remove_pkg(pkg_name, remove_checksum=True) print "Done." @@ -137,7 +137,7 @@ return tarballs -def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False): +def process_all_packages(pkgmgr, client_dir, remove=False): """Process a full upload of packages as a directory upload.""" test_dir = os.path.join(client_dir, "tests") site_test_dir = os.path.join(client_dir, "site_tests") @@ -185,22 +185,16 @@ os.chdir(temp_dir) client_utils.system('md5sum * > packages.checksum') os.chdir(cwd) - for path in upload_paths: - print "Uploading to: " + path - pkgmgr.upload_pkg(temp_dir, path) + pkgmgr.upload_pkg(temp_dir) client_utils.run('rm -rf ' + temp_dir) else: - for repo_url in upload_paths: - process_packages(pkgmgr, 'test', tests, client_dir, repo_url, - remove=remove) - process_packages(pkgmgr, 'test', site_tests, client_dir, repo_url, - remove=remove) - process_packages(pkgmgr, 'client', 'autotest', client_dir, repo_url, - remove=remove) - process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url, - remove=remove) - process_packages(pkgmgr, 'profiler', profilers, prof_dir, repo_url, - remove=remove) + process_packages(pkgmgr, 'test', tests, client_dir,remove=remove) + process_packages(pkgmgr, 'test', site_tests, client_dir, remove=remove) + process_packages(pkgmgr, 'client', 'autotest', client_dir, + remove=remove) + process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove) + process_packages(pkgmgr, 'profiler', profilers, prof_dir, + remove=remove) # Get the list of sub directories present in a directory @@ -241,10 +235,6 @@ if len(upload_paths) == 0: return - pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, - upload_paths=upload_paths, - run_function_dargs={'timeout':600}) - client_dir = os.path.join(autotest_dir, "client") # Bail out if the client directory does not exist @@ -267,35 +257,39 @@ # we should not be getting here assert(False) + if options.repo: + upload_path_list = [options.repo] + else: + upload_path_list = upload_paths + + pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, + upload_paths=upload_path_list, + run_function_dargs={'timeout':600}) + if options.all: - if options.repo: - upload_path_list = [options.repo] - else: - upload_path_list = upload_paths - process_all_packages(pkgmgr, client_dir, upload_path_list, - remove=remove_flag) + process_all_packages(pkgmgr, client_dir, remove=remove_flag) if options.client: process_packages(pkgmgr, 'client', 'autotest', client_dir, - options.repo, remove=remove_flag) + remove=remove_flag) if options.dep: process_packages(pkgmgr, 'dep', options.dep, dep_dir, - options.repo, remove=remove_flag) + remove=remove_flag) if options.test: process_packages(pkgmgr, 'test', options.test, client_dir, - options.repo, remove=remove_flag) + remove=remove_flag) if options.prof: process_packages(pkgmgr, 'profiler', options.prof, prof_dir, - options.repo, remove=remove_flag) + remove=remove_flag) if options.file: if remove_flag: - pkgmgr.remove_pkg(options.file, options.repo, remove_checksum=True) + pkgmgr.remove_pkg(options.file, remove_checksum=True) else: - pkgmgr.upload_pkg(options.file, options.repo, update_checksum=True) + pkgmgr.upload_pkg(options.file, update_checksum=True) if __name__ == "__main__": _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
