build-support: add a nicer logger to python scripts Change-Id: Ic648fadfbb11eb3e0a8f137ae3f3143e52149101 Reviewed-on: http://gerrit.cloudera.org:8080/10131 Reviewed-by: Alexey Serbin <aser...@cloudera.com> Tested-by: Alexey Serbin <aser...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/953a7070 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/953a7070 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/953a7070 Branch: refs/heads/master Commit: 953a7070f6a743410f9b8dc74e85b66023d8ab56 Parents: b463969 Author: Todd Lipcon <t...@apache.org> Authored: Thu Apr 19 16:44:22 2018 -0700 Committer: Alexey Serbin <aser...@cloudera.com> Committed: Fri Apr 20 05:52:05 2018 +0000 ---------------------------------------------------------------------- build-support/build_source_release.py | 5 +++-- build-support/check_compatibility.py | 4 ++-- build-support/clang_tidy_gerrit.py | 4 +++- build-support/dist_test.py | 24 +++++++++++++----------- build-support/gen_version_info.py | 5 +++-- build-support/iwyu.py | 5 ++--- build-support/kudu_util.py | 14 ++++++++++++++ build-support/push_to_asf.py | 4 ++-- 8 files changed, 42 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/build_source_release.py ---------------------------------------------------------------------- diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py index 35ffa40..5ef39b2 100755 --- a/build-support/build_source_release.py +++ b/build-support/build_source_release.py @@ -31,7 +31,8 @@ try: except ImportError: import urllib -from kudu_util import check_output, confirm_prompt, Colors, get_my_email, get_upstream_commit, ROOT +from kudu_util import check_output, confirm_prompt, Colors, get_my_email, get_upstream_commit, \ + init_logging, ROOT def check_repo_not_dirty(): @@ -190,5 +191,5 @@ def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + init_logging() main() http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/check_compatibility.py ---------------------------------------------------------------------- diff --git a/build-support/check_compatibility.py b/build-support/check_compatibility.py index a8b8e3e..61d5661 100755 --- a/build-support/check_compatibility.py +++ b/build-support/check_compatibility.py @@ -32,7 +32,7 @@ import subprocess import sys import tempfile -from kudu_util import check_output +from kudu_util import check_output, init_logging JAVA_ACC_GIT_URL = "https://github.com/lvc/japi-compliance-checker.git"; @@ -180,7 +180,6 @@ def run_java_acc(src_name, src, dst_name, dst): def main(argv): - logging.basicConfig(level=logging.INFO) parser = optparse.OptionParser( usage="usage: %prog SRC..[DST]") parser.add_option("-f", "--force-download", dest="force_download_deps", @@ -223,4 +222,5 @@ def main(argv): if __name__ == "__main__": + init_logging() main(sys.argv) http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/clang_tidy_gerrit.py ---------------------------------------------------------------------- diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py index 3e77abe..d29a792 100755 --- a/build-support/clang_tidy_gerrit.py +++ b/build-support/clang_tidy_gerrit.py @@ -31,6 +31,8 @@ import sys import unittest import tempfile +from kudu_util import init_logging + ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) CLANG_TIDY_DIFF = os.path.join( @@ -176,7 +178,7 @@ class TestClangTidyGerrit(unittest.TestCase): if __name__ == "__main__": # Basic setup and argument parsing. - logging.basicConfig(level=logging.INFO) + init_logging() parser = argparse.ArgumentParser( description="Run clang-tidy on a patch, optionally posting warnings as comments to gerrit") parser.add_argument("-n", "--no-gerrit", action="store_true", http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/dist_test.py ---------------------------------------------------------------------- diff --git a/build-support/dist_test.py b/build-support/dist_test.py index 132b465..87ac820 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -39,6 +39,8 @@ import shutil import subprocess import time +from kudu_util import init_logging + TEST_TIMEOUT_SECS = int(os.environ.get('TEST_TIMEOUT_SECS', '900')) ARTIFACT_ARCHIVE_GLOBS = ["build/*/test-logs/**/*"] ISOLATE_SERVER = os.environ.get('ISOLATE_SERVER', @@ -165,7 +167,7 @@ def get_test_executions(options): cwd=rel_to_abs("build/latest")) out, err = p.communicate() if p.returncode != 0: - print >>sys.stderr, "Unable to list tests with ctest" + logging.error("Unable to list tests with ctest") sys.exit(1) lines = deque(out.splitlines()) execs = [] @@ -259,7 +261,7 @@ def ldd_deps(exe): LDD_CACHE[exe] = (out, err, p.returncode) out, err, rc = LDD_CACHE[exe] if rc != 0: - print >>sys.stderr, "failed to run ldd on ", exe + logging.warning("failed to run ldd on %s", exe) return [] ret = [] for l in out.splitlines(): @@ -289,7 +291,7 @@ def create_archive_input(staging, execution, """ argv = execution.argv if not argv[0].endswith('run-test.sh') or len(argv) < 2: - print >>sys.stderr, "Unable to handle test: ", argv + logging.warning("Unable to handle test: %s", argv) return abs_test_exe = os.path.realpath(argv[1]) rel_test_exe = abs_to_rel(abs_test_exe, staging) @@ -382,8 +384,8 @@ def create_task_json(staging, }] * replicate_tasks if len(tasks) > MAX_TASKS_PER_JOB: - print >>sys.stderr, "Job contains %d tasks which is more than the maximum %d" % ( - len(tasks), MAX_TASKS_PER_JOB) + logging.error("Job contains %d tasks which is more than the maximum %d", + len(tasks), MAX_TASKS_PER_JOB) sys.exit(1) outmap = {"tasks": tasks} @@ -406,7 +408,7 @@ def run_isolate(staging): '-dump-json=' + staging.archive_dump_path(), '--'] + staging.gen_json_paths()) except: - print >>sys.stderr, "Failed to run", isolate_path + logging.error("Failed to run %s", isolate_path) raise def submit_tasks(staging, options): @@ -418,9 +420,9 @@ def submit_tasks(staging, options): by 'create_task_json()'. """ if not os.path.exists(DIST_TEST_HOME): - print >>sys.stderr, "Cannot find dist_test tools at path %s " \ - "Set the DIST_TEST_HOME environment variable to the path to the dist_test directory. " \ - % DIST_TEST_HOME, + logging.error("Cannot find dist_test tools at path %s " \ + "Set the DIST_TEST_HOME environment variable to the path to the dist_test directory. ", + DIST_TEST_HOME) raise OSError("Cannot find path to dist_test tools") client_py_path = os.path.join(DIST_TEST_HOME, "bin", "client") try: @@ -430,7 +432,7 @@ def submit_tasks(staging, options): cmd.append(staging.tasks_json_path()) subprocess.check_call(cmd) except: - print >>sys.stderr, "Failed to run", client_py_path + logging.error("Failed to run %s", client_py_path) raise def get_flakies(): @@ -514,7 +516,6 @@ def add_loop_test_subparser(subparsers): def main(argv): - logging.basicConfig(level=logging.INFO) p = argparse.ArgumentParser() p.add_argument("--collect-tmpdir", dest="collect_tmpdir", action="store_true", help="Collect the test tmpdir of failed tasks as test artifacts", default=False) @@ -528,4 +529,5 @@ def main(argv): if __name__ == "__main__": + init_logging() main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/gen_version_info.py ---------------------------------------------------------------------- diff --git a/build-support/gen_version_info.py b/build-support/gen_version_info.py index d8aa88a..fbb0985 100755 --- a/build-support/gen_version_info.py +++ b/build-support/gen_version_info.py @@ -30,7 +30,7 @@ import sys import time from time import strftime, localtime -from kudu_util import check_output +from kudu_util import check_output, init_logging def output_up_to_date(path, id_hash): """ @@ -46,7 +46,6 @@ def output_up_to_date(path, id_hash): return m.group(1) == id_hash def main(): - logging.basicConfig(level=logging.INFO) parser = optparse.OptionParser( usage="usage: %prog --version=<version> <output path>") parser.add_option("-v", "--version", help="Set version number", type="string", @@ -127,5 +126,7 @@ def main(): """ % locals()) return 0 + if __name__ == "__main__": + init_logging() sys.exit(main()) http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/iwyu.py ---------------------------------------------------------------------- diff --git a/build-support/iwyu.py b/build-support/iwyu.py index 4fdcbdd..2638cdd 100755 --- a/build-support/iwyu.py +++ b/build-support/iwyu.py @@ -28,7 +28,7 @@ import re import subprocess import sys -from kudu_util import get_upstream_commit, check_output, ROOT, Colors +from kudu_util import get_upstream_commit, check_output, ROOT, Colors, init_logging import iwyu.fix_includes from iwyu.fix_includes import ParseAndMergeIWYUOutput @@ -268,7 +268,6 @@ def main(argv): else: return _do_iwyu(flags, paths) - if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + init_logging() sys.exit(main(sys.argv)) http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/kudu_util.py ---------------------------------------------------------------------- diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py index eac5115..15d1059 100644 --- a/build-support/kudu_util.py +++ b/build-support/kudu_util.py @@ -22,6 +22,7 @@ from __future__ import print_function +import logging import os import subprocess import sys @@ -42,6 +43,19 @@ def get_upstream_commit(): """ Return the last commit hash that appears to have been committed by gerrit. """ return check_output(_GET_UPSTREAM_COMMIT_SCRIPT).strip().decode('utf-8') +def init_logging(): + logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s: %(message)s') + logging.getLogger().addFilter(ColorFilter()) + +class ColorFilter(logging.Filter): + """ logging.Filter implementation which colorizes the output to console. """ + def filter(self, record): + if record.levelno >= logging.ERROR: + record.msg = Colors.RED + record.msg + Colors.RESET + elif record.levelno >= logging.WARNING: + record.msg = Colors.YELLOW + record.msg + Colors.RESET + return True class Colors(object): """ ANSI color codes. """ http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/push_to_asf.py ---------------------------------------------------------------------- diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py index 5b4d15f..1752dbe 100755 --- a/build-support/push_to_asf.py +++ b/build-support/push_to_asf.py @@ -40,7 +40,7 @@ import re import subprocess import sys -from kudu_util import check_output, confirm_prompt, Colors, get_my_email +from kudu_util import check_output, confirm_prompt, Colors, get_my_email, init_logging APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/kudu.git"; GERRIT_URL = "ssh://<username>@gerrit.cloudera.org:29418/kudu" @@ -235,5 +235,5 @@ def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + init_logging() main()
