[
https://issues.apache.org/jira/browse/BEAM-4302?focusedWorklogId=114047&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-114047
]
ASF GitHub Bot logged work on BEAM-4302:
----------------------------------------
Author: ASF GitHub Bot
Created on: 21/Jun/18 04:09
Start Date: 21/Jun/18 04:09
Worklog Time Spent: 10m
Work Description: yifanzou commented on a change in pull request #5406:
[BEAM-4302] add beam dependency checks
URL: https://github.com/apache/beam/pull/5406#discussion_r197002640
##########
File path:
.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
##########
@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import os.path
+import re
+import traceback
+from datetime import datetime
+from bigquery_client_utils import BigQueryClientUtils
+
+
+_MAX_STALE_DAYS = 360
+_MAX_MINOR_VERSION_DIFF = 3
+_PYPI_URL = "https://pypi.org/project/";
+_MAVEN_CENTRAL_URL = "http://search.maven.org/#search|gav|1|"
+
+class InvalidFormatError(Exception):
+ def __init__(self, message):
+ super(InvalidFormatError, self).__init__(message)
+
+
+def extract_results(file_path):
+ """
+ Extract the Java/Python dependency reports and return a collection of
out-of-date dependencies.
+ Args:
+ file_path: the path of the raw reports
+ Return:
+ outdated_deps: a collection of dependencies who has updates
+ """
+ outdated_deps = []
+ try:
+ with open(file_path) as raw_report:
+ see_oudated_deps = False
+ for line in raw_report:
+ if see_oudated_deps:
+ outdated_deps.append(line)
+ if line.startswith('The following dependencies have later '):
+ see_oudated_deps = True
+ raw_report.close()
+ return outdated_deps
+ except Exception, e:
+ raise
+
+
+def extract_single_dep(dep):
+ """
+ Extract a single dependency check record from Java and Python reports.
+ Args:
+ dep: e.g "- org.assertj:assertj-core [2.5.0 -> 3.10.0]".
+ Return:
+ dependency name, current version, latest version.
+ """
+ pattern = " - ([\s\S]*)\[([\s\S]*) -> ([\s\S]*)\]"
+ match = re.match(pattern, dep)
+ if match is None:
+ raise InvalidFormatError("Failed to extract the dependency information:
{}".format(dep))
+ return match.group(1).strip(), match.group(2).strip(), match.group(3).strip()
+
+
+def prioritize_dependencies(deps, sdk_type, project_id, dataset_id, table_id):
+ """
+ Extracts and analyze dependency versions and release dates.
+ Returns a collection of dependencies which is "high priority" in html format:
+ 1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 ->
3.10.0]
+ 2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz
[1.5 -> 1.8]
+ 3. dependency has not been updated for more than 6 months.
+
+ Args:
+ deps: A collection of outdated dependencies.
+ Return:
+ high_priority_deps: A collection of dependencies which need to be taken
care of before next release.
+ """
+ high_priority_deps = []
+ bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
+
+ for dep in deps:
+ try:
+ dep_name, curr_ver, latest_ver = extract_single_dep(dep)
+ curr_release_date, latest_release_date =
query_dependency_release_dates(bigquery_client,
+
dep_name,
+
curr_ver,
+
latest_ver)
+ if sdk_type == 'Java':
+ # extract the groupid and artifactid
+ group_id, artifact_id = dep_name.split(":")
+ dep_details_url = "{0}g:\"{1}\" AND
a:\"{2}\"".format(_MAVEN_CENTRAL_URL, group_id, artifact_id)
+ else:
+ dep_details_url = _PYPI_URL + dep_name
+
+ dep_info = """<tr>
+ <td><a href=\'{0}\'>{1}</a></td>
+ <td>{2}</td>
+ <td>{3}</td>
+ <td>{4}</td>
+ <td>{5}</td>
+ </tr>\n""".format(dep_details_url,
+ dep_name,
+ curr_ver,
+ latest_ver,
+ curr_release_date,
+ latest_release_date)
+ if compare_dependency_versions(curr_ver, latest_ver):
+ high_priority_deps.append(dep_info)
+ elif compare_dependency_release_dates(curr_release_date,
latest_release_date):
+ high_priority_deps.append(dep_info)
+ except:
+ traceback.print_exc()
+ continue
+
+ return high_priority_deps
+
+
+def compare_dependency_versions(curr_ver, latest_ver):
+ """
+ Compare the current using version and the latest version.
+ Return true if a major version change was found, or 3 minor versions that
the current version is behind.
+ Args:
+ curr_ver
+ latest_ver
+ Return:
+ boolean
+ """
+ if curr_ver is None or latest_ver is None:
+ return True
+ else:
+ curr_ver_splitted = curr_ver.split('.')
+ latest_ver_splitted = latest_ver.split('.')
+ curr_major_ver = curr_ver_splitted[0]
+ latest_major_ver = latest_ver_splitted[0]
+ # compare major versions
+ if curr_major_ver != latest_major_ver:
Review comment:
Personally, I prefer to include any dependencies that has differences on
MAJOR versions in the report, because
1. Few people would look into this Jenkins job's log to check errors. In
most cases, raising errors is equivalent to ignoring.
2. Not all packages follow the semantic versioning that allows us to compare
like that.
But, I am also fine to change it if you think validating major versions and
raising errors is a better option.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 114047)
Time Spent: 61h 50m (was: 61h 40m)
> Fix to dependency hell
> ----------------------
>
> Key: BEAM-4302
> URL: https://issues.apache.org/jira/browse/BEAM-4302
> Project: Beam
> Issue Type: New Feature
> Components: testing
> Reporter: yifan zou
> Assignee: yifan zou
> Priority: Major
> Time Spent: 61h 50m
> Remaining Estimate: 0h
>
> # For Java, a daily Jenkins test to compare version of all Beam dependencies
> to the latest version available in Maven Central.
> # For Python, a daily Jenkins test to compare versions of all Beam
> dependencies to the latest version available in PyPI.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
