This is an automated email from the ASF dual-hosted git repository. bbejeck pushed a commit to branch Merge_AK_to_CCS_10_08_2020 in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 30761ff64d8a612de96bf430c0358edce2fb7fbb Author: elismaga <e...@confluent.io> AuthorDate: Fri Aug 7 17:06:39 2020 -0700 ST-3458: Implement Nano versioning (#338) --- Jenkinsfile | 13 +++- build.gradle | 2 +- ci.py | 88 ++++++++++++++++++++++ gradle.properties | 2 +- streams/quickstart/java/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- streams/quickstart/pom.xml | 2 +- tests/README.md | 2 +- tests/kafkatest/__init__.py | 4 +- .../tests/streams/streams_upgrade_test.py | 4 +- tests/kafkatest/utils/util.py | 9 +++ tests/kafkatest/version.py | 2 +- 12 files changed, 120 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 834837c..3fd5921 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,6 +25,8 @@ def config = jobConfig { timeoutHours = 4 runMergeCheck = false downStreamValidate = true + downStreamRepos = ["common",] + disableConcurrentBuilds = true } def retryFlagsString(jobConfig) { @@ -38,6 +40,7 @@ def publishStep(String vaultSecret) { sh "./gradlewAll --init-script ${GRADLE_NEXUS_SETTINGS} --no-daemon uploadArchives" } } + def job = { // https://github.com/confluentinc/common-tools/blob/master/confluent/config/dev/versions.json def kafkaMuckrakeVersionMap = [ @@ -48,10 +51,14 @@ def job = { "master": "master" ] + if (!config.isReleaseJob) { + ciTool("ci-update-version ${env.WORKSPACE} kafka") + } + stage("Check compilation compatibility with Scala 2.12") { sh "./gradlew clean assemble spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain " + "--no-daemon --stacktrace -PxmlSpotBugsReport=true -PscalaVersion=2.12" - } + } stage("Compile and validate") { @@ -61,6 +68,10 @@ def job = { if (config.publish) { stage("Publish to artifactory") { + if (!config.isReleaseJob && !config.isPrJob) { + ciTool("ci-push-tag ${env.WORKSPACE} kafka") + } + if (config.isDevJob) { publishStep('artifactory_snapshots_settings') } else if (config.isPreviewJob) { diff --git a/build.gradle b/build.gradle index e6d54a3..546b4fd 100644 --- a/build.gradle +++ b/build.gradle @@ -141,7 +141,7 @@ ext { userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ? maxTestRetryFailures.toInteger() : 0 skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean() - shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && !version.contains("-alpha") && project.gradle.startParameter.taskNames.any { it.contains("upload") } + shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && !version.contains("-alpha") && !version.matches("[0-9]*\\.[0-9]*\\.[0-9]*-[0-9]*-(ccs|ce)") && project.gradle.startParameter.taskNames.any { it.contains("upload") } mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : '' mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : '' diff --git a/ci.py b/ci.py new file mode 100644 index 0000000..195aa59 --- /dev/null +++ b/ci.py @@ -0,0 +1,88 @@ +#!/usr/bin/python + +import os +import logging +import re +import subprocess +import sys + +from confluent.ci.scripts.ci_utils import run_cmd, regex_replace, replace +logging.basicConfig(level=logging.INFO, format='%(message)s') +log = logging.getLogger(__name__) + + +class CI: + + def __init__(self, new_version, repo_path): + """Initialize class variables""" + # List of all the files that were modified by this script so the parent + # script that runs this update can commit them. + self.updated_files = [] + # The new version + self.new_version = new_version + # The path the root of the repo so we can use full absolute paths + self.repo_path = repo_path + + def run_update(self): + """Update all the files with the new version""" + log.info("Running additional version updates for kafka") + self.update_kafkatest() + self.update_quickstart() + log.info("Finished all kafka additional version updates.") + + def update_kafkatest(self): + """Update kafka test python scripts""" + log.info("Updating kafkatest init script.") + init_file = os.path.join(self.repo_path, "tests/kafkatest/__init__.py") + # Determine if this is a ccs or ce kafka version. + kafka_qualifier = self.new_version.split("-")[2] + dev_version = "{}.dev0".format(self.new_version.split("-{}".format(kafka_qualifier))[0]) + replace(init_file, "__version__", "__version__ = '{}'".format(dev_version)) + self.updated_files.append(init_file) + log.info("Updating ducktape version.py") + ducktape_version_file = os.path.join(self.repo_path, "tests/kafkatest/version.py") + # The version in this file does not contain the qualifier + ducktape_version = self.new_version.split(kafka_qualifier)[0] + regex_replace(ducktape_version_file, + "^DEV_VERSION = KafkaVersion.*", + "DEV_VERSION = KafkaVersion(\"{}\")".format(ducktape_version)) + self.updated_files.append(ducktape_version_file) + + def update_quickstart(self): + """Uodate the streams quick start pom files.""" + log.info("Updating streams quickstart pom files.") + quickstart_pom = os.path.join(self.repo_path, "streams/quickstart/pom.xml") + self.update_project_versions(quickstart_pom, self.new_version) + self.updated_files.append(quickstart_pom) + # Do not need to explicitly update this file because the above command updates all pom files in the project. + # Just need to add it to the list of modified files. + quickstart_java_pom = os.path.join(self.repo_path, "streams/quickstart/java/pom.xml") + self.updated_files.append(quickstart_java_pom) + # The maven plugin has fails to process this pom file because it is an archetype style, so have to use regex. + log.info("Updating streams quickstart archetype pom") + archetype_resources_pom = os.path.join(self.repo_path, + "streams/quickstart/java/src/main/resources/archetype-resources/pom.xml") + regex_replace(archetype_resources_pom, + "<kafka\.version>.*</kafka\.version>", + "<kafka.version>{}</kafka.version>".format(self.new_version)) + self.updated_files.append(archetype_resources_pom) + + def update_project_versions(self, pom_file, new_version): + """Set the project version in the pom files to the new project version.""" + cmd = ["mvn", "--batch-mode", "versions:set", + "-DnewVersion={} ".format(new_version), + "-DallowSnapshots=false", + "-DgenerateBackupPoms=false", + "-DprocessAllModules=true", + "-DprocessDependencies=false", + "-DprocessPlugins=false", + "-f", + pom_file] + log.info("Updating pom files with new project version.") + _, success = run_cmd(cmd, self.repo_path) + + if not success: + log.error("Failed to set the new version in the pom files.") + sys.exit(1) + + log.info("Finished updating the pom files with new project version.") diff --git a/gradle.properties b/gradle.properties index 827db3b..e6c5dd3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,7 +20,7 @@ group=org.apache.kafka # - tests/kafkatest/__init__.py # - tests/kafkatest/version.py (variable DEV_VERSION) # - kafka-merge-pr.py -version=6.1.0-ccs-SNAPSHOT +version=6.1.0-0-ccs scalaVersion=2.13.3 task=build org.gradle.jvmargs=-Xmx2g -Xss4m -XX:+UseParallelGC diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml index 7e83aa6..172ed6e 100644 --- a/streams/quickstart/java/pom.xml +++ b/streams/quickstart/java/pom.xml @@ -26,7 +26,7 @@ <parent> <groupId>org.apache.kafka</groupId> <artifactId>streams-quickstart</artifactId> - <version>2.7.0-SNAPSHOT</version> + <version>6.1.0-0-ccs</version> <relativePath>..</relativePath> </parent> diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml index f0d5444..64f815a 100644 --- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml +++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml @@ -29,7 +29,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <kafka.version>6.1.0-ccs-SNAPSHOT</kafka.version> + <kafka.version>6.1.0-0-ccs</kafka.version> <slf4j.version>1.7.7</slf4j.version> <log4j.version>1.2.17</log4j.version> </properties> diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml index f1918c2..844b917 100644 --- a/streams/quickstart/pom.xml +++ b/streams/quickstart/pom.xml @@ -22,7 +22,7 @@ <groupId>org.apache.kafka</groupId> <artifactId>streams-quickstart</artifactId> <packaging>pom</packaging> - <version>6.1.0-ccs-SNAPSHOT</version> + <version>6.1.0-0-ccs</version> <name>Kafka Streams :: Quickstart</name> diff --git a/tests/README.md b/tests/README.md index ae677e9..528e300 100644 --- a/tests/README.md +++ b/tests/README.md @@ -38,7 +38,7 @@ TC_PATHS="tests/kafkatest/tests/client/pluggable_test.py::PluggableConsumerTest. ``` * Run a specific test method with specific parameters ``` -TC_PATHS="tests/kafkatest/tests/streams/streams_upgrade_test.py::StreamsUpgradeTest.test_metadata_upgrade" _DUCKTAPE_OPTIONS='--parameters '\''{"from_version":"0.10.1.1","to_version":"2.6.0-SNAPSHOT"}'\' bash tests/docker/run_tests.sh +TC_PATHS="tests/kafkatest/tests/streams/streams_upgrade_test.py::StreamsUpgradeTest.test_metadata_upgrade" _DUCKTAPE_OPTIONS='--parameters '\''{"from_version":"0.10.1.1","to_version":"2.6.0"}'\' bash tests/docker/run_tests.sh ``` * Run tests with a different JVM ``` diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py index f024994..cdf7600 100644 --- a/tests/kafkatest/__init__.py +++ b/tests/kafkatest/__init__.py @@ -21,5 +21,5 @@ # # Instead, in development branches, the version should have a suffix of the form ".devN" # -# For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0" -__version__ = '6.1.0.dev0' \ No newline at end of file +# For example, when Kafka is at version 1.0.0-0, this should be something like "1.0.0-0.dev0" +__version__ = '6.1.0-0.dev0' diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py index aa9bbee..ec8a64a 100644 --- a/tests/kafkatest/tests/streams/streams_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py @@ -457,8 +457,8 @@ class StreamsUpgradeTest(Test): self.old_processors.remove(processor) self.upgraded_processors.append(processor) - # checking for the dev version which should be the only SNAPSHOT - log_monitor.wait_until("Kafka version.*" + self.base_version_number + ".*SNAPSHOT", + # checking for the dev version + log_monitor.wait_until("Kafka version.*" + self.base_version_number + ".*", timeout_sec=60, err_msg="Could not detect Kafka Streams version " + str(DEV_VERSION) + " in " + str(node.account)) log_monitor.offset = 5 diff --git a/tests/kafkatest/utils/util.py b/tests/kafkatest/utils/util.py index 60f6007..fa56737 100644 --- a/tests/kafkatest/utils/util.py +++ b/tests/kafkatest/utils/util.py @@ -54,6 +54,15 @@ def _kafka_jar_versions(proc_string): # Pattern example: kafka-streams-0.11.0.0-SNAPSHOT.jar versions.extend(re.findall("kafka-[a-z]+-([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)", proc_string)) + # Pattern example: kafka-stream_2.11-6.0.0-0.jar (nano version implemetation) + versions.extend(re.findall("kafka_[0-9]+\.[0-9]+-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)", proc_string)) + + # Pattern example: kafka-1.0.0-0/bin/../libs/* + versions.extend(re.findall("kafka-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)/", proc_string)) + + # Pattern example: kafka-streams-1.0.0-0.jar + versions.extend(re.findall("kafka-[a-z]+-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)[\.-][a-zA-z]", proc_string)) + return set(versions) diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index cbb93fd..cb6cfd3 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -75,7 +75,7 @@ def get_version(node=None): return DEV_BRANCH DEV_BRANCH = KafkaVersion("dev") -DEV_VERSION = KafkaVersion("6.1.0-SNAPSHOT") +DEV_VERSION = KafkaVersion("6.1.0-0") # 0.8.2.x versions V_0_8_2_1 = KafkaVersion("0.8.2.1")