This is an automated email from the ASF dual-hosted git repository.
bbejeck pushed a commit to branch Merge_AK_to_CCS_10_08_2020
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 30761ff64d8a612de96bf430c0358edce2fb7fbb
Author: elismaga <e...@confluent.io>
AuthorDate: Fri Aug 7 17:06:39 2020 -0700
ST-3458: Implement Nano versioning (#338)
---
Jenkinsfile | 13 +++-
build.gradle | 2 +-
ci.py | 88 ++++++++++++++++++++++
gradle.properties | 2 +-
streams/quickstart/java/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
streams/quickstart/pom.xml | 2 +-
tests/README.md | 2 +-
tests/kafkatest/__init__.py | 4 +-
.../tests/streams/streams_upgrade_test.py | 4 +-
tests/kafkatest/utils/util.py | 9 +++
tests/kafkatest/version.py | 2 +-
12 files changed, 120 insertions(+), 12 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 834837c..3fd5921 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -25,6 +25,8 @@ def config = jobConfig {
timeoutHours = 4
runMergeCheck = false
downStreamValidate = true
+ downStreamRepos = ["common",]
+ disableConcurrentBuilds = true
}
def retryFlagsString(jobConfig) {
@@ -38,6 +40,7 @@ def publishStep(String vaultSecret) {
sh "./gradlewAll --init-script ${GRADLE_NEXUS_SETTINGS} --no-daemon
uploadArchives"
}
}
+
def job = {
//
https://github.com/confluentinc/common-tools/blob/master/confluent/config/dev/versions.json
def kafkaMuckrakeVersionMap = [
@@ -48,10 +51,14 @@ def job = {
"master": "master"
]
+ if (!config.isReleaseJob) {
+ ciTool("ci-update-version ${env.WORKSPACE} kafka")
+ }
+
stage("Check compilation compatibility with Scala 2.12") {
sh "./gradlew clean assemble spotlessScalaCheck checkstyleMain
checkstyleTest spotbugsMain " +
"--no-daemon --stacktrace -PxmlSpotBugsReport=true
-PscalaVersion=2.12"
- }
+ }
stage("Compile and validate") {
@@ -61,6 +68,10 @@ def job = {
if (config.publish) {
stage("Publish to artifactory") {
+ if (!config.isReleaseJob && !config.isPrJob) {
+ ciTool("ci-push-tag ${env.WORKSPACE} kafka")
+ }
+
if (config.isDevJob) {
publishStep('artifactory_snapshots_settings')
} else if (config.isPreviewJob) {
diff --git a/build.gradle b/build.gradle
index e6d54a3..546b4fd 100644
--- a/build.gradle
+++ b/build.gradle
@@ -141,7 +141,7 @@ ext {
userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ?
maxTestRetryFailures.toInteger() : 0
skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
- shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") &&
!version.contains("-alpha") && project.gradle.startParameter.taskNames.any {
it.contains("upload") }
+ shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") &&
!version.contains("-alpha") &&
!version.matches("[0-9]*\\.[0-9]*\\.[0-9]*-[0-9]*-(ccs|ce)") &&
project.gradle.startParameter.taskNames.any { it.contains("upload") }
mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername
: ''
diff --git a/ci.py b/ci.py
new file mode 100644
index 0000000..195aa59
--- /dev/null
+++ b/ci.py
@@ -0,0 +1,88 @@
+#!/usr/bin/python
+
+import os
+import logging
+import re
+import subprocess
+import sys
+
+from confluent.ci.scripts.ci_utils import run_cmd, regex_replace, replace
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+log = logging.getLogger(__name__)
+
+
+class CI:
+
+ def __init__(self, new_version, repo_path):
+ """Initialize class variables"""
+ # List of all the files that were modified by this script so the parent
+ # script that runs this update can commit them.
+ self.updated_files = []
+ # The new version
+ self.new_version = new_version
+ # The path the root of the repo so we can use full absolute paths
+ self.repo_path = repo_path
+
+ def run_update(self):
+ """Update all the files with the new version"""
+ log.info("Running additional version updates for kafka")
+ self.update_kafkatest()
+ self.update_quickstart()
+ log.info("Finished all kafka additional version updates.")
+
+ def update_kafkatest(self):
+ """Update kafka test python scripts"""
+ log.info("Updating kafkatest init script.")
+ init_file = os.path.join(self.repo_path, "tests/kafkatest/__init__.py")
+ # Determine if this is a ccs or ce kafka version.
+ kafka_qualifier = self.new_version.split("-")[2]
+ dev_version =
"{}.dev0".format(self.new_version.split("-{}".format(kafka_qualifier))[0])
+ replace(init_file, "__version__", "__version__ =
'{}'".format(dev_version))
+ self.updated_files.append(init_file)
+ log.info("Updating ducktape version.py")
+ ducktape_version_file = os.path.join(self.repo_path,
"tests/kafkatest/version.py")
+ # The version in this file does not contain the qualifier
+ ducktape_version = self.new_version.split(kafka_qualifier)[0]
+ regex_replace(ducktape_version_file,
+ "^DEV_VERSION = KafkaVersion.*",
+ "DEV_VERSION = KafkaVersion(\"{}\")".format(ducktape_version))
+ self.updated_files.append(ducktape_version_file)
+
+ def update_quickstart(self):
+ """Uodate the streams quick start pom files."""
+ log.info("Updating streams quickstart pom files.")
+ quickstart_pom = os.path.join(self.repo_path,
"streams/quickstart/pom.xml")
+ self.update_project_versions(quickstart_pom, self.new_version)
+ self.updated_files.append(quickstart_pom)
+ # Do not need to explicitly update this file because the above command
updates all pom files in the project.
+ # Just need to add it to the list of modified files.
+ quickstart_java_pom = os.path.join(self.repo_path,
"streams/quickstart/java/pom.xml")
+ self.updated_files.append(quickstart_java_pom)
+ # The maven plugin has fails to process this pom file because it is an
archetype style, so have to use regex.
+ log.info("Updating streams quickstart archetype pom")
+ archetype_resources_pom = os.path.join(self.repo_path,
+
"streams/quickstart/java/src/main/resources/archetype-resources/pom.xml")
+ regex_replace(archetype_resources_pom,
+ "<kafka\.version>.*</kafka\.version>",
+ "<kafka.version>{}</kafka.version>".format(self.new_version))
+ self.updated_files.append(archetype_resources_pom)
+
+ def update_project_versions(self, pom_file, new_version):
+ """Set the project version in the pom files to the new project
version."""
+ cmd = ["mvn", "--batch-mode", "versions:set",
+ "-DnewVersion={} ".format(new_version),
+ "-DallowSnapshots=false",
+ "-DgenerateBackupPoms=false",
+ "-DprocessAllModules=true",
+ "-DprocessDependencies=false",
+ "-DprocessPlugins=false",
+ "-f",
+ pom_file]
+ log.info("Updating pom files with new project version.")
+ _, success = run_cmd(cmd, self.repo_path)
+
+ if not success:
+ log.error("Failed to set the new version in the pom files.")
+ sys.exit(1)
+
+ log.info("Finished updating the pom files with new project version.")
diff --git a/gradle.properties b/gradle.properties
index 827db3b..e6c5dd3 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -20,7 +20,7 @@ group=org.apache.kafka
# - tests/kafkatest/__init__.py
# - tests/kafkatest/version.py (variable DEV_VERSION)
# - kafka-merge-pr.py
-version=6.1.0-ccs-SNAPSHOT
+version=6.1.0-0-ccs
scalaVersion=2.13.3
task=build
org.gradle.jvmargs=-Xmx2g -Xss4m -XX:+UseParallelGC
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
index 7e83aa6..172ed6e 100644
--- a/streams/quickstart/java/pom.xml
+++ b/streams/quickstart/java/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.kafka</groupId>
<artifactId>streams-quickstart</artifactId>
- <version>2.7.0-SNAPSHOT</version>
+ <version>6.1.0-0-ccs</version>
<relativePath>..</relativePath>
</parent>
diff --git
a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
index f0d5444..64f815a 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
@@ -29,7 +29,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <kafka.version>6.1.0-ccs-SNAPSHOT</kafka.version>
+ <kafka.version>6.1.0-0-ccs</kafka.version>
<slf4j.version>1.7.7</slf4j.version>
<log4j.version>1.2.17</log4j.version>
</properties>
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
index f1918c2..844b917 100644
--- a/streams/quickstart/pom.xml
+++ b/streams/quickstart/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.kafka</groupId>
<artifactId>streams-quickstart</artifactId>
<packaging>pom</packaging>
- <version>6.1.0-ccs-SNAPSHOT</version>
+ <version>6.1.0-0-ccs</version>
<name>Kafka Streams :: Quickstart</name>
diff --git a/tests/README.md b/tests/README.md
index ae677e9..528e300 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -38,7 +38,7 @@
TC_PATHS="tests/kafkatest/tests/client/pluggable_test.py::PluggableConsumerTest.
```
* Run a specific test method with specific parameters
```
-TC_PATHS="tests/kafkatest/tests/streams/streams_upgrade_test.py::StreamsUpgradeTest.test_metadata_upgrade"
_DUCKTAPE_OPTIONS='--parameters
'\''{"from_version":"0.10.1.1","to_version":"2.6.0-SNAPSHOT"}'\' bash
tests/docker/run_tests.sh
+TC_PATHS="tests/kafkatest/tests/streams/streams_upgrade_test.py::StreamsUpgradeTest.test_metadata_upgrade"
_DUCKTAPE_OPTIONS='--parameters
'\''{"from_version":"0.10.1.1","to_version":"2.6.0"}'\' bash
tests/docker/run_tests.sh
```
* Run tests with a different JVM
```
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
index f024994..cdf7600 100644
--- a/tests/kafkatest/__init__.py
+++ b/tests/kafkatest/__init__.py
@@ -21,5 +21,5 @@
#
# Instead, in development branches, the version should have a suffix of the
form ".devN"
#
-# For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be
something like "1.0.0.dev0"
-__version__ = '6.1.0.dev0'
\ No newline at end of file
+# For example, when Kafka is at version 1.0.0-0, this should be something like
"1.0.0-0.dev0"
+__version__ = '6.1.0-0.dev0'
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py
b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index aa9bbee..ec8a64a 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -457,8 +457,8 @@ class StreamsUpgradeTest(Test):
self.old_processors.remove(processor)
self.upgraded_processors.append(processor)
- # checking for the dev version which should be the only
SNAPSHOT
- log_monitor.wait_until("Kafka version.*" +
self.base_version_number + ".*SNAPSHOT",
+ # checking for the dev version
+ log_monitor.wait_until("Kafka version.*" +
self.base_version_number + ".*",
timeout_sec=60,
err_msg="Could not detect Kafka
Streams version " + str(DEV_VERSION) + " in " + str(node.account))
log_monitor.offset = 5
diff --git a/tests/kafkatest/utils/util.py b/tests/kafkatest/utils/util.py
index 60f6007..fa56737 100644
--- a/tests/kafkatest/utils/util.py
+++ b/tests/kafkatest/utils/util.py
@@ -54,6 +54,15 @@ def _kafka_jar_versions(proc_string):
# Pattern example: kafka-streams-0.11.0.0-SNAPSHOT.jar
versions.extend(re.findall("kafka-[a-z]+-([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)",
proc_string))
+ # Pattern example: kafka-stream_2.11-6.0.0-0.jar (nano version
implemetation)
+
versions.extend(re.findall("kafka_[0-9]+\.[0-9]+-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)",
proc_string))
+
+ # Pattern example: kafka-1.0.0-0/bin/../libs/*
+ versions.extend(re.findall("kafka-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)/",
proc_string))
+
+ # Pattern example: kafka-streams-1.0.0-0.jar
+
versions.extend(re.findall("kafka-[a-z]+-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+)[\.-][a-zA-z]",
proc_string))
+
return set(versions)
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
index cbb93fd..cb6cfd3 100644
--- a/tests/kafkatest/version.py
+++ b/tests/kafkatest/version.py
@@ -75,7 +75,7 @@ def get_version(node=None):
return DEV_BRANCH
DEV_BRANCH = KafkaVersion("dev")
-DEV_VERSION = KafkaVersion("6.1.0-SNAPSHOT")
+DEV_VERSION = KafkaVersion("6.1.0-0")
# 0.8.2.x versions
V_0_8_2_1 = KafkaVersion("0.8.2.1")
