This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new fabe44b721 Upgrade to RocksDB 9.9.3 (#4580)
fabe44b721 is described below
commit fabe44b7212310109b9645c5de69974ae01bfb84
Author: Lari Hotari <[email protected]>
AuthorDate: Tue Apr 22 19:04:52 2025 +0300
Upgrade to RocksDB 9.9.3 (#4580)
* Upgrade to RocksDB 9.9.3
* Refactor compatibility tests and fix issue with RocksDB 9.x compatibility
tests
* Fix logging in integration tests
* Upgrade Zookeeper version in integration tests
* Update README
* Fix formatting
* Fix formatting
---
.../src/main/resources/LICENSE-all.bin.txt | 6 +-
.../src/main/resources/LICENSE-server.bin.txt | 6 +-
.../bookie/storage/ldb/KeyValueStorageRocksDB.java | 7 +-
pom.xml | 2 +-
tests/README.md | 28 ++++++-
.../backwardcompat/TestCompatUpgradeDirect.groovy | 8 +-
.../TestCompatUpgradeDowngrade.groovy | 91 +++++++++++++---------
.../conf/default_rocksdb.conf | 40 ++++++++++
.../conf/entry_location_rocksdb.conf | 73 +++++++++++++++++
.../conf/ledger_metadata_rocksdb.conf | 40 ++++++++++
.../tests/backwardcompat/TestCompatUpgrade.groovy | 65 ++++++++--------
.../all-released-versions-image/Dockerfile | 4 +-
.../3-node-all-version-unstarted.yaml | 2 +-
tests/integration-tests-utils/pom.xml | 9 +++
.../integration/utils/BookKeeperClusterUtils.java | 53 +++++++++++--
15 files changed, 341 insertions(+), 93 deletions(-)
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index 1098666fa9..f9bc31c8e1 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -271,7 +271,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22]
- lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22]
- lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
- lib/com.beust-jcommander-1.82.jar [24]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -382,7 +382,7 @@ Apache Software License, Version 2.
[20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
[21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
[22] Source available at
https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
[25] Source available at
https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -645,7 +645,7 @@ This private header is also used by Apple's open source
*
http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under
the following license.
+lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under
the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index e638be7a4f..54bd60e648 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -271,7 +271,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22]
- lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22]
- lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
- lib/com.beust-jcommander-1.82.jar [24]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -378,7 +378,7 @@ Apache Software License, Version 2.
[20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
[21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
[22] Source available at
https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
[25] Source available at
https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -640,7 +640,7 @@ This private header is also used by Apple's open source
*
http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under
the following license.
+lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under
the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index b870fb5939..1caea81be2 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -49,6 +49,7 @@ import org.rocksdb.ChecksumType;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.CompressionType;
+import org.rocksdb.ConfigOptions;
import org.rocksdb.DBOptions;
import org.rocksdb.Env;
import org.rocksdb.InfoLogLevel;
@@ -159,8 +160,10 @@ public class KeyValueStorageRocksDB implements
KeyValueStorage {
DBOptions dbOptions = new DBOptions();
final List<ColumnFamilyDescriptor> cfDescs = new ArrayList<>();
final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
- try {
- OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(),
dbOptions, cfDescs, false);
+ try (final ConfigOptions cfgOpts = new ConfigOptions()
+ .setIgnoreUnknownOptions(false)
+ .setEnv(Env.getDefault())) {
+ OptionsUtil.loadOptionsFromFile(cfgOpts, dbFilePath, dbOptions,
cfDescs);
// Configure file path
String logPath = conf.getString(ROCKSDB_LOG_PATH, "");
if (!logPath.isEmpty()) {
diff --git a/pom.xml b/pom.xml
index 5acc3f342e..b88b02aa8b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,7 +167,7 @@
<protoc3.version>${protobuf.version}</protoc3.version>
<protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
<reflections.version>0.9.11</reflections.version>
- <rocksdb.version>7.10.2</rocksdb.version>
+ <rocksdb.version>9.9.3</rocksdb.version>
<shrinkwrap.version>3.3.0</shrinkwrap.version>
<slf4j.version>2.0.12</slf4j.version>
<snakeyaml.version>2.0</snakeyaml.version>
diff --git a/tests/README.md b/tests/README.md
index dff705c4bc..53f5410464 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -4,7 +4,8 @@
### Workaround for running the tests with Mac Apple Silicon
-The current version of the outdated maven plugin requires a workaround.
+The current version of the outdated Maven plugin requires a workaround. This
is also necessary for running the tests
+with an outdated Arquillian Cube version.
Install socat
```bash
@@ -39,13 +40,33 @@ You can add the function to your shell profile
(`~/.bashrc`, `~/.zshrc`, etc.).
### Support for connecting to docker network from Mac host
-You will also need to install
[docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It
allows the tests to connect to the docker network from the Mac host.
+You will also need to install
[docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It
allows the
+tests to connect to the Docker network from the Mac host.
+Disable host network mode in Docker Desktop settings before enabling
`docker-mac-net-connect`, as they cannot be enabled
+simultaneously.
```bash
brew install chipmk/tap/docker-mac-net-connect
sudo brew services start chipmk/tap/docker-mac-net-connect
```
+You will also need to add the following to the Docker Desktop Docker Engine
configuration on MacOS:
+```yaml
+{
+ "default-network-opts": {
+ "bridge": {
+ "com.docker.network.bridge.gateway_mode_ipv4": "nat-unprotected"
+ }
+ }
+}
+```
+
+Stop `docker-mac-net-connect` when you no longer need it:
+
+```bash
+sudo brew services stop chipmk/tap/docker-mac-net-connect
+```
+
## Running the tests
Remember to start the unix socket proxy for docker as described in the
previous section and set the `DOCKER_HOST` environment variable in the terminal
where you run the tests.
@@ -53,6 +74,9 @@ Remember to start the unix socket proxy for docker as
described in the previous
### Building the docker images together with the project source code
```bash
+# remove possible remaining python client versions from previous builds
+git clean -fdx -- stream/clients/python
+# build the project and docker images
mvn -B -nsu clean install -Pdocker -DskipTests
docker images | grep apachebookkeeper
```
diff --git
a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy
b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy
index cb71dc27c4..c66393be63 100644
---
a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy
+++
b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy
@@ -17,6 +17,8 @@
*/
package org.apache.bookkeeper.tests.backwardcompat
+import static
org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.VERSION_4_1_x
+
import com.github.dockerjava.api.DockerClient
import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
@@ -49,8 +51,8 @@ class TestCompatUpgradeDirect {
String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION
int numEntries = 10
-
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
"4.1.0"))
- def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0")
+
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
VERSION_4_1_x))
+ def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x)
def v410BK = v410CL.newBookKeeper(zookeeper)
def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion)
def currentBK = currentCL.newBookKeeper(zookeeper)
@@ -102,7 +104,7 @@ class TestCompatUpgradeDirect {
def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion)
def currentBK = currentCL.newBookKeeper(zookeeper)
- def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0")
+ def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x)
def v410BK = v410CL.newBookKeeper(zookeeper)
try {
diff --git
a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy
b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy
index 74e0c9486a..e448cc27b4 100644
---
a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy
+++
b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy
@@ -17,15 +17,14 @@
*/
package org.apache.bookkeeper.tests.backwardcompat
+import static
org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.*
+
import com.github.dockerjava.api.DockerClient
-import java.util.concurrent.TimeUnit
-import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
+import com.google.common.collect.Lists
import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
import org.jboss.arquillian.junit.Arquillian
import org.jboss.arquillian.test.api.ArquillianResource
-import org.junit.AfterClass
import org.junit.Assert
-import org.junit.BeforeClass
import org.junit.FixMethodOrder
import org.junit.Test
import org.junit.runner.RunWith
@@ -57,62 +56,80 @@ class TestCompatUpgradeDowngrade {
@Test
public void atest_000_setUp() throws Exception {
LOG.info("Running metaformat")
- BookKeeperClusterUtils.legacyMetadataFormat(docker)
+ legacyMetadataFormat(docker)
LOG.info("Setting ledger storage")
- for (String version: BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) {
- BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+ List<String> versions =
Lists.newArrayList(UPGRADE_DOWNGRADE_TEST_VERSIONS)
+ versions.add(CURRENT_VERSION)
+
+ File testRocksDbConfDir = new File(getClass().getClassLoader()
+
.getResource("TestCompatUpgradeDowngrade/conf/default_rocksdb.conf").toURI()).getParentFile()
+
+ boolean useRocksDbVersion5 = false
+ boolean useKxxHash = false
+ for (String version: versions) {
+ appendToAllBookieConf(docker, version,
"ledgerStorageClass",
"org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
+
+ try {
+ // format_version 5 has been supported since RocksDB 6.6.x
+ appendToAllBookieConf(docker, version,
+ "dbStorage_rocksDB_format_version", "5")
+ // kxxHash has been supported for a very long time
+ appendToAllBookieConf(docker, version,
+ "dbStorage_rocksDB_checksum_type", "kxxHash")
+
+ // copy rocksdb.conf to all bookies since some released
versions don't contain the format_version
+ // which is necessary for backwards compatibility
+ copyToAllBookies(docker, version, testRocksDbConfDir)
+ appendToAllBookieConf(docker, version,
+ "entryLocationRocksdbConf",
+ "conf/entry_location_rocksdb.conf")
+ appendToAllBookieConf(docker, version,
+ "ledgerMetadataRocksdbConf",
+ "conf/ledger_metadata_rocksdb.conf")
+ appendToAllBookieConf(docker, version,
+ "defaultRocksdbConf",
+ "conf/default_rocksdb.conf")
+ } catch (Exception e) {
+ LOG.warn(version + ": Failed to set rocksdb configs, might be
ok for some older version", e)
+ }
}
- BookKeeperClusterUtils.appendToAllBookieConf(docker,
BookKeeperClusterUtils.CURRENT_VERSION,
- "ledgerStorageClass",
- "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
}
-
// will ignore older non-supported versions
@Test
- public void upgradeDowngrade_010() {
- upgradeDowngrade("4.12.1", "4.13.0")
- }
-
- @Test
- public void upgradeDowngrade_011() {
- upgradeDowngrade("4.13.0", "4.14.8")
- }
-
- @Test
- public void upgradeDowngrade_012() {
- upgradeDowngrade("4.14.8", "4.15.5")
+ public void upgradeDowngrade_for_4_14_x_and_4_15_x() {
+ upgradeDowngrade(VERSION_4_14_x, VERSION_4_15_x)
}
@Test
- public void upgradeDowngrade_013() {
- upgradeDowngrade("4.15.5", "4.16.5")
+ public void upgradeDowngrade_for_4_15_x_and_4_16_x() {
+ upgradeDowngrade(VERSION_4_15_x, VERSION_4_16_x)
}
@Test
- public void upgradeDowngrade_014() {
- upgradeDowngrade("4.16.5", "4.17.0")
+ public void upgradeDowngrade_for_4_16_x_and_4_17_x() {
+ upgradeDowngrade(VERSION_4_16_x, VERSION_4_17_x)
}
@Test
- public void upgradeDowngrade_015() {
- String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION
- upgradeDowngrade("4.17.0", currentVersion)
+ public void upgradeDowngrade_for_4_17_x_and_CurrentMaster() {
+ String currentVersion = CURRENT_VERSION
+ upgradeDowngrade(VERSION_4_17_x, currentVersion)
}
private void upgradeDowngrade(String oldVersion, String newVersion) throws
Exception {
LOG.info("Testing upgrade/downgrade to/from from {} to {}",
oldVersion, newVersion)
- String zookeeper =
BookKeeperClusterUtils.zookeeperConnectString(docker)
+ String zookeeper = zookeeperConnectString(docker)
int numEntries = 10
LOG.info("Starting bookies with old version {}", oldVersion)
-
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
oldVersion))
+ Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion))
def oldCL = MavenClassLoader.forBookKeeperVersion(oldVersion)
def oldBK = oldCL.newBookKeeper(zookeeper)
@@ -128,8 +145,8 @@ class TestCompatUpgradeDowngrade {
testRead(ledger1, numEntries, oldBK, oldCL)
LOG.info("Upgrade: Stopping all bookies, starting with new version
{}", newVersion)
- Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker))
-
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
newVersion))
+ Assert.assertTrue(stopAllBookies(docker))
+ Assert.assertTrue(startAllBookiesWithVersion(docker, newVersion))
LOG.info("Reading ledger with old client")
testRead(ledger0, numEntries, oldBK, oldCL)
@@ -140,8 +157,8 @@ class TestCompatUpgradeDowngrade {
testRead(ledger1, numEntries, newBK, newCL)
LOG.info("Downgrade: Stopping all bookies, starting with old
version {}", oldVersion)
- Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker))
-
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
oldVersion))
+ Assert.assertTrue(stopAllBookies(docker))
+ Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion))
LOG.info("Reading ledgers with old client")
testRead(ledger0, numEntries, oldBK, oldCL)
@@ -160,7 +177,7 @@ class TestCompatUpgradeDowngrade {
testRead(ledger4, numEntries, newBK, newCL)
} finally {
- BookKeeperClusterUtils.stopAllBookies(docker)
+ stopAllBookies(docker)
newBK.close()
newCL.close()
diff --git
a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf
new file mode 100644
index 0000000000..ccedbb79eb
--- /dev/null
+++
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf
@@ -0,0 +1,40 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# When modifying this file, please also modify the configuration files(at
+# bookkeeper-server/src/test/resources/conf) in the
+# test case to ensure unit test coverage.
+
+[DBOptions]
+ # set by jni: options.setCreateIfMissing
+ create_if_missing=true
+ # set by jni: options.setInfoLogLevel
+ info_log_level=INFO_LEVEL
+ # set by jni: options.setKeepLogFileNum
+ keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
+
+ [CFOptions "default"]
+ #no default setting in CFOptions
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
\ No newline at end of file
diff --git
a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf
new file mode 100644
index 0000000000..e4dc394243
--- /dev/null
+++
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf
@@ -0,0 +1,73 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# When modifying this file, please also modify the configuration files(at
+# bookkeeper-server/src/test/resources/conf) in the
+# test case to ensure unit test coverage.
+
+[DBOptions]
+ # set by jni: options.setCreateIfMissing
+ create_if_missing=true
+ # set by jni: options.setInfoLogLevel
+ info_log_level=INFO_LEVEL
+ # set by jni: options.setKeepLogFileNum
+ keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
+ # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism
+ max_background_jobs=32
+ # set by jni: options.setMaxSubcompactions
+ max_subcompactions=1
+ # set by jni: options.setMaxTotalWalSize
+ max_total_wal_size=536870912
+ # set by jni: options.setMaxOpenFiles
+ max_open_files=-1
+ # set by jni: options.setDeleteObsoleteFilesPeriodMicros
+ delete_obsolete_files_period_micros=3600000000
+
+[CFOptions "default"]
+ # set by jni: options.setCompressionType
+ compression=kLZ4Compression
+ # set by jni: options.setWriteBufferSize
+ write_buffer_size=67108864
+ # set by jni: options.setMaxWriteBufferNumber
+ max_write_buffer_number=4
+ # set by jni: options.setNumLevels
+ num_levels=7
+ # set by jni: options.setLevelZeroFileNumCompactionTrigger
+ level0_file_num_compaction_trigger=4
+ # set by jni: options.setMaxBytesForLevelBase
+ max_bytes_for_level_base=268435456
+ # set by jni: options.setTargetFileSizeBase
+ target_file_size_base=67108864
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setBlockSize
+ block_size=65536
+ # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory()
/ ledgerDirsSize / 10;
+ block_cache=206150041
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
+ # set by jni: tableOptions.setFilterPolicy,
bloomfilter:[bits_per_key]:[use_block_based_builder]
+ filter_policy=rocksdb.BloomFilter:10:false
+ # set by jni: tableOptions.setCacheIndexAndFilterBlocks
+ cache_index_and_filter_blocks=true
\ No newline at end of file
diff --git
a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf
new file mode 100644
index 0000000000..ccedbb79eb
--- /dev/null
+++
b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf
@@ -0,0 +1,40 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# When modifying this file, please also modify the configuration files(at
+# bookkeeper-server/src/test/resources/conf) in the
+# test case to ensure unit test coverage.
+
+[DBOptions]
+ # set by jni: options.setCreateIfMissing
+ create_if_missing=true
+ # set by jni: options.setInfoLogLevel
+ info_log_level=INFO_LEVEL
+ # set by jni: options.setKeepLogFileNum
+ keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
+
+ [CFOptions "default"]
+ #no default setting in CFOptions
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
\ No newline at end of file
diff --git
a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy
b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy
index c987bd5770..20475f1484 100644
---
a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy
+++
b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy
@@ -17,6 +17,8 @@
*/
package org.apache.bookkeeper.tests.backwardcompat
+import static
org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.*
+
import com.github.dockerjava.api.DockerClient
import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
@@ -122,84 +124,83 @@ class TestCompatUpgrade {
@Test
public void test_000() throws Exception {
BookKeeperClusterUtils.legacyMetadataFormat(docker)
-
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
"4.8.2"))
+
Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker,
VERSION_4_8_x))
}
@Test
- public void test_001_482to492() throws Exception {
- testUpgrade("4.8.2", "4.9.2")
+ public void test_001_48xto49x() throws Exception {
+ testUpgrade(VERSION_4_8_x, VERSION_4_9_x)
}
@Test
- public void test_002_492to4100() throws Exception {
- testUpgrade("4.9.2", "4.10.0")
+ public void test_002_49xto410x() throws Exception {
+ testUpgrade(VERSION_4_9_x, VERSION_4_10_x)
}
@Test
- public void test_003_4100to4111() throws Exception {
- testUpgrade("4.10.0", "4.11.1")
+ public void test_003_410xto411x() throws Exception {
+ testUpgrade(VERSION_4_10_x, VERSION_4_11_x)
}
@Test
- public void test_004_4111to4121() throws Exception {
- testUpgrade("4.11.1", "4.12.1")
+ public void test_004_411xto412x() throws Exception {
+ testUpgrade(VERSION_4_11_x, VERSION_4_12_x)
}
@Test
- public void test_005_4121to4130() throws Exception {
- testUpgrade("4.12.1", "4.13.0")
+ public void test_005_412xto413x() throws Exception {
+ testUpgrade(VERSION_4_12_x, VERSION_4_13_x)
}
@Test
- public void test_006_4130to4148() throws Exception {
- testUpgrade("4.13.0", "4.14.8")
+ public void test_006_413xto414x() throws Exception {
+ testUpgrade(VERSION_4_13_x, VERSION_4_14_x)
}
@Test
- public void test_007_4148to4155() throws Exception {
- testUpgrade("4.14.8", "4.15.5")
+ public void test_007_414xto415x() throws Exception {
+ testUpgrade(VERSION_4_14_x, VERSION_4_15_x)
}
@Test
- public void test_007_4148to4155_crc32c() throws Exception {
- testUpgrade("4.14.8", "4.15.5", "CRC32C")
+ public void test_007_414xto415x_crc32c() throws Exception {
+ testUpgrade(VERSION_4_14_x, VERSION_4_15_x, "CRC32C")
}
@Test
- public void test_008_4155to4165() throws Exception {
- testUpgrade("4.15.5", "4.16.5")
+ public void test_008_415xto416x() throws Exception {
+ testUpgrade(VERSION_4_15_x, VERSION_4_16_x)
}
@Test
- public void test_008_4155to4165_crc32c() throws Exception {
- testUpgrade("4.15.5", "4.16.5", "CRC32C")
+ public void test_008_415xto416x_crc32c() throws Exception {
+ testUpgrade(VERSION_4_15_x, VERSION_4_16_x, "CRC32C")
}
@Test
- public void test_008_4165to4170_crc32c() throws Exception {
- testUpgrade("4.16.5", "4.17.0", "CRC32C")
+ public void test_008_416xto417x_crc32c() throws Exception {
+ testUpgrade(VERSION_4_16_x, VERSION_4_17_x, "CRC32C")
}
@Test
- public void test_009_4165toCurrentMaster() throws Exception {
- testUpgrade("4.17.0", BookKeeperClusterUtils.CURRENT_VERSION)
+ public void test_009_417xtoCurrentMaster() throws Exception {
+ testUpgrade(VERSION_4_17_x, CURRENT_VERSION)
}
@Test
- public void test_009_4165toCurrentMaster_crc32c() throws Exception {
- testUpgrade("4.17.0", BookKeeperClusterUtils.CURRENT_VERSION, "CRC32C")
+ public void test_009_417xtoCurrentMaster_crc32c() throws Exception {
+ testUpgrade(VERSION_4_17_x, CURRENT_VERSION, "CRC32C")
}
// old version pulsar upgrade tests
@Test
- public void test_010_4100to4148_crc32c() throws Exception {
- testUpgrade("4.10.0", "4.14.8", "CRC32C")
+ public void test_010_410xto414x_crc32c() throws Exception {
+ testUpgrade(VERSION_4_10_x, VERSION_4_14_x, "CRC32C")
}
// old version pulsar upgrade tests
@Test
- public void test_010_4100to4170_crc32c() throws Exception {
- testUpgrade("4.10.0", "4.17.0", "CRC32C")
+ public void test_010_410xto417x_crc32c() throws Exception {
+ testUpgrade(VERSION_4_10_x, VERSION_4_17_x, "CRC32C")
}
-
}
diff --git a/tests/docker-images/all-released-versions-image/Dockerfile
b/tests/docker-images/all-released-versions-image/Dockerfile
index 6bbdad2c30..0051d1b7e6 100644
--- a/tests/docker-images/all-released-versions-image/Dockerfile
+++ b/tests/docker-images/all-released-versions-image/Dockerfile
@@ -51,8 +51,8 @@ RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookke
RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc}
RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.8/bookkeeper-server-4.14.8-bin.tar.gz{,.sha512,.asc}
RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.15.5/bookkeeper-server-4.15.5-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.5/bookkeeper-server-4.16.5-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.0/bookkeeper-server-4.17.0-bin.tar.gz{,.sha512,.asc}
+RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.6/bookkeeper-server-4.16.6-bin.tar.gz{,.sha512,.asc}
+RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.1/bookkeeper-server-4.17.1-bin.tar.gz{,.sha512,.asc}
RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS
RUN wget -nv
http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O
KEYS.old
diff --git
a/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml
b/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml
index a422f4372f..5cd627f258 100644
---
a/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml
+++
b/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml
@@ -22,7 +22,7 @@ networks:
driver: bridge
zookeeper*:
- image: zookeeper:3.6.2
+ image: zookeeper:3.9.3
await:
strategy:
org.apache.bookkeeper.tests.integration.utils.ZooKeeperAwaitStrategy
aliases:
diff --git a/tests/integration-tests-utils/pom.xml
b/tests/integration-tests-utils/pom.xml
index 0a972558b9..b5aead0500 100644
--- a/tests/integration-tests-utils/pom.xml
+++ b/tests/integration-tests-utils/pom.xml
@@ -74,6 +74,15 @@
<artifactId>javax.ws.rs-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ </dependency>
+
</dependencies>
<build>
diff --git
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java
index a6c8e8604f..b8eb44516e 100644
---
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java
+++
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java
@@ -18,6 +18,8 @@
package org.apache.bookkeeper.tests.integration.utils;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
@@ -39,12 +41,27 @@ import org.slf4j.LoggerFactory;
*/
public class BookKeeperClusterUtils {
public static final String CURRENT_VERSION =
System.getProperty("currentVersion");
+
+ public static final String VERSION_4_1_x = "4.1.0";
+ public static final String VERSION_4_8_x = "4.8.2";
+ public static final String VERSION_4_9_x = "4.9.2";
+ public static final String VERSION_4_10_x = "4.10.0";
+ public static final String VERSION_4_11_x = "4.11.1";
+ public static final String VERSION_4_12_x = "4.12.1";
+ public static final String VERSION_4_13_x = "4.13.0";
+ public static final String VERSION_4_14_x = "4.14.8";
+ public static final String VERSION_4_15_x = "4.15.5";
+ public static final String VERSION_4_16_x = "4.16.6";
+ public static final String VERSION_4_17_x = "4.17.1";
+
public static final List<String> OLD_CLIENT_VERSIONS =
- Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1",
- "4.13.0", "4.14.8", "4.15.5", "4.16.5", "4.17.0");
+ Arrays.asList(VERSION_4_8_x, VERSION_4_9_x, VERSION_4_10_x,
VERSION_4_11_x, VERSION_4_12_x,
+ VERSION_4_13_x, VERSION_4_14_x, VERSION_4_15_x,
VERSION_4_16_x, VERSION_4_17_x);
private static final List<String>
OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT =
- Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1",
- "4.13.0", "4.14.8", "4.15.5", "4.16.5", "4.17.0");
+ Arrays.asList(VERSION_4_9_x, VERSION_4_10_x, VERSION_4_11_x,
VERSION_4_12_x,
+ VERSION_4_13_x, VERSION_4_14_x, VERSION_4_15_x,
VERSION_4_16_x, VERSION_4_17_x);
+ public static final List<String> UPGRADE_DOWNGRADE_TEST_VERSIONS =
+ Arrays.asList(VERSION_4_14_x, VERSION_4_15_x, VERSION_4_16_x,
VERSION_4_17_x);
private static final Logger LOG =
LoggerFactory.getLogger(BookKeeperClusterUtils.class);
@@ -143,16 +160,38 @@ public class BookKeeperClusterUtils {
DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e",
sedProgram, confFile);
}
- public static void appendToAllBookieConf(DockerClient docker, String
version, String key, String value)
+ public static void copyToAllBookies(DockerClient docker, String version,
File sourceDirectory)
+ throws Exception {
+ for (String containerId : allBookies()) {
+ CopyArchiveToContainerCmd copyArchiveToContainerCmd =
docker.copyArchiveToContainerCmd(containerId);
+
copyArchiveToContainerCmd.withHostResource(sourceDirectory.getAbsolutePath());
+ copyArchiveToContainerCmd.withRemotePath("/opt/bookkeeper/" +
version);
+ copyArchiveToContainerCmd.exec();
+ }
+ }
+
+ public static void appendToAllBookieConf(DockerClient docker, String
version, String confFile0,
+ String key, String value)
throws Exception {
for (String b : allBookies()) {
- appendToBookieConf(docker, b, version, key, value);
+ appendToBookieConf(docker, b, version, confFile0, key, value);
}
}
+ public static void appendToAllBookieConf(DockerClient docker, String
version,
+ String key, String value)
+ throws Exception {
+ appendToAllBookieConf(docker, version, "conf/bk_server.conf", key,
value);
+ }
+
public static void appendToBookieConf(DockerClient docker, String
containerId,
String version, String key, String
value) throws Exception {
- String confFile = "/opt/bookkeeper/" + version +
"/conf/bk_server.conf";
+ appendToBookieConf(docker, containerId, version,
"conf/bk_server.conf", key, value);
+ }
+
+ public static void appendToBookieConf(DockerClient docker, String
containerId,
+ String version, String confFile0,
String key, String value) throws Exception {
+ String confFile = "/opt/bookkeeper/" + version + "/" + confFile0;
String sedProgram = String.format("$a%s=%s", key, value);
DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e",
sedProgram, confFile);
}
