This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch cassandra-3.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push: new 02048cffea Remove -l / -m / -h designation and have two options: free or paid tier circle config 02048cffea is described below commit 02048cffeae4267054541f96741b16b1d0d89a20 Author: Andrés de la Peña <a.penya.gar...@gmail.com> AuthorDate: Tue Feb 28 11:44:31 2023 +0000 Remove -l / -m / -h designation and have two options: free or paid tier circle config patch by Andrés de la Peña; reviewed by Ekaterina Dimitrova and Josh McKenzie for CASSANDRA-18012 --- .circleci/config-2_1.yml.high_res.patch | 105 - .circleci/{config.yml.LOWRES => config.yml.FREE} | 0 .circleci/config.yml.HIGHRES | 3103 -------------------- .circleci/{config.yml.MIDRES => config.yml.PAID} | 0 .circleci/{config-2_1.yml => config_template.yml} | 0 ...id_res.patch => config_template.yml.PAID.patch} | 0 .circleci/generate.sh | 107 +- .circleci/readme.md | 98 +- 8 files changed, 93 insertions(+), 3320 deletions(-) diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch deleted file mode 100644 index d62ba2d740..0000000000 --- a/.circleci/config-2_1.yml.high_res.patch +++ /dev/null @@ -1,105 +0,0 @@ ---- config-2_1.yml 2023-02-07 19:05:10.000000000 -0500 -+++ config-2_1.yml.HIGHRES 2023-02-07 19:25:03.000000000 -0500 -@@ -40,8 +40,8 @@ - CASSANDRA_SKIP_SYNC: true - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - DTEST_BRANCH: trunk -- CCM_MAX_HEAP_SIZE: 1024M -- CCM_HEAP_NEWSIZE: 256M -+ CCM_MAX_HEAP_SIZE: 2048M -+ CCM_HEAP_NEWSIZE: 512M - - # Whether the repeated test iterations should stop on the first failure by default. - REPEATED_TESTS_STOP_ON_FAILURE: false -@@ -127,14 +127,14 @@ - j8_par_executor: &j8_par_executor - executor: - name: java8-executor -- #exec_resource_class: xlarge -- parallelism: 4 -+ exec_resource_class: xlarge -+ parallelism: 100 - - j8_small_par_executor: &j8_small_par_executor - executor: - name: java8-executor -- #exec_resource_class: xlarge -- parallelism: 1 -+ exec_resource_class: xlarge -+ parallelism: 2 - - j8_small_executor: &j8_small_executor - executor: -@@ -145,34 +145,38 @@ - j8_medium_par_executor: &j8_medium_par_executor - executor: - name: java8-executor -- #exec_resource_class: xlarge -- parallelism: 1 -+ exec_resource_class: xlarge -+ parallelism: 2 - - j8_seq_executor: &j8_seq_executor - executor: - name: java8-executor -- #exec_resource_class: xlarge -+ exec_resource_class: xlarge - parallelism: 1 # sequential, single container tests: no parallelism benefits - - j8_repeated_utest_executor: &j8_repeated_utest_executor - executor: - name: java8-executor -- parallelism: 4 -+ exec_resource_class: xlarge -+ parallelism: 100 - - j8_repeated_dtest_executor: &j8_repeated_dtest_executor - executor: - name: java8-executor -- parallelism: 4 -+ exec_resource_class: xlarge -+ parallelism: 100 - - j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor - executor: - name: java8-executor -- parallelism: 4 -+ exec_resource_class: xlarge -+ parallelism: 100 - - j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor - executor: - name: java8-executor -- parallelism: 4 -+ exec_resource_class: xlarge -+ parallelism: 100 - - separate_jobs: &separate_jobs - jobs: -@@ -579,7 +583,7 @@ - pytest_extra_args: '--skip-resource-intensive-tests' - - j8_dtests_large_vnode: -- <<: *j8_par_executor -+ <<: *j8_seq_executor - steps: - - attach_workspace: - at: /home/cassandra -@@ -593,7 +597,7 @@ - pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests' - - j8_dtests_large: -- <<: *j8_par_executor -+ <<: *j8_seq_executor - steps: - - attach_workspace: - at: /home/cassandra -@@ -1187,7 +1191,7 @@ - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" -- -+ - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=<<parameters.target>> - testtag="" diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.FREE similarity index 100% rename from .circleci/config.yml.LOWRES rename to .circleci/config.yml.FREE diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES deleted file mode 100644 index 6719225b5e..0000000000 --- a/.circleci/config.yml.HIGHRES +++ /dev/null @@ -1,3103 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: 2 -jobs: - j8_jvm_upgrade_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_compression_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-compression - testtag="" - if [[ $target == "test-compression" ]]; then - testtag="compression" - fi - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_large_vnode: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_large_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes -- [...] - - run: - name: Run dtests (j8_large_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest an [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_large_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_large_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlshlib_cython_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run cqlshlib Unit Tests - command: | - export PATH=$JAVA_HOME/bin:$PATH - export cython="yes" - time mv ~/cassandra /tmp - cd /tmp/cassandra/ - ./pylib/cassandra-cqlsh-tests.sh $(pwd) - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/pylib - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_large_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_LARGE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=32" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=long-testsome - testtag="" - if [[ $target == "test-compression" ]]; then - testtag="compression" - fi - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=testsome - testtag="" - if [[ $target == "test-compression" ]]; then - testtag="compression" - fi - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_large: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_large_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resou [...] - - run: - name: Run dtests (j8_large_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pyt [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_large_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_large_without_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=32" - fi - - upgrade_arg="" - if true; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra - - run: - name: Build Cassandra - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean jar build-test - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - no_output_timeout: 15m - - run: - name: Run eclipse-warnings - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_vnode_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTESTS}" == "<nil>" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if true; then - vnodes_args="--use-vnodes --num-tokens=32" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgrade_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_upgradetests_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --exe [...] - - run: - name: Run dtests (j8_upgradetests_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually u [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_upgradetests_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_upgradetests_without_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-jvm-dtest-some - testtag="" - if [[ $target == "test-compression" ]]; then - testtag="compression" - fi - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-in [...] - - run: - name: Run dtests (j8_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_vnode: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...] - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-jvm-dtest-some - testtag="" - if [[ $target == "test-compression" ]]; then - testtag="compression" - fi - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_ant_test: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test - no_output_timeout: 15m - command: | - if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_ANT_TEST_TARGET} - class_path=${REPEATED_ANT_TEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "long-test" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTESTS}" == "<nil>" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=32" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (long-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - ant long-test -Dno-build-test=true - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlshlib_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run cqlshlib Unit Tests - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra/ - ./pylib/cassandra-cqlsh-tests.sh $(pwd) - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/pylib - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests_large_vnode_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_LARGE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if true; then - vnodes_args="--use-vnodes --num-tokens=32" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_compression: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist-compression) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtest_jars_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Build Cassandra DTest jars - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - mkdir ~/dtest_jars - git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do - # check out the correct cassandra version: - git remote set-branches --add apache '$branch' - git fetch --depth 1 apache $branch - git checkout $branch - git clean -fd - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - done - # and build the dtest-jar for the branch under test - ${ANT_HOME}/bin/ant realclean - git checkout origin/$CIRCLE_BRANCH - git clean -fd - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - ls -l ~/dtest_jars - no_output_timeout: 15m - - persist_to_workspace: - root: /home/cassandra - paths: - - dtest_jars - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -workflows: - version: 2 - separate_tests: - jobs: - - start_build: - type: approval - - build: - requires: - - start_build - - start_j8_unit_tests: - type: approval - - j8_unit_tests: - requires: - - start_j8_unit_tests - - build - - start_j8_jvm_dtests: - type: approval - - j8_jvm_dtests: - requires: - - start_j8_jvm_dtests - - build - - start_j8_cqlshlib_tests: - type: approval - - j8_cqlshlib_tests: - requires: - - start_j8_cqlshlib_tests - - build - - start_j8_cqlshlib_cython_tests: - type: approval - - j8_cqlshlib_cython_tests: - requires: - - start_j8_cqlshlib_cython_tests - - build - - start_utests_long: - type: approval - - utests_long: - requires: - - start_utests_long - - build - - start_utests_compression: - type: approval - - utests_compression: - requires: - - start_utests_compression - - build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - start_j8_dtest_jars_build - - build - - start_jvm_upgrade_dtests: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtests - - j8_dtest_jars_build - - start_j8_dtests: - type: approval - - j8_dtests: - requires: - - start_j8_dtests - - build - - start_j8_dtests_vnode: - type: approval - - j8_dtests_vnode: - requires: - - start_j8_dtests_vnode - - build - - start_j8_dtests_large: - type: approval - - j8_dtests_large: - requires: - - start_j8_dtests_large - - build - - start_j8_dtests_large_vnode: - type: approval - - j8_dtests_large_vnode: - requires: - - start_j8_dtests_large_vnode - - build - - start_j8_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_j8_upgrade_dtests - - build - pre-commit_tests: - jobs: - - start_pre-commit_tests: - type: approval - - build: - requires: - - start_pre-commit_tests - - j8_unit_tests: - requires: - - build - - j8_jvm_dtests: - requires: - - build - - j8_cqlshlib_tests: - requires: - - build - - j8_cqlshlib_cython_tests: - requires: - - build - - start_utests_long: - type: approval - - utests_long: - requires: - - start_utests_long - - build - - start_utests_compression: - type: approval - - utests_compression: - requires: - - start_utests_compression - - build - - start_jvm_upgrade_dtests: - type: approval - - j8_dtest_jars_build: - requires: - - start_jvm_upgrade_dtests - - build - - j8_jvm_upgrade_dtests: - requires: - - j8_dtest_jars_build - - j8_dtests: - requires: - - build - - j8_dtests_vnode: - requires: - - build - - start_j8_dtests_large: - type: approval - - j8_dtests_large: - requires: - - start_j8_dtests_large - - build - - j8_dtests_large_vnode: - requires: - - start_j8_dtests_large - - build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_upgrade_dtests - - build diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.PAID similarity index 100% rename from .circleci/config.yml.MIDRES rename to .circleci/config.yml.PAID diff --git a/.circleci/config-2_1.yml b/.circleci/config_template.yml similarity index 100% rename from .circleci/config-2_1.yml rename to .circleci/config_template.yml diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config_template.yml.PAID.patch similarity index 100% rename from .circleci/config-2_1.yml.mid_res.patch rename to .circleci/config_template.yml.PAID.patch diff --git a/.circleci/generate.sh b/.circleci/generate.sh index 6308da477b..ee62a9b9da 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -30,13 +30,12 @@ die () print_help() { - echo "Usage: $0 [-l|-m|-h|-f|-e]" - echo " -a Generate the default config.yml using low resources and the three templates" - echo " (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES). Use this for" - echo " permanent changes in config-2_1.yml that will be committed to the main repo." - echo " -l Generate config.yml using low resources" - echo " -m Generate config.yml using mid resources" - echo " -h Generate config.yml using high resources" + echo "Usage: $0 [-f|-p|-a|-e|-i]" + echo " -a Generate the config.yml, config.yml.FREE and config.yml.PAID expanded configuration" + echo " files from the main config_template.yml reusable configuration file." + echo " Use this for permanent changes in config that will be committed to the main repo." + echo " -f Generate config.yml for tests compatible with the CircleCI free tier resources" + echo " -p Generate config.yml for tests compatible with the CircleCI paid tier resources" echo " -e <key=value> Environment variables to be used in the generated config.yml, e.g.:" echo " -e DTEST_BRANCH=CASSANDRA-8272" echo " -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git" @@ -60,28 +59,25 @@ print_help() echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable" echo " -e REPEATED_ANT_TEST_COUNT=500" echo " For the complete list of environment variables, please check the" - echo " list of examples in config-2_1.yml and/or the documentation." + echo " list of examples in config_template.yml and/or the documentation." echo " If you want to specify multiple environment variables simply add" - echo " multiple -e options. The flags -l/-m/-h should be used when using -e." - echo " -f Stop checking that the environment variables are known" + echo " multiple -e options. The flags -f/-p should be used when using -e." + echo " -i Ignore unknown environment variables" } all=false -lowres=false -midres=false -highres=false +free=false +paid=false env_vars="" has_env_vars=false check_env_vars=true -while getopts "e:almhf" opt; do +while getopts "e:afpi" opt; do case $opt in a ) all=true ;; - l ) lowres=true + f ) free=true ;; - m ) midres=true - ;; - h ) highres=true + p ) paid=true ;; e ) if (!($has_env_vars)); then env_vars="$OPTARG" @@ -90,7 +86,7 @@ while getopts "e:almhf" opt; do fi has_env_vars=true ;; - f ) check_env_vars=false + i ) check_env_vars=false ;; \?) die "Invalid option: -$OPTARG" ;; @@ -131,52 +127,40 @@ if $has_env_vars && $check_env_vars; then done fi -if $lowres; then - ($all || $midres || $highres) && die "Cannot use option -l with options -a, -m or -h" - echo "Generating new config.yml file with low resources from config-2_1.yml" - circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml - rm $BASEDIR/config.yml.LOWRES.tmp - -elif $midres; then - ($all || $lowres || $highres) && die "Cannot use option -m with options -a, -l or -h" - echo "Generating new config.yml file with middle resources from config-2_1.yml" - patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch - circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml - rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp +if $free; then + ($all || $paid) && die "Cannot use option -f with options -a or -p" + echo "Generating new config.yml file for free tier from config_template.yml" + circleci config process $BASEDIR/config_template.yml > $BASEDIR/config.yml.FREE.tmp + cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp > $BASEDIR/config.yml + rm $BASEDIR/config.yml.FREE.tmp -elif $highres; then - ($all || $lowres || $midres) && die "Cannot use option -h with options -a, -l or -m" - echo "Generating new config.yml file with high resources from config-2_1.yml" - patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch - circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml - rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp +elif $paid; then + ($all || $free) && die "Cannot use option -p with options -a or -f" + echo "Generating new config.yml file for paid tier from config_template.yml" + patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch + circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp + cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml + rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp elif $all; then - ($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option -a with options -l, -m, -h or -e" - echo "Generating new config.yml file with low resources and LOWRES/MIDRES/HIGHRES templates from config-2_1.yml" - - # setup lowres - circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml.LOWRES - rm $BASEDIR/config.yml.LOWRES.tmp + ($free || $paid || $has_env_vars) && die "Cannot use option -a with options -f, -p or -e" + echo "Generating new default config.yml file for free tier and FREE/PAID templates from config_template.yml." + echo "Make sure you commit the newly generated config.yml, config.yml.FREE and config.yml.PAID files" + echo "after running this command if you want them to persist." - # setup midres - patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch - circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml.MIDRES - rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp + # setup config for free tier + circleci config process $BASEDIR/config_template.yml > $BASEDIR/config.yml.FREE.tmp + cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp > $BASEDIR/config.yml.FREE + rm $BASEDIR/config.yml.FREE.tmp - # setup highres - patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch - circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp - cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml.HIGHRES - rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp + # setup config for paid tier + patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch + circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp + cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml.PAID + rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp - # copy lower into config.yml to make sure this gets updated - cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml + # copy free tier into config.yml to make sure this gets updated + cp $BASEDIR/config.yml.FREE $BASEDIR/config.yml elif (!($has_env_vars)); then print_help @@ -290,7 +274,6 @@ delete_repeated_jobs() delete_repeated_jobs "config.yml" if $all; then - delete_repeated_jobs "config.yml.LOWRES" - delete_repeated_jobs "config.yml.MIDRES" - delete_repeated_jobs "config.yml.HIGHRES" + delete_repeated_jobs "config.yml.FREE" + delete_repeated_jobs "config.yml.PAID" fi diff --git a/.circleci/readme.md b/.circleci/readme.md index 9eb37678f8..e65be5158a 100644 --- a/.circleci/readme.md +++ b/.circleci/readme.md @@ -22,42 +22,47 @@ This directory contains the configuration for CircleCI continous integration platform. The file `config.yml` is the configuration file that is read by CircleCI. This file is -automatically generated by the `generate.sh` script from the `config-2_1.yml` file. +automatically generated by the `generate.sh` script from the `config_template.yml` file. The provided `config.yml` file uses low resources so users of the CircleCI free tier can -use it. Additionally, there are three versions of this file using different resources so +use it. Additionally, there are two versions of this file using different resources so users who have access to premium CircleCI resources can use larger instances and more -parallelism. These files are `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES`. -The default `config.yml` file is just a copy of `config.yml.LOWRES`. +parallelism. These files are `config.yml.FREE` and `config.yml.PAID`. +The default `config.yml` file is just a copy of `config.yml.FREE`. ## Switching to higher resource settings -This directory contains generated files for low, middle and high resource settings. -Switch between them by copying the correct file to `config.yml` and committing the result: +This directory contains generated files for free and paid resource settings. +Switch between them by copying the correct file to `config.yml` and committing the result. +For example, for using higher resources benefiting from CircleCI's paid tier: -`cp .circleci/config.yml.HIGHRES .circleci/config.yml` +`cp .circleci/config.yml.PAID .circleci/config.yml` -Alternatively, you can run the `generate.sh` script with the flags `-l`/`-m`/`-h` -to regenerate the `config.yml` file from `config-2_1.yml` using LOWRES/MIDRES/HIGHRES. -This script validates and applies any changes to the `config-2_1.yml` file, and it +And for using lower resources comaptible with CircleCI's free tier: + +`cp .circleci/config.yml.FREE .circleci/config.yml` + +Alternatively, you can run the `generate.sh` script with the flags `-f`/`-p` +to regenerate the `config.yml` file from `config_template.yml` using free or paid resources. +This script validates and applies any changes to the `config_template.yml` file, and it requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be installed. ## Setting environment variables -Both `config-2_1.yml` and `config.yml` files contain a set of environment variables +Both `config_template.yml` and `config.yml` files contain a set of environment variables defining things like what dtest repo and branch to use, what tests could be repeatedly run, etc. These environment variables can be directly edited in the `config.yml` file, although if you do this you should take into account that the entire set of env vars is repeated on -every job. +every job. -A probably better approach is editing them in `config-2_1.yml` and then regenerate the +A probably better approach is editing them in `config_template.yml` and then regenerate the `config.yml` file using the `generate.sh` script. You can also directly pass environment variable values to the `generate.sh` script with the `-e` flag. For example, to set the dtest repo and branch with MIDRES config you can run: ``` -generate.sh -m \ +generate.sh -p \ -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git \ -e DTEST_BRANCH=CASSANDRA-8272 @@ -69,7 +74,7 @@ The script will then create jobs to run each of these new or modified tests for number of times, to verify that they are stable. You can use environment variables to specify the number of iterations of each type of test: ``` -generate.sh -m \ +generate.sh -p \ -e REPEATED_UTESTS_COUNT=500 \ -e REPEATED_UTESTS_LONG_COUNT=100 \ -e REPEATED_SIMULATOR_DTESTS_COUNT=500 \ @@ -82,12 +87,12 @@ generate.sh -m \ ``` You can also specify whether the iteration should fail on the first test failure: ``` -generate.sh -m -e REPEATED_TESTS_STOP_ON_FAILURE=false +generate.sh -p -e REPEATED_TESTS_STOP_ON_FAILURE=false ``` In addition to the automatically detected tests, it's also possible to provide lists of specific tests to be repeated: ``` -generate.sh -m \ +generate.sh -p \ -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \ -e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \ -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \ @@ -99,7 +104,7 @@ generate.sh -m \ For particular Ant test targets that are not included in the regular test suites, you can use the `run_repeated_utest` job: ``` -generate.sh -m \ +generate.sh -p \ -e REPEATED_ANT_TEST_TARGET=test-cdc \ -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \ -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \ @@ -107,7 +112,7 @@ generate.sh -m \ ``` Putting all together, you can have runs as complex as: ``` -generate.sh -m \ +generate.sh -p \ -e REPEATED_TESTS_STOP_ON_FAILURE=true \ -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \ -e REPEATED_UTESTS_COUNT=500 \ @@ -135,45 +140,38 @@ default medium config and Large containers. ## Updating the config For configuration changes meant to be permanent in the Apache repo you should never edit -the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then -regenerate the `config.yml`, `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES` +the `config.yml` file manually. Instead, you should edit the `config_template.yml` file and then +regenerate the `config.yml`, `config.yml.FREE` and `config.yml.PAID` files by runnining the `generate.sh` script with `-a` flag. For using this script you need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install). As for temporal changes done while working in a patch, such as pointing to you dtest repo or -running a test repeatedly, you can either directly edit `config.yml` or edit `config-2_1.yml` -and then regenerate `config.yml` with the `generate.sh` script using a `-l`/`-m`/`-h` flag. +running a test repeatedly, you can either directly edit `config.yml` or edit `config_template.yml` +and then regenerate `config.yml` with the `generate.sh` script using a `-f`/`-p` flag. When this flag is used only the `config.yml` will be generated. Please note that any previous swapping or edition of the generated files will be overriden -by running `generate.sh` with `-a` argument, returning `config.yml` to the default LOWRES. So if -you previously swapped your `config.yml` to MIDRES or HIGHRES you would need to either swap it -again or use the `-l`/`-m`/`-h` script flags. +by running `generate.sh` with `-a` argument, returning `config.yml` to the default FREE. So if +you previously swapped your `config.yml` to MIDRES you would need to either swap it +again or use the `-f`/`-p` script flags. Read below for details how to generate the files manually without the `generate.sh` script: -1. make your edits to config-2_1.yml - let it stay at lowres settings -2. generate a valid LOWRES file: - `circleci config process config-2_1.yml > config.yml.LOWRES` -3. add the Apache license header to the newly created LOWRES file: - `cat license.yml config.yml.LOWRES > config.yml.LOWRES.new && mv config.yml.LOWRES.new config.yml.LOWRES` -4. then apply the highres patch to config-2_1.yml; - `patch -o config-2_1.yml.HIGHRES config-2_1.yml config-2_1.yml.high_res.patch` - (this creates a new file `config-2_1.yml.HIGHRES` instead of in-place patching - config-2_1.yml) - Note that if the patch no longer applies to `config-2_1.yml` a new patch file - is needed, do this by manually making `config-2_1.yml` high resource and create +1. make your edits to config_template.yml - let it stay at free tier settings +2. generate a valid FREE file: + `circleci config process config_template.yml > config.yml.FREE` +3. add the Apache license header to the newly created FREE file: + `cat license.yml config.yml.FREE > config.yml.FREE.new && mv config.yml.FREE.new config.yml.FREE` +4. then apply the PAID patch to config_template.yml; + `patch -o config_template.yml.PAID config_template.yml config_template.yml.PAID.patch` + (this creates a new file `config_template.yml.PAID` instead of in-place patching + config_template.yml). + Note that if the patch no longer applies to `config_template.yml` a new patch file + is needed, do this by manually creating and editing `config_template.yml.PAID`, and create the patch file based on the diff (don't commit it though). -5. generate the HIGHRES file: - `circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES` -6. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES` -7. add the Apache license header to the newly created HIGHRES file: - `cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv config.yml.HIGHRES.new config.yml.HIGHRES` -8. repeat the last steps to generate the MIDRES file: - ``` - patch -o config-2_1.yml.MIDRES config-2_1.yml config-2_1.yml.mid_res.patch - circleci config process config-2_1.yml.MIDRES > config.yml.MIDRES - rm config-2_1.yml.MIDRES - cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv config.yml.MIDRES.new config.yml.MIDRES - ``` -9. finally, remember to update the config.yml +5. generate the PAID file: + `circleci config process config_template.yml.PAID > config.yml.PAID` +6. remove the temporary patched PAID file: `rm config_template.yml.PAID` +7. add the Apache license header to the newly created PAID file: + `cat license.yml config.yml.PAID > config.yml.PAID.new && mv config.yml.PAID.new config.yml.PAID` +8. finally, remember to update the config.yml --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org