This is an automated email from the ASF dual-hosted git repository. marcuse pushed a commit to branch cassandra-3.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit a6dd3b89d94a5a97d4b304242dc49085866cda43 Merge: 4e6c0fa 6e3bf54 Author: Marcus Eriksson <marc...@apache.org> AuthorDate: Wed Jan 29 09:33:31 2020 +0100 Merge branch 'cassandra-2.2' into cassandra-3.0 .circleci/config-2_1.yml | 56 +++++++++++--- .circleci/config-2_1.yml.high_res.patch | 14 +++- .circleci/config.yml | 125 +++++++++++++++++++++++++++----- .circleci/config.yml.HIGHRES | 125 +++++++++++++++++++++++++++----- .circleci/config.yml.LOWRES | 125 +++++++++++++++++++++++++++----- 5 files changed, 375 insertions(+), 70 deletions(-) diff --cc .circleci/config-2_1.yml index a4410fe,3a1eff5..6b59a4f --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@@ -23,6 -23,12 +23,18 @@@ j8_par_executor: &j8_par_executo #exec_resource_class: xlarge parallelism: 4 + j8_small_par_executor: &j8_small_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge + parallelism: 1 + ++j8_medium_par_executor: &j8_medium_par_executor ++ executor: ++ name: java8-executor ++ #exec_resource_class: xlarge ++ parallelism: 1 ++ j8_seq_executor: &j8_seq_executor executor: name: java8-executor @@@ -54,16 -60,6 +66,15 @@@ with_dtests_jobs: &with_dtest_job - utests_compression: requires: - start_utests_compression + - start_jvm_upgrade_dtest: + type: approval ++ - j8_dtest_jars_build: + requires: + - build - - dtest_jars_build: - requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: - - dtest_jars_build ++ - j8_dtest_jars_build # Java 8 dtests (on request) - start_j8_dtests: type: approval @@@ -130,18 -125,6 +141,18 @@@ jobs - cassandra - .m2 - dtest_jars_build: ++ j8_dtest_jars_build: + executor: java8-executor + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - build_cassandra_dtest_jars + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + j8_unit_tests: <<: *j8_par_executor steps: @@@ -156,17 -139,13 +167,25 @@@ steps: - attach_workspace: at: /home/cassandra - - run_junit_tests: - target: test-jvm-dtest-forking + - create_junit_containers: + classlistprefix: distributed + extra_filters: "| grep -v upgrade" + - log_environment + - run_parallel_junit_tests: + classlistprefix: distributed + j8_jvm_upgrade_dtests: + <<: *j8_seq_executor + steps: + - attach_workspace: + at: /home/cassandra - - run_junit_tests: - target: test-jvm-upgrade-dtest-forking ++ - create_junit_containers: ++ classlistprefix: distributed ++ extra_filters: "| grep upgrade" ++ - log_environment ++ - run_parallel_junit_tests: ++ classlistprefix: distributed + utests_long: <<: *j8_seq_executor steps: @@@ -412,7 -357,7 +441,10 @@@ commands export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> no_output_timeout: <<parameters.no_output_timeout>> - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config-2_1.yml.high_res.patch index 22f434d,847a08f..3c85668 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@@ -10,7 -10,13 +10,19 @@@ --- > exec_resource_class: xlarge > parallelism: 100 - 29c29 + 29,30c29,30 + < #exec_resource_class: xlarge + < parallelism: 1 + --- + > exec_resource_class: xlarge + > parallelism: 2 -35c35 ++35,36c35,36 ++< #exec_resource_class: xlarge ++< parallelism: 1 ++--- ++> exec_resource_class: xlarge ++> parallelism: 2 ++41c41 < #exec_resource_class: xlarge --- > exec_resource_class: xlarge diff --cc .circleci/config.yml index d80b78d,ae582af..817d726 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@@ -1,51 -1,5 +1,92 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) ++ name: Determine distributed Tests to Run ++ command: | ++ # reminder: this code (along with all the steps) is independently executed on every circle container ++ # so the goal here is to get the circleci script to return the tests *this* container will run ++ # which we do via the `circleci` cli tool. ++ ++ rm -fr ~/cassandra-dtest/upgrade_tests ++ echo "***java tests***" ++ ++ # get all of our unit test filenames ++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt ++ ++ # split up the unit tests into groups based on the number of containers we have ++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" ++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ no_output_timeout: 15m ++ - run: ++ name: Log Environment Information ++ command: | ++ echo '*** id ***' ++ id ++ echo '*** cat /proc/cpuinfo ***' ++ cat /proc/cpuinfo ++ echo '*** free -m ***' ++ free -m ++ echo '*** df -m ***' ++ df -m ++ echo '*** ifconfig -a ***' ++ ifconfig -a ++ echo '*** uname -a ***' ++ uname -a ++ echo '*** mount ***' ++ mount ++ echo '*** env ***' ++ env ++ echo '*** java ***' ++ which java ++ java -version ++ - run: ++ name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -341,6 -132,6 +382,9 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@@ -442,10 -216,7 +527,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -575,6 -343,6 +660,9 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@@ -602,7 -370,123 +690,7 @@@ - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: - j8_dtests-with-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-res [...] - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inte [...] - - run: - name: Run dtests (j8_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not th [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: ++ j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 resource_class: medium @@@ -706,6 -610,6 +784,15 @@@ workflows - utests_compression: requires: - start_utests_compression ++ - start_jvm_upgrade_dtest: ++ type: approval ++ - j8_dtest_jars_build: ++ requires: ++ - build ++ - start_jvm_upgrade_dtest ++ - j8_jvm_upgrade_dtests: ++ requires: ++ - j8_dtest_jars_build - start_j8_dtests: type: approval requires: diff --cc .circleci/config.yml.HIGHRES index 94ca5b2,e5ed274..b1e138c --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@@ -1,51 -1,5 +1,92 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) ++ name: Determine distributed Tests to Run ++ command: | ++ # reminder: this code (along with all the steps) is independently executed on every circle container ++ # so the goal here is to get the circleci script to return the tests *this* container will run ++ # which we do via the `circleci` cli tool. ++ ++ rm -fr ~/cassandra-dtest/upgrade_tests ++ echo "***java tests***" ++ ++ # get all of our unit test filenames ++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt ++ ++ # split up the unit tests into groups based on the number of containers we have ++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" ++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ no_output_timeout: 15m ++ - run: ++ name: Log Environment Information ++ command: | ++ echo '*** id ***' ++ id ++ echo '*** cat /proc/cpuinfo ***' ++ cat /proc/cpuinfo ++ echo '*** free -m ***' ++ free -m ++ echo '*** df -m ***' ++ df -m ++ echo '*** ifconfig -a ***' ++ ifconfig -a ++ echo '*** uname -a ***' ++ uname -a ++ echo '*** mount ***' ++ mount ++ echo '*** env ***' ++ env ++ echo '*** java ***' ++ which java ++ java -version ++ - run: ++ name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -295,7 -170,7 +336,7 @@@ - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run - name: Determine distributed Tests to Run ++ name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@@ -309,7 -184,7 +350,7 @@@ # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@@ -341,7 -216,7 +382,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -437,15 -297,53 +481,56 @@@ - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) - name: Determine unit Tests to Run ++ name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt ++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: - name: Run Unit Tests (testclasslist-compression) ++ name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking ++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -529,62 -439,36 +614,65 @@@ - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run - name: Clone Cassandra dtest Repository (via git) ++ name: Determine unit Tests to Run command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m - run: - name: Configure virtualenv and python Dependencies + name: Log Environment Information command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inte [...] + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version - run: - name: Run dtests (j8_without_vnodes) + name: Run Unit Tests (testclasslist-compression) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not th [...] - store_test_results: - path: /tmp/results + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + path: /tmp/cassandra/build/test/output + destination: junitxml - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + path: /tmp/cassandra/build/test/logs + destination: logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant @@@ -602,13 -486,13 +690,13 @@@ - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: - j8_upgradetests-no-vnodes: ++ j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge + resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 100 + parallelism: 1 steps: - attach_workspace: at: /home/cassandra @@@ -706,6 -610,6 +784,15 @@@ workflows - utests_compression: requires: - start_utests_compression ++ - start_jvm_upgrade_dtest: ++ type: approval ++ - j8_dtest_jars_build: ++ requires: ++ - build ++ - start_jvm_upgrade_dtest ++ - j8_jvm_upgrade_dtests: ++ requires: ++ - j8_dtest_jars_build - start_j8_dtests: type: approval requires: diff --cc .circleci/config.yml.LOWRES index d80b78d,ae582af..817d726 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@@ -1,51 -1,5 +1,92 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) ++ name: Determine distributed Tests to Run ++ command: | ++ # reminder: this code (along with all the steps) is independently executed on every circle container ++ # so the goal here is to get the circleci script to return the tests *this* container will run ++ # which we do via the `circleci` cli tool. ++ ++ rm -fr ~/cassandra-dtest/upgrade_tests ++ echo "***java tests***" ++ ++ # get all of our unit test filenames ++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt ++ ++ # split up the unit tests into groups based on the number of containers we have ++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt ++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" ++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt ++ no_output_timeout: 15m ++ - run: ++ name: Log Environment Information ++ command: | ++ echo '*** id ***' ++ id ++ echo '*** cat /proc/cpuinfo ***' ++ cat /proc/cpuinfo ++ echo '*** free -m ***' ++ free -m ++ echo '*** df -m ***' ++ df -m ++ echo '*** ifconfig -a ***' ++ ifconfig -a ++ echo '*** uname -a ***' ++ uname -a ++ echo '*** mount ***' ++ mount ++ echo '*** env ***' ++ env ++ echo '*** java ***' ++ which java ++ java -version ++ - run: ++ name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -341,6 -132,6 +382,9 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@@ -442,10 -216,7 +527,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -575,6 -343,6 +660,9 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@@ -602,7 -370,123 +690,7 @@@ - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: - j8_dtests-with-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-res [...] - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inte [...] - - run: - name: Run dtests (j8_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not th [...] - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: ++ j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 resource_class: medium @@@ -706,6 -610,6 +784,15 @@@ workflows - utests_compression: requires: - start_utests_compression ++ - start_jvm_upgrade_dtest: ++ type: approval ++ - j8_dtest_jars_build: ++ requires: ++ - build ++ - start_jvm_upgrade_dtest ++ - j8_jvm_upgrade_dtests: ++ requires: ++ - j8_dtest_jars_build - start_j8_dtests: type: approval requires: --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org