Repository: cassandra Updated Branches: refs/heads/trunk 6fc3699ad -> e73bc32a9
Migrate dtests to use pytest and python3 Patch by Michael Kjellman; Reviewed by Ariel Weisberg for CASSANDRA-14134 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e73bc32a Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e73bc32a Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e73bc32a Branch: refs/heads/trunk Commit: e73bc32a940175a21700721bafaa593aa86a1a5f Parents: 6fc3699 Author: Michael Kjellman <kjell...@apple.com> Authored: Mon Jan 8 18:12:35 2018 -0500 Committer: Ariel Weisberg <aweisb...@apple.com> Committed: Mon Jan 29 16:21:50 2018 -0500 ---------------------------------------------------------------------- .circleci/config.yml | 339 +++++++++++++++++++ CHANGES.txt | 1 + circle.yml | 18 - .../io/compress/CompressionMetadata.java | 3 +- .../org/apache/cassandra/io/util/FileUtils.java | 47 ++- .../org/apache/cassandra/utils/SyncUtil.java | 16 +- 6 files changed, 399 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/.circleci/config.yml ---------------------------------------------------------------------- diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..27efa2d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,339 @@ +default_env_vars: &default_env_vars + JAVA_HOME: /usr/local/openjdk8u154-cassandra-b02 + ANT_HOME: /usr/local/apache-ant-1.10.1 + LANG: en_US.UTF-8 + JDK_HOME: /usr/local/openjdk8u154-cassandra-b02 + JAVA8_HOME: /usr/local/openjdk8u154-cassandra-b02 + JAVA7_HOME: /usr/local/openjdk7u82-cassandra-b02 + KEEP_TEST_DIR: true + DEFAULT_DIR: /home/cassandra/cassandra-dtest + PYTHONIOENCODING: utf-8 + PYTHONUNBUFFERED: true + CASS_DRIVER_NO_EXTENSIONS: true + CASS_DRIVER_NO_CYTHON: true +# For environments with xlarge instances, use more memory +high_capacity_env_vars: &high_capacity_env_vars + <<: *default_env_vars + CCM_MAX_HEAP_SIZE: 2048M + CCM_HEAP_NEWSIZE: 512M +# For environments with limited memory (e.g the free OSS CircleCI Tier) +resource_constrained_env_vars: &resource_constrained_env_vars + <<: *default_env_vars + CCM_MAX_HEAP_SIZE: 1024M + CCM_HEAP_NEWSIZE: 256M +# Settings for users who do not have a paid CircleCI account +default_env_settings: &default_env_settings + resource_class: medium + parallelism: 4 +# Settings for users with high-capacity, paid CircleCI account +high_capacity_env_settings: &high_capacity_env_settings + resource_class: xlarge + parallelism: 100 +default_jobs: &default_jobs + jobs: + - build + - unit_tests: + requires: + - build +with_dtests_jobs: &with_dtest_jobs + jobs: + - build + - unit_tests: + requires: + - build + - dtests-with-vnodes: + requires: + - build + - dtests-no-vnodes: + requires: + - build +with_dtest_jobs_only: &with_dtest_jobs_only + jobs: + - build + - dtests-with-vnodes: + requires: + - build + - dtests-no-vnodes: + requires: + - build +# Set env_settings, env_vars, and workflows/build_and_run_tests based on environment +env_settings: &env_settings + # <<: *default_env_settings + <<: *high_capacity_env_settings +env_vars: &env_vars + # <<: *resource_constrained_env_vars + <<: *high_capacity_env_vars +workflows: + version: 2 + build_and_run_tests: *default_jobs + #build_and_run_tests: *with_dtest_jobs_only + #build_and_run_tests: *with_dtest_jobs +docker_image: &docker_image kjellman/cassandra-test:0.4.3 +version: 2 +jobs: + build: + <<: *env_settings + parallelism: 1 # This job doesn't benefit from parallelism + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + docker: + - image: *docker_image + environment: + <<: *env_vars + steps: + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + - run: + name: Clone Cassandra Repository (via git) + command: | + export LANG=en_US.UTF-8 + git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + - run: + name: Build Cassandra + command: | + export LANG=en_US.UTF-8 + export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8" + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + cd ~/cassandra + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + unit_tests: + <<: *env_settings + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + docker: + - image: *docker_image + environment: + <<: *env_vars + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Tests to Run + no_output_timeout: 15m + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + export LANG=en_US.UTF-8 + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + - run: + name: Run Unit Tests + command: | + export LANG=en_US.UTF-8 + export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8" + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + #Skip all syncing to disk to avoid performance issues in flaky CI environments + export CASSANDRA_SKIP_SYNC=true + + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + dtests-with-vnodes: + <<: *env_settings + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + docker: + - image: *docker_image + environment: + <<: *env_vars + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + export LANG=en_US.UTF-8 + git clone --single-branch --branch master --depth 1 git://github.com/apache/cassandra-dtest.git ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + export LANG=en_US.UTF-8 + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + export CASS_DRIVER_NO_EXTENSIONS=true + export CASS_DRIVER_NO_CYTHON=true + pip3 install --exists-action w -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run + no_output_timeout: 5m + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + export LANG=en_US.UTF-8 + cd cassandra-dtest + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + + echo "***Collected DTests (with vnodes)***" + set -eo pipefail && ./run_dtests.py --use-vnodes --dtest-print-tests-only --skip-resource-intensive-tests --dtest-print-tests-output=/tmp/all_dtest_tests_with_vnodes + set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_with_vnodes > /tmp/split_dtest_tests_with_vnodes.txt + cat /tmp/split_dtest_tests_with_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_with_vnodes_final.txt + # cat /tmp/split_dtest_tests_with_vnodes.txt + cat /tmp/split_dtest_tests_with_vnodes_final.txt + - run: + name: Run dtests (with vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_with_vnodes_final.txt` + #Skip all syncing to disk to avoid performance issues in flaky CI environments + export CASSANDRA_SKIP_SYNC=true + set -o pipefail && cd ~/cassandra-dtest && pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=/tmp/results/dtests/pytest_result_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --skip-resource-intensive-tests --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_with_vnodes_logs + dtests-no-vnodes: + <<: *env_settings + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + docker: + - image: *docker_image + environment: + <<: *env_vars + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + export LANG=en_US.UTF-8 + git clone --single-branch --branch master --depth 1 git://github.com/apache/cassandra-dtest.git ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + export LANG=en_US.UTF-8 + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + export CASS_DRIVER_NO_EXTENSIONS=true + export CASS_DRIVER_NO_CYTHON=true + pip3 install --exists-action w -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run + no_output_timeout: 5m + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + export LANG=en_US.UTF-8 + cd cassandra-dtest + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + + echo "***Collected DTests (without vnodes)***" + ./run_dtests.py --dtest-print-tests-only --skip-resource-intensive-tests --dtest-print-tests-output=/tmp/all_dtest_tests_without_vnodes + set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_without_vnodes > /tmp/split_dtest_tests_without_vnodes.txt + cat /tmp/split_dtest_tests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_without_vnodes_final.txt + # cat /tmp/split_dtest_tests_without_vnodes.txt + cat /tmp/split_dtest_tests_without_vnodes_final.txt + - run: + name: Run dtests (without vnodes) + no_output_timeout: 15m + command: | + # for now require at least 50 circleci containers to run the dtests (with less resources the tests won't produce reliable results or will fail to complete) + if [ $CIRCLE_NODE_TOTAL -gt 0 ]; then + source ~/env/bin/activate + export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin + + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_without_vnodes_final.txt` + #Skip all syncing to disk to avoid performance issues in flaky CI environments + export CASSANDRA_SKIP_SYNC=true + set -o pipefail && cd ~/cassandra-dtest && pytest --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_novnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --skip-resource-intensive-tests --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout-novnodes.txt + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_no_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_no_vnodes_logs http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d8ae88f..a5bdb42 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Migrate dtests to use pytest and python3 (CASSANDRA-14134) * Allow storage port to be configurable per node (CASSANDRA-7544) * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182) * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/circle.yml ---------------------------------------------------------------------- diff --git a/circle.yml b/circle.yml deleted file mode 100644 index ae848aa..0000000 --- a/circle.yml +++ /dev/null @@ -1,18 +0,0 @@ -machine: - java: - version: 'oraclejdk8' - -test: - pre: - - sudo apt-get update; sudo apt-get install wamerican: - parallel: true - override: - - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test -Dtest.runners=1;; 1) ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac: - parallel: true - - post: - - mkdir -p $CIRCLE_TEST_REPORTS/junit/: - parallel: true - - case $CIRCLE_NODE_INDEX in [0123]) find ./build/test/output/ -iname "*.xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \; ;;esac: - parallel: true - http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java index 391baee..fa0f9f7 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java +++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java @@ -55,6 +55,7 @@ import org.apache.cassandra.io.util.Memory; import org.apache.cassandra.io.util.SafeMemory; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.SyncUtil; import org.apache.cassandra.utils.concurrent.Transactional; import org.apache.cassandra.utils.concurrent.Ref; @@ -413,7 +414,7 @@ public class CompressionMetadata out.writeLong(offsets.getLong(i * 8L)); out.flush(); - fos.getFD().sync(); + SyncUtil.sync(fos); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/src/java/org/apache/cassandra/io/util/FileUtils.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index 70595fc..b8be84f 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -19,6 +19,7 @@ package org.apache.cassandra.io.util; import java.io.*; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -29,12 +30,16 @@ import java.nio.file.attribute.FileStoreAttributeView; import java.text.DecimalFormat; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.apache.cassandra.utils.SyncUtil; import sun.nio.ch.DirectBuffer; import org.apache.cassandra.concurrent.ScheduledExecutors; @@ -557,14 +562,46 @@ public final class FileUtils write(file, Arrays.asList(lines), StandardOpenOption.TRUNCATE_EXISTING); } + /** + * Write lines to a file adding a newline to the end of each supplied line using the provided open options. + * + * If open option sync or dsync is provided this will not open the file with sync or dsync since it might end up syncing + * many times for a lot of lines. Instead it will write all the lines and sync once at the end. Since the file is + * never returned there is not much difference from the perspective of the caller. + * @param file + * @param lines + * @param options + */ public static void write(File file, List<String> lines, StandardOpenOption ... options) { - try + Set<StandardOpenOption> optionsSet = new HashSet<>(Arrays.asList(options)); + //Emulate the old FileSystemProvider.newOutputStream behavior for open options. + if (optionsSet.isEmpty()) + { + optionsSet.add(StandardOpenOption.CREATE); + optionsSet.add(StandardOpenOption.TRUNCATE_EXISTING); + } + boolean sync = optionsSet.remove(StandardOpenOption.SYNC); + boolean dsync = optionsSet.remove(StandardOpenOption.DSYNC); + optionsSet.add(StandardOpenOption.WRITE); + + Path filePath = file.toPath(); + try (FileChannel fc = filePath.getFileSystem().provider().newFileChannel(filePath, optionsSet); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(Channels.newOutputStream(fc), CHARSET.newEncoder()))) { - Files.write(file.toPath(), - lines, - CHARSET, - options); + for (CharSequence line: lines) { + writer.append(line); + writer.newLine(); + } + + if (sync) + { + SyncUtil.force(fc, true); + } + else if (dsync) + { + SyncUtil.force(fc, false); + } } catch (IOException ex) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/e73bc32a/src/java/org/apache/cassandra/utils/SyncUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/SyncUtil.java b/src/java/org/apache/cassandra/utils/SyncUtil.java index 64d64cf..1917e8b 100644 --- a/src/java/org/apache/cassandra/utils/SyncUtil.java +++ b/src/java/org/apache/cassandra/utils/SyncUtil.java @@ -28,8 +28,11 @@ import java.nio.channels.FileChannel; import java.util.concurrent.atomic.AtomicInteger; import org.apache.cassandra.config.Config; +import org.apache.cassandra.service.CassandraDaemon; import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * A wrapper around various mechanisms for syncing files that makes it possible it intercept @@ -38,12 +41,14 @@ import com.google.common.base.Preconditions; */ public class SyncUtil { - public static boolean SKIP_SYNC = Boolean.getBoolean(Config.PROPERTY_PREFIX + "skip_sync"); + public static final boolean SKIP_SYNC; private static final Field mbbFDField; private static final Field fdClosedField; private static final Field fdUseCountField; + private static final Logger logger = LoggerFactory.getLogger(SyncUtil.class); + static { Field mbbFDFieldTemp = null; @@ -80,6 +85,15 @@ public class SyncUtil { } fdUseCountField = fdUseCountTemp; + + //If skipping syncing is requested by any means then skip them. + boolean skipSyncProperty = Boolean.getBoolean(Config.PROPERTY_PREFIX + "skip_sync"); + boolean skipSyncEnv = Boolean.valueOf(System.getenv().getOrDefault("CASSANDRA_SKIP_SYNC", "false")); + SKIP_SYNC = skipSyncProperty || skipSyncEnv; + if (SKIP_SYNC) + { + logger.info("Skip fsync enabled due to property {} and environment {}", skipSyncProperty, skipSyncEnv); + } } public static MappedByteBuffer force(MappedByteBuffer buf) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
