This is an automated email from the ASF dual-hosted git repository. mhanson pushed a commit to branch mass-test-run in repository https://gitbox.apache.org/repos/asf/geode.git
commit ec5465dc7d98b960623d630af835c59021f01382 Author: Robert Houghton <rhough...@pivotal.io> AuthorDate: Mon Jan 27 15:20:46 2020 -0800 Mass-test-run branch, no windows or publication jobs Co-authored-by: Robert Houghton <rhough...@pivotal.io> Co-authored-by: Mark Hanson <mhans...@pivotal.io> --- ci/pipelines/geode-build/jinja.template.yml | 310 +--------------------------- ci/pipelines/shared/jinja.variables.yml | 77 +------ 2 files changed, 9 insertions(+), 378 deletions(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index da6e399..f9dcbda 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -92,66 +92,6 @@ GRADLE_GLOBAL_ARGS: ((gradle-global-args)) {%- endfor -%} {% endmacro %} -groups: -- name: main - jobs: - - {{ build_test.name }} - {{- all_gating_jobs() | indent(2) }} - - UpdatePassingTokens - {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - - PublishArtifacts - {%- endif %} - {%- for flavor in (benchmarks.flavors) %} - - Benchmark{{flavor.title}} - {%- endfor %} -- name: complete - jobs: - - {{ build_test.name }} - {%- for test in (tests) if not test.name=="StressNew" -%} - {%- for java_test_version in (java_test_versions) %} - - {{test.name}}Test{{java_test_version.name}} - {%- endfor -%} - {%- endfor %} - - UpdatePassingTokens - {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - - PublishArtifacts - {%- endif %} - {%- for flavor in (benchmarks.flavors) %} - - Benchmark{{flavor.title}} - {%- endfor %} -- name: linux - jobs: - - {{ build_test.name }} - {%- for test in (tests) if test.PLATFORM=="linux" and not test.name=="StressNew" -%} - {% for java_test_version in (java_test_versions) %} - - {{test.name}}Test{{java_test_version.name}} - {%- endfor -%} - {%- endfor %} - {%- for flavor in (benchmarks.flavors) %} - - Benchmark{{flavor.title}} - {%- endfor %} -- name: windows - jobs: - - {{ build_test.name }} - {%- for test in (tests) if test.PLATFORM=="windows" -%} - {% for java_test_version in (java_test_versions) %} - - {{test.name}}Test{{java_test_version.name}} - {%- endfor -%} - {%- endfor %} -{%- for java_test_version in (java_test_versions) %} -- name: {{java_test_version.name}} - jobs: - - {{ build_test.name }} - {%- for test in (tests) if not test.name=="StressNew" %} - - {{test.name}}Test{{java_test_version.name}} - {%- endfor -%} -{%- endfor %} -- name: Semver Management - jobs: - {%- for semverPiece in ['major', 'minor', 'patch'] %} - - Bump{{ semverPiece.title() }} - {%- endfor %} - resources: - name: concourse-metadata-resource type: concourse-metadata-resource @@ -173,12 +113,7 @@ resources: paths: - ci/* {{ github_access() | indent(4) }} -- name: geode-benchmarks - type: git - source: - branch: {{benchmarks.benchmark_branch}} - depth: 1 - uri: https://github.com/apache/geode-benchmarks.git + - name: geode-build-version type: semver source: @@ -187,12 +122,7 @@ resources: initial_version: {{ metadata.initial_version }} json_key: ((!concourse-gcp-key)) key: semvers/((pipeline-prefix))((geode-build-branch))/version -- name: geode-passing-tokens - type: gcs-resource - source: - bucket: ((artifact-bucket)) - json_key: ((concourse-gcp-key)) - versioned_file: semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json + - name: alpine-tools-image type: docker-image source: @@ -201,12 +131,6 @@ resources: repository: gcr.io/((gcp-project))/((pipeline-prefix))alpine-tools tag: latest -- name: windows-builder-image-family - type: gci - source: - key: ((concourse-gcp-key)) - family_project: ((gcp-project)) - family: ((pipeline-prefix))windows-geode-builder - name: linux-builder-image-family type: gci @@ -233,16 +157,8 @@ resource_types: source: repository: frodenas/gcs-resource jobs: -{% for semverPiece in ['major', 'minor', 'patch'] -%} -- name: Bump{{ semverPiece.title() }} - serial: true - plan: - - get: geode-build-version - params: { bump: {{ semverPiece }} } - - put: geode-build-version - params: - file: geode-build-version/number -{% endfor %} + + - name: {{build_test.name}} public: true max_in_flight: {{build_test.MAX_IN_FLIGHT}} @@ -388,224 +304,6 @@ jobs: - name: instance-data timeout: 1h -- name: UpdatePassingTokens - public: true - serial: true - plan: - - aggregate: - - get: alpine-tools-image - - get: geode - passed: &update-token-passed-anchor -{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - - PublishArtifacts -{% else %} - {% for flavor in (benchmarks.flavors) %} - - Benchmark{{flavor.title}} - {% endfor %} -{% endif %} - trigger: true - - get: geode-build-version - trigger: true - passed: *update-token-passed-anchor - - task: couple-sha-and-build-id - image: alpine-tools-image - config: - platform: linux - inputs: - - name: geode - - name: geode-build-version - outputs: - - name: geode-passing-tokens - run: - path: bash - args: - - -cx - - | - pushd geode - GEODE_SHA=$(git rev-parse HEAD) - popd - GEODE_SEMVER=$(cat geode-build-version/number) - - GS_PATH=gs://((artifact-bucket))/semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json - CURRENT_PASSING_SHA=$(gsutil cat ${GS_PATH} | jq -r .ref) - set -e - # Check that the incoming GEODE_SHA is a descendent of the currently stored value. - # Keeps us from winding back the repository in the case of an out-of-order pipeline pass - if [ -z "${CURRENT_PASSING_SHA}" ] || (cd geode; git merge-base --is-ancestor ${CURRENT_PASSING_SHA} ${GEODE_SHA}); then - cat > geode-passing-tokens/passing-build-tokens.json <<JSON - { - "ref": "${GEODE_SHA}", - "semver": "${GEODE_SEMVER}" - } - JSON - fi - - aggregate: - - put: geode-passing-tokens - params: - file: geode-passing-tokens/passing-build-tokens.json - -{% for run_var in (benchmarks.flavors) %} -- name: Benchmark{{ run_var.title }} - public: true - max_in_flight: 3 - plan: - - get: geode-ci - passed: - {{ all_gating_jobs() | indent(4) }} - - get: alpine-tools-image - - aggregate: - - get: geode - passed: - {{ all_gating_jobs() | indent(6) }} - trigger: true - - get: geode-benchmarks - - get: geode-build-version - trigger: true - passed: - {{ all_gating_jobs() | indent(6) }} - - put: concourse-metadata-resource - - do: - - task: run_benchmarks{{ run_var.title }} - image: alpine-tools-image - config: - platform: linux - params: - AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id)) - AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key)) - AWS_DEFAULT_REGION: us-west-2 - AWS_REGION: us-west-2 - ARTIFACT_BUCKET: ((artifact-bucket)) - BENCHMARKS_BRANCH: {{benchmarks.benchmark_branch}} - BASELINE_BRANCH: {{benchmarks.baseline_branch}} - BASELINE_VERSION: {{benchmarks.baseline_version}} - FLAGS: {{ run_var.flag }} - TAG_POSTFIX: {{ run_var.title }} - TEST_OPTIONS: {{ run_var.options }} - run: - path: geode-ci/ci/scripts/run_benchmarks.sh - inputs: - - name: geode - - name: geode-ci - - name: geode-benchmarks - - name: concourse-metadata-resource - outputs: - - name: results - timeout: 8h - ensure: - do: - - task: cleanup_benchmarks - image: alpine-tools-image - config: - platform: linux - params: - AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id)) - AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key)) - AWS_DEFAULT_REGION: us-west-2 - AWS_REGION: us-west-2 - ARTIFACT_BUCKET: ((artifact-bucket)) - BASELINE_BRANCH: {{benchmarks.baseline_branch}} - BASELINE_VERSION: {{benchmarks.baseline_version}} - FLAGS: {{ run_var.flag }} - TAG_POSTFIX: {{ run_var.title }} - TEST_OPTIONS: {{ run_var.options }} - run: - path: geode-ci/ci/scripts/cleanup_benchmarks.sh - inputs: - - name: geode - - name: geode-ci - - name: geode-benchmarks - - name: concourse-metadata-resource - - name: results -{% endfor %} -{% if repository.upstream_fork != "apache" or repository.branch == "develop" %} -- name: PublishArtifacts - public: true - plan: - - aggregate: - - get: geode-ci - passed: &publish-passed-inputs - {% for flavor in (benchmarks.flavors) %} - - Benchmark{{flavor.title}} - {% endfor %} - - get: alpine-tools-image - - get: geode - passed: *publish-passed-inputs - trigger: true - - get: linux-builder-image-family - - aggregate: - - get: geode-build-version - trigger: true - passed: *publish-passed-inputs - - put: concourse-metadata-resource - {{ init_retry()|indent(4) }} - - task: create_instance - image: alpine-tools-image - config: - platform: linux - params: - {{ common_instance_params(publish_artifacts) | indent(8) }} - GEODE_BRANCH: {{repository.branch}} - GEODE_FORK: {{repository.fork}} - JAVA_BUILD_VERSION: {{ java_build_version.version }} - IMAGE_FAMILY_NAME: ((pipeline-prefix))linux-geode-builder - run: - path: geode-ci/ci/scripts/create_instance.sh - inputs: - - name: concourse-metadata-resource - - name: geode-ci - - name: geode - - name: attempts-log - path: old - outputs: - - name: instance-data - - name: attempts-log - path: new - timeout: 20m - attempts: 5 - - task: rsync_code_up - image: alpine-tools-image - config: - platform: linux - run: - path: geode-ci/ci/scripts/rsync_code_up.sh - inputs: - - name: geode-ci - - name: geode - - name: instance-data - timeout: 5m - - task: publish - image: alpine-tools-image - config: - platform: linux - params: - MAINTENANCE_VERSION: ((geode-build-branch)) - ARTIFACT_BUCKET: ((artifact-bucket)) - SERVICE_ACCOUNT: ((!concourse-gcp-account)) - JAVA_BUILD_VERSION: {{ java_build_version.version }} - MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket)) - GRADLE_GLOBAL_ARGS: ((gradle-global-args)) - run: - path: geode-ci/ci/scripts/execute_publish.sh - inputs: - - name: instance-data - - name: geode - - name: geode-ci - - name: geode-build-version - ensure: - do: - - task: delete_instance - image: alpine-tools-image - config: - platform: linux - run: - path: geode-ci/ci/scripts/delete_instance.sh - inputs: - - name: geode-ci - - name: instance-data - timeout: 1h - -{% endif %} - {%- for test in tests if not test.name=="StressNew" %} {%- set parameters = {} %} {%- do deep_merge(parameters, test) %} diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml index a82cdca..3931254 100644 --- a/ci/pipelines/shared/jinja.variables.yml +++ b/ci/pipelines/shared/jinja.variables.yml @@ -84,7 +84,7 @@ tests: DUNIT_PARALLEL_FORKS: '0' EXECUTE_TEST_TIMEOUT: 30m GRADLE_TASK: test - MAX_IN_FLIGHT: 1 + MAX_IN_FLIGHT: 5 PARALLEL_DUNIT: 'false' PARALLEL_GRADLE: 'true' PLATFORM: linux @@ -96,7 +96,7 @@ tests: DUNIT_PARALLEL_FORKS: '0' EXECUTE_TEST_TIMEOUT: 1h30m GRADLE_TASK: acceptanceTest - MAX_IN_FLIGHT: 2 + MAX_IN_FLIGHT: 5 PARALLEL_DUNIT: 'false' PARALLEL_GRADLE: 'false' PLATFORM: linux @@ -108,7 +108,7 @@ tests: DUNIT_PARALLEL_FORKS: '24' EXECUTE_TEST_TIMEOUT: 3h00m GRADLE_TASK: distributedTest - MAX_IN_FLIGHT: 3 + MAX_IN_FLIGHT: 5 PARALLEL_DUNIT: 'true' PLATFORM: linux RAM: '250' @@ -119,7 +119,7 @@ tests: DUNIT_PARALLEL_FORKS: '48' EXECUTE_TEST_TIMEOUT: 40m GRADLE_TASK: integrationTest - MAX_IN_FLIGHT: 1 + MAX_IN_FLIGHT: 5 PARALLEL_DUNIT: 'true' PLATFORM: linux RAM: '90' @@ -130,75 +130,8 @@ tests: DUNIT_PARALLEL_FORKS: '48' EXECUTE_TEST_TIMEOUT: 1h GRADLE_TASK: upgradeTest - MAX_IN_FLIGHT: 2 + MAX_IN_FLIGHT: 5 PARALLEL_DUNIT: 'true' PLATFORM: linux RAM: '210' name: Upgrade -- ARTIFACT_SLUG: stressnewtestfiles - CALL_STACK_TIMEOUT: '20700' - CPUS: '96' - DUNIT_PARALLEL_FORKS: '24' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: repeatTest - PARALLEL_DUNIT: 'true' - PARALLEL_GRADLE: 'false' - PLATFORM: linux - RAM: '280' - name: StressNew -- ARTIFACT_SLUG: windows-acceptancetestfiles - CPUS: '16' - DUNIT_PARALLEL_FORKS: '0' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: :geode-assembly:acceptanceTest - MAX_IN_FLIGHT: 3 - PARALLEL_DUNIT: 'false' - PARALLEL_GRADLE: 'false' - PLATFORM: windows - RAM: '64' - name: WindowsAcceptance -- ARTIFACT_SLUG: windows-gfshdistributedtest - CPUS: '16' - DUNIT_PARALLEL_FORKS: '0' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: distributedTest - GRADLE_TASK_OPTIONS: -PtestCategory=org.apache.geode.test.junit.categories.GfshTest - MAX_IN_FLIGHT: 5 - PARALLEL_DUNIT: 'false' - PARALLEL_GRADLE: 'false' - PLATFORM: windows - RAM: '64' - name: WindowsGfshDistributed -- ARTIFACT_SLUG: windows-integrationtestfiles - CPUS: '16' - DUNIT_PARALLEL_FORKS: '0' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: integrationTest - GRADLE_TASK_OPTIONS: -x geode-core:integrationTest - MAX_IN_FLIGHT: 2 - PARALLEL_DUNIT: 'false' - PARALLEL_GRADLE: 'false' - PLATFORM: windows - RAM: '64' - name: WindowsIntegration -- ARTIFACT_SLUG: windows-coreintegrationtestfiles - CPUS: '16' - DUNIT_PARALLEL_FORKS: '0' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: geode-core:integrationTest - MAX_IN_FLIGHT: 6 - PARALLEL_DUNIT: 'false' - PARALLEL_GRADLE: 'false' - PLATFORM: windows - RAM: '64' - name: WindowsCoreIntegration -- ARTIFACT_SLUG: windows-unittestfiles - CPUS: '16' - DUNIT_PARALLEL_FORKS: '0' - EXECUTE_TEST_TIMEOUT: 6h - GRADLE_TASK: test - MAX_IN_FLIGHT: 1 - PARALLEL_DUNIT: 'false' - PLATFORM: windows - RAM: '64' - name: WindowsUnit