This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4820 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 910f08b0edd8abdd36deb0dedcfe80811e1d9166 Author: tallison <[email protected]> AuthorDate: Fri Aug 14 08:23:55 2026 -0400 TIKA-4820 -- refactor ci to improve wall clock and environment --- .github/workflows/main-jdk17-build.yml | 69 ++++++++++++++++++++-- .github/workflows/main-jdk17-locale-build.yml | 65 ++++++++++++++++++++ .../main-jdk17-windows-build-multi-locale.yml | 51 ---------------- .github/workflows/main-jdk17-windows-build.yml | 25 +++++--- .github/workflows/split-packages.yml | 6 ++ 5 files changed, 154 insertions(+), 62 deletions(-) diff --git a/.github/workflows/main-jdk17-build.yml b/.github/workflows/main-jdk17-build.yml index f002368c6f..5f1a0e1568 100644 --- a/.github/workflows/main-jdk17-build.yml +++ b/.github/workflows/main-jdk17-build.yml @@ -27,6 +27,26 @@ on: paths-ignore: - 'docs/**' +# Superseded PR runs are dead work: cancel them. Never cancel a main build -- +# that run is the record for a commit that is already merged. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # Single source of truth for the build/integration-tests shard split: the + # `integration-tests` job runs exactly these modules and the `build` job + # excludes exactly these modules, both derived from this one list. + # + # Do NOT split this into two hand-maintained lists. The dangerous direction is + # silent: a module excluded from `build` but missing from the shard runs + # nowhere, and CI stays green while the tests stop existing. + # + # These six are ~15 of the build's 37 minutes -- five are testcontainers-backed + # (Elasticsearch/OpenSearch/Solr/Kafka/MinIO) and tika-pipes-integration-tests + # forks JVMs. Adding a slow IT module here is the intended way to rebalance. + IT_MODULES: ':tika-pipes-integration-tests,:tika-pipes-es-integration-tests,:tika-pipes-opensearch-integration-tests,:tika-pipes-solr-integration-tests,:tika-pipes-kafka-integration-tests,:tika-pipes-s3-integration-tests' + jobs: build: runs-on: ubuntu-latest @@ -45,14 +65,54 @@ jobs: cache: 'maven' - name: Install external tools run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl + # Everything except $IT_MODULES, which the integration-tests job owns. + # sed turns each ':artifactId' into the '!:artifactId' exclusion form. - name: Build with Maven - run: mvn clean apache-rat:check test install javadoc:aggregate -Pci -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + run: | + mvn clean apache-rat:check test install javadoc:aggregate -Pci \ + -pl "$(echo "$IT_MODULES" | sed 's/:/!:/g')" \ + -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + + integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 45 + # No `needs: build`, same reasoning as e2e-tests below: this job installs the + # reactor itself, so gating on build would only serialize it. + strategy: + matrix: + java: [ '17' ] + + steps: + - uses: actions/checkout@v6 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + cache: 'maven' + - name: Install external tools + run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl + # Full reactor for the same reason as the e2e job: these modules depend on + # the tika-pipes plugin zips, and a zip-type dependency never matches a + # jar-packaging reactor module, so -am cannot supply them. -Pfast keeps it + # to ~3 min; the tests themselves run in the next step. + - name: Install all modules (produces the plugin zips) + run: mvn clean install -Pfast -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + # apache-rat:check here too -- the build job no longer sees these modules, + # so without this they would drop out of license checking entirely. + - name: Run integration tests + run: | + mvn clean apache-rat:check test -Pci \ + -pl "$IT_MODULES" \ + -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" e2e-tests: runs-on: ubuntu-latest # full-reactor install added; 30 was tight for it timeout-minutes: 45 - needs: build + # No `needs: build` on purpose: this job checks out and installs the reactor + # itself, so it consumes nothing from `build`. Gating on it only serialized + # ~5 min onto the critical path. strategy: matrix: java: [ '17' ] @@ -69,9 +129,10 @@ jobs: # zip-type dependency never matches a jar-packaging reactor module, so -am treats every # plugin zip as external and skips building those modules entirely. install (not verify) # because the assembly sets attach=false (TIKA-4723) and an install-phase install-file is - # what puts each zip in the local repo. Tests are skipped here; the build job owns them. + # what puts each zip in the local repo. -Pfast (skipTests + rat/checkstyle/spotless + # off) because the build job owns all of that; this job only needs the artifacts. - name: Install all modules (produces the plugin zips) - run: mvn clean install -DskipTests -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + run: mvn clean install -Pfast -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # -pl must name the leaf modules: tika-e2e-tests is an aggregator pom, and Maven does # not pull in a selected aggregator's children, so this job built two pom-only modules # and ran zero tests while reporting green. diff --git a/.github/workflows/main-jdk17-locale-build.yml b/.github/workflows/main-jdk17-locale-build.yml new file mode 100644 index 0000000000..e4d5533eb1 --- /dev/null +++ b/.github/workflows/main-jdk17-locale-build.yml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# tr_TR is the highest-value locale to test in Java: dotless-i means +# "TIFF".toLowerCase() is not "tiff" unless the call passes Locale.ROOT. +# +# Linux, not Windows: locale bugs are JVM-level, so the 2x-cost Windows runner +# buys nothing here -- main-jdk17-windows-build covers the OS-specific surface. +# +# Locale via -Duser.language/-Duser.country, not LANG/LC_ALL: the JVM silently +# falls back to en_US when the named locale is not generated on the runner, so +# the env-var form can pass while testing nothing. +# +# push-only, like the jdk21/jdk25 builds: locale regressions are rare and not +# usually PR-specific, so a full reactor build per PR is not worth the cost. +name: main jdk17 locale build (tr_TR) + +on: + push: + branches: [ main ] + paths-ignore: + - 'docs/**' + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + java: [ '17' ] + + steps: + - uses: actions/checkout@v6 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + cache: 'maven' + - name: Install external tools + run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl + # The Docker-backed integration tests spin up Elasticsearch/OpenSearch/Solr/Kafka/MinIO + # for ~7.5 min and carry no locale signal, so they are excluded here; the main jdk17 + # build runs them. If a new testcontainers module appears, add it to this list -- + # forgetting only makes this job slower, it does not weaken it. + - name: Build with Maven (tr_TR locale) + run: | + mvn clean test install -Pci \ + -pl '!:tika-pipes-es-integration-tests,!:tika-pipes-kafka-integration-tests,!:tika-pipes-opensearch-integration-tests,!:tika-pipes-s3-integration-tests,!:tika-pipes-solr-integration-tests' \ + -Duser.language=tr -Duser.country=TR \ + -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" diff --git a/.github/workflows/main-jdk17-windows-build-multi-locale.yml b/.github/workflows/main-jdk17-windows-build-multi-locale.yml deleted file mode 100644 index c407d6c93d..0000000000 --- a/.github/workflows/main-jdk17-windows-build-multi-locale.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: main jdk17 windows build (multi-locale) - -on: - pull_request: - branches: [ main ] - paths-ignore: - - 'docs/**' - push: - branches: [ main ] - paths-ignore: - - 'docs/**' - -jobs: - build: - runs-on: windows-latest - timeout-minutes: 60 - strategy: - matrix: - java: [ '17' ] - locale: [ 'tr_TR.UTF-8', 'de_DE.UTF-8' ] - env: - LANG: ${{ matrix.locale }} - LC_ALL: ${{ matrix.locale }} - - steps: - - uses: actions/checkout@v6 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - cache: 'maven' - - name: Build with Maven - run: mvn clean test install javadoc:aggregate -Pci -Pe2e -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" diff --git a/.github/workflows/main-jdk17-windows-build.yml b/.github/workflows/main-jdk17-windows-build.yml index e3864eaa9c..3530b0572b 100644 --- a/.github/workflows/main-jdk17-windows-build.yml +++ b/.github/workflows/main-jdk17-windows-build.yml @@ -15,13 +15,23 @@ # limitations under the License. # -name: main jdk17 windows build +# The one Windows job. Two things it uniquely covers: +# - path handling: the checkout dir below deliberately contains a space +# - an alternate (non-en_US) locale +# +# Locale is set with -Duser.language/-Duser.country, NOT LANG/LC_ALL. Those env +# vars are POSIX-only: the Windows JVM reads the OS locale via Win32 and ignores +# them, and even on Linux the JVM silently falls back to en_US when the named +# locale is not generated on the box. -D always applies. +# +# push-only, like the jdk21/jdk25 and tr_TR builds. This is the most expensive +# job in CI -- a full reactor on a 2x-cost runner, ~48 min -- and it gated PR +# wall clock all by itself while every Linux job finished in ~22. Windows-only +# regressions are real but rare, so they are caught on main within the hour +# rather than paid for on every PR push. +name: main jdk17 windows build (de_DE) on: - pull_request: - branches: [ main ] - paths-ignore: - - 'docs/**' push: branches: [ main ] paths-ignore: @@ -36,6 +46,7 @@ jobs: java: [ '17' ] steps: + # The space in this path is deliberate -- it is what catches path-quoting bugs. - uses: actions/checkout@v6 with: path: 'tika build dir' @@ -45,6 +56,6 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.java }} cache: 'maven' - - name: Build with Maven + - name: Build with Maven (de_DE locale) working-directory: 'tika build dir' - run: mvn clean test install javadoc:aggregate -Pci -Pe2e -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + run: mvn clean test install javadoc:aggregate -Pci -Pe2e -Duser.language=de -Duser.country=DE -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" diff --git a/.github/workflows/split-packages.yml b/.github/workflows/split-packages.yml index 77d2cd7d18..2790a97e8d 100644 --- a/.github/workflows/split-packages.yml +++ b/.github/workflows/split-packages.yml @@ -23,6 +23,12 @@ on: push: branches: [ main ] +# Superseded PR runs are dead work: cancel them. Never cancel a main build -- +# that run is the record for a commit that is already merged. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check-split-packages: runs-on: ubuntu-latest
