This is an automated email from the ASF dual-hosted git repository. joewitt pushed a commit to branch support/nifi-1.11.x in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 046691840420f952c705aee22f58d5a47f65d291 Author: Joe Witt <[email protected]> AuthorDate: Thu Feb 20 10:55:34 2020 -0500 NIFI-7175 removed appveyor and travis builds which have become unreliable. Added Github Actions based CI build for pull requests and pushes. Removed unit tests which make unreadable or unaccesible dirs. Includes windows build as well as ubuntu. we can do localization based builds later as this is all proven stable. Signed-off-by: Pierre Villard <[email protected]> This closes #4058. --- .github/workflows/ci-workflow.yml | 111 ++++++++++++++++++ .travis-output-filters | 15 --- .travis.sh | 28 ----- .travis.yml | 119 ------------------- README.md | 2 +- appveyor.yml | 38 ------ .../nifi/processors/standard/TestGetFile.java | 127 ++------------------- pom.xml | 2 +- 8 files changed, 122 insertions(+), 320 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml new file mode 100644 index 0000000..54993c6 --- /dev/null +++ b/.github/workflows/ci-workflow.yml @@ -0,0 +1,111 @@ +name: ci-workflow + +on: [push, pull_request] + +jobs: + ubuntu-build: + + timeout-minutes: 90 + runs-on: ubuntu-latest + strategy: + matrix: + java: [ '1.8', '11' ] + name: Ubuntu Build NiFi - JDK${{ matrix.java }} + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Check NPM Cache + uses: actions/[email protected] + with: + path: ~/.npm + key: linux-${{ matrix.java }}-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + linux-${{ matrix.java }}-npm- + - name: Check Maven Com Cache + uses: actions/[email protected] + with: + path: ~/.m2/repository/com + key: linux-${{ matrix.java }}-maven-com-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-${{ matrix.java }}-maven-com- + - name: Check Maven Org Cache + uses: actions/[email protected] + with: + path: ~/.m2/repository/org + key: linux-${{ matrix.java }}-maven-org-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-${{ matrix.java }}-maven-org- + - name: Check Maven Net Cache + uses: actions/[email protected] + with: + path: ~/.m2/repository/net + key: linux-${{ matrix.java }}-maven-net-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-${{ matrix.java }}-maven-net- + - name: Check Maven IO Cache + uses: actions/[email protected] + with: + path: ~/.m2/repository/io + key: linux-${{ matrix.java }}-maven-io-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-${{ matrix.java }}-maven-io- + - name: Set up JDK ${{ matrix.java }} + uses: actions/[email protected] + with: + java-version: ${{ matrix.java }} + - name: Build with Maven + env: + MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN + run: | + mvn -version + mvn -T 1C install -B -Pcontrib-check,include-grpc -Ddir-only -ntp -ff + rm -rf ~/.m2/repository/org/apache/nifi + + windows-build: + + runs-on: windows-latest + name: Windows Build NiFi - JDK 1.8 + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Check Maven Com Cache + uses: actions/[email protected] + with: + path: ~\.m2\repository\com + key: windows-maven-com-${{ hashFiles('**\pom.xml') }} + restore-keys: | + windows-maven-com- + - name: Check Maven Org Cache + uses: actions/[email protected] + with: + path: ~\.m2\repository\org + key: windows-maven-org-${{ hashFiles('**\pom.xml') }} + restore-keys: | + windows-maven-org- + - name: Check Maven Net Cache + uses: actions/[email protected] + with: + path: ~\.m2\repository\net + key: windows-maven-net-${{ hashFiles('**\pom.xml') }} + restore-keys: | + windows-maven-net- + - name: Check Maven IO Cache + uses: actions/[email protected] + with: + path: ~\.m2\repository\io + key: windows-maven-io-${{ hashFiles('**\pom.xml') }} + restore-keys: | + windows-maven-io- + - name: Set up JDK 1.8 + uses: actions/[email protected] + with: + java-version: 1.8 + - name: Build with Maven + env: + MAVEN_OPTS: -Xmx2g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN + run: | + mvn -version + mvn -T 1C install -B -Ddir-only -ntp -ff -DskipTests + Remove-Item -path '~\.m2\repository\org\apache\nifi' -recurse -force \ No newline at end of file diff --git a/.travis-output-filters b/.travis-output-filters deleted file mode 100644 index f401281..0000000 --- a/.travis-output-filters +++ /dev/null @@ -1,15 +0,0 @@ -Downloading from -Copying -Progress -Resolving deltas: -[WARNING] ***************************************************************** -[WARNING] * Your build is requesting parallel execution, but project * -[WARNING] * contains the following plugin(s) that have goals not marked * -[WARNING] * as @threadSafe to support parallel building. * -[WARNING] * While this /may/ work fine, please look for plugin updates * -[WARNING] * and/or request plugins be made thread-safe. * -[WARNING] * If reporting an issue, report it against the plugin in * -[WARNING] * question, not against maven-core * -[WARNING] The following plugins are not marked @threadSafe in -[WARNING] org.apache.rat:apache-rat-plugin -[WARNING] Enable debug to see more precisely which goals are not marked @threadSafe. diff --git a/.travis.sh b/.travis.sh deleted file mode 100755 index adeae67..0000000 --- a/.travis.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -# Replace variables seems to be the only option to pass proper values to surefire -# Note: The reason the sed is done as part of script is to ensure the pom hack -# won't affect the 'clean install' above -set -x -if [ "$USER_LANGUAGE" != "default" ]; then - sed -i.bak -e "s|<maven\.surefire\.arguments/>|<maven\.surefire\.arguments>-Duser.language=${USER_LANGUAGE} -Duser.region=${USER_REGION}</maven\.surefire\.arguments>|" pom.xml - diff pom.xml pom.xml.bak - rm pom.xml.bak -fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a5fd764..0000000 --- a/.travis.yml +++ /dev/null @@ -1,119 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -language: java - -matrix: - include: - - stage: "Cache" - name: "Cache Dependencies - no explicit jdk specified" - script: >- - if [ ! -d "$HOME"/.m2/repository ] || [ ! "$(ls -A "$HOME"/.m2/repository)" ] ; then - echo "Cached Maven repository does not exist, downloading dependencies..." - mvn -V -T 1C -pl !./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-nar,!./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors dependency:go-offline -Dsilent -ntp --fail-never \ - | grep -v -F -f .travis-output-filters -e "Failed to execute goal on project" - else - echo "Cached Maven repository exists, skipping dependency cache" - fi - - stage: "Cache" - name: "Cache Dependencies - openjdk8 & openjdk11" - jdk: - - openjdk8 - - openjdk11 - script: >- - if [ ! -d "$HOME"/.m2/repository ] || [ ! "$(ls -A "$HOME"/.m2/repository)" ] ; then - echo "Cached Maven repository does not exist, downloading dependencies..." - mvn -V -T 1C -pl !./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-nar,!./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors dependency:go-offline -Dsilent -ntp --fail-never \ - | grep -v -F -f .travis-output-filters -e "Failed to execute goal on project" - else - echo "Cached Maven repository exists, skipping dependency cache" - fi - - # For the build stages, use -pl to exclude the following from the build to reduce build time: - # non-api nar modules - # assembly modules - # nifi-maven-archetype directories that have "__" in the name - - stage: "Build" - name: "Build Java 8 EN" - jdk: openjdk8 - script: >- - mvn clean install -V -T 1C - -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"` - -Pcontrib-check,include-grpc -Ddir-only - -Dmaven.surefire.arguments="-Duser.language=en -Duser.region=US" - | grep -v -F -f .travis-output-filters - && test ${PIPESTATUS[0]} -eq 0 - - stage: "Build" - name: "Build Java 8 FR" - jdk: openjdk8 - script: >- - mvn clean install -V -T 1C - -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"` - -Pcontrib-check,include-grpc -Ddir-only - -Dmaven.surefire.arguments="-Duser.language=fr -Duser.region=FR" - | grep -v -F -f .travis-output-filters - && test ${PIPESTATUS[0]} -eq 0 - - stage: "Build" - name: "Build Java 8 JA" - jdk: openjdk8 - script: >- - mvn clean install -V -T 1C - -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"` - -Pcontrib-check,include-grpc -Ddir-only - -Dmaven.surefire.arguments="-Duser.language=ja -Duser.region=JP" - | grep -v -F -f .travis-output-filters - && test ${PIPESTATUS[0]} -eq 0 - - stage: "Build" - name: "Build Java 11 EN" - jdk: openjdk11 - script: >- - mvn clean install -V -T 1C - -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"` - -Pcontrib-check,include-grpc -Ddir-only - -Dmaven.surefire.arguments="-Duser.language=en -Duser.region=US" - | grep -v -F -f .travis-output-filters - && test ${PIPESTATUS[0]} -eq 0 - -os: - - linux - -# Caches mvn repository in order to speed up builds -cache: - directories: - - $HOME/.m2 - - $HOME/.npm - -before_cache: - # Remove nifi repo again to save travis from caching it - - rm -rf $HOME/.m2/repository/org/apache/nifi/ - -# before_install aids in a couple workarounds for issues within the Travis-CI environment -# 1. Workaround for buffer overflow issues with OpenJDK versions of java as per https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165135711 -# 2. Workaround for non-existent Maven repository as per: https://github.com/travis-ci/travis-ci/issues/4629#issuecomment-131214465 -before_install: - - sudo hostname "$(hostname | cut -c1-63)" - - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts - - sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml - - echo "MAVEN_OPTS='-Xmx2g -XX:+UseG1GC'" > ~/.mavenrc - - cat ~/.mavenrc - # Remove nifi repo again to save travis from caching it - - rm -rf $HOME/.m2/repository/org/apache/nifi/ - # Install Maven 3.6.1 - - wget -O /tmp/apache-maven-3.6.1-bin.tar.gz https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz - - tar xzf /tmp/apache-maven-3.6.1-bin.tar.gz -C /tmp - - export M2_HOME=/tmp/apache-maven-3.6.1 - - export PATH=$M2_HOME/bin:$PATH - -install: true diff --git a/README.md b/README.md index 410efbb..72ca973 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ --> [<img src="https://nifi.apache.org/assets/images/apache-nifi-logo.svg"; width="300" height="126" alt="Apache NiFi"/>][nifi] -[](https://travis-ci.org/apache/nifi) +[ [](https://hub.docker.com/r/apache/nifi/) [](https://nifi.apache.org/download.html) [](https://s.apache.org/nifi-community-slack) diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 3e31c19..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '1.0.0-SNAPSHOT-{build}' -os: Windows Server 2012 -environment: - JAVA_HOME: C:\Program Files\Java\jdk1.8.0 -install: - - ps: | - Add-Type -AssemblyName System.IO.Compression.FileSystem - if (!(Test-Path -Path "C:\maven" )) { - (new-object System.Net.WebClient).DownloadFile( - 'http://www.us.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.zip', - 'C:\maven-bin.zip' - ) - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven") - } - - cmd: SET M2_HOME=C:\maven\apache-maven-3.1.1 - - cmd: SET PATH=%M2_HOME%\bin;%JAVA_HOME%\bin;%PATH%% - - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g - - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g -build_script: - - mvn clean package --batch-mode -DskipTests -Ddir-only -cache: - - C:\maven\ - - C:\Users\appveyor\.m2 diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java index 7ffdfd8..bfa075a 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java @@ -16,114 +16,30 @@ */ package org.apache.nifi.processors.standard; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Test; import java.io.File; import java.io.IOException; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Set; -import org.apache.nifi.flowfile.attributes.CoreAttributes; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TestGetFile { @Test - public void testWithInaccessibleDir() throws IOException { - // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event - if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) { - return; - } - File inaccessibleDir = new File("target/inaccessible"); - inaccessibleDir.deleteOnExit(); - inaccessibleDir.mkdir(); - Set<PosixFilePermission> posixFilePermissions = new HashSet<>(); - Files.setPosixFilePermissions(inaccessibleDir.toPath(), posixFilePermissions); - final TestRunner runner = TestRunners.newTestRunner(new GetFile()); - runner.setProperty(GetFile.DIRECTORY, inaccessibleDir.getAbsolutePath()); - try { - runner.run(); - fail(); - } catch (AssertionError e) { - assertTrue(e.getCause().getMessage() - .endsWith("does not have sufficient permissions (i.e., not writable and readable)")); - } - } - - @Test - public void testWithUnreadableDir() throws IOException { - // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event - if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) { - return; - } - File unreadableDir = new File("target/unreadable"); - unreadableDir.deleteOnExit(); - unreadableDir.mkdir(); - Set<PosixFilePermission> posixFilePermissions = new HashSet<>(); - posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE); - posixFilePermissions.add(PosixFilePermission.GROUP_WRITE); - posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE); - posixFilePermissions.add(PosixFilePermission.OTHERS_WRITE); - posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE); - posixFilePermissions.add(PosixFilePermission.OWNER_WRITE); - Files.setPosixFilePermissions(unreadableDir.toPath(), posixFilePermissions); - final TestRunner runner = TestRunners.newTestRunner(new GetFile()); - runner.setProperty(GetFile.DIRECTORY, unreadableDir.getAbsolutePath()); - try { - runner.run(); - fail(); - } catch (AssertionError e) { - Throwable ex = e.getCause(); - assertTrue(e.getCause().getMessage() - .endsWith("does not have sufficient permissions (i.e., not writable and readable)")); - } - } - - @Test - public void testWithUnwritableDir() throws IOException { - // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event - if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) { - return; - } - File unwritableDir = new File("target/unwritable"); - unwritableDir.deleteOnExit(); - unwritableDir.mkdir(); - Set<PosixFilePermission> posixFilePermissions = new HashSet<>(); - posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE); - posixFilePermissions.add(PosixFilePermission.GROUP_READ); - posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE); - posixFilePermissions.add(PosixFilePermission.OTHERS_READ); - posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE); - posixFilePermissions.add(PosixFilePermission.OWNER_READ); - Files.setPosixFilePermissions(unwritableDir.toPath(), posixFilePermissions); - final TestRunner runner = TestRunners.newTestRunner(new GetFile()); - runner.setProperty(GetFile.DIRECTORY, unwritableDir.getAbsolutePath()); - try { - runner.run(); - fail(); - } catch (AssertionError e) { - assertTrue(e.getCause().getMessage() - .endsWith("does not have sufficient permissions (i.e., not writable and readable)")); - } - } - - @Test public void testFilePickedUp() throws IOException { final File directory = new File("target/test/data/in"); deleteDirectory(directory); @@ -237,29 +153,6 @@ public class TestGetFile { } catch (Exception doNothing) { } - boolean verifyPermissions = false; - try { - /* If you mount an NTFS partition in Linux, you are unable to change the permissions of the files, - * because every file has the same permissions, controlled by the 'fmask' and 'dmask' mount options. - * Executing a chmod command will not fail, but it does not change the file's permissions. - * From Java perspective the NTFS mount point, as a FileStore supports the 'unix' and 'posix' file - * attribute views, but the setPosixFilePermissions() has no effect. - * - * If you set verifyPermissions to true without the following extra check, the test case will fail - * on a file system, where Nifi source is located on a NTFS mount point in Linux. - * The purpose of the extra check is to ensure, that setPosixFilePermissions() changes the file's - * permissions, and set verifyPermissions, after we are convinced. - */ - Set<PosixFilePermission> perms = PosixFilePermissions.fromString("r--r-----"); - Files.setPosixFilePermissions(targetPath, perms); - Set<PosixFilePermission> permsAfterSet = Files.getPosixFilePermissions(targetPath); - if (perms.equals(permsAfterSet)) { - verifyPermissions = true; - } - - } catch (Exception doNothing) { - } - final TestRunner runner = TestRunners.newTestRunner(new GetFile()); runner.setProperty(GetFile.DIRECTORY, "target/test/data/in"); runner.run(); @@ -276,8 +169,6 @@ public class TestGetFile { fail(); } } - if (verifyPermissions) { - successFiles.get(0).assertAttributeEquals("file.permissions", "r--r-----"); - } + //permissions are not verified as these are very environmentally specific } } diff --git a/pom.xml b/pom.xml index 1fe9527..a19223c 100644 --- a/pom.xml +++ b/pom.xml @@ -691,7 +691,7 @@ <exclude>nbactions.xml</exclude> <!-- courtesy excludes for netbeans users --> <exclude>DEPENDENCIES</exclude> <!-- auto generated file by apache's maven config while building sources.zip --> <exclude>.github/PULL_REQUEST_TEMPLATE.md</exclude> <!-- PR Template for GitHub that does not have a mechanism of including comments --> - <exclude>.travis-output-filters</exclude> <!-- grep patterns to exclude from travis output --> + <exclude>.github/workflows/ci-workflow.yml</exclude> </excludes> </configuration> </plugin>
![https://nifi.apache.org/assets/images/apache-nifi-logo.svg"](https://nifi.apache.org/assets/images/apache-nifi-logo.svg"){kind=link}
