Repository: incubator-flink Updated Branches: refs/heads/master e8056afe7 -> 4ce973f61
Fixes on CDH4 build / Maven improvements: - remove usages of commons-lang (we use only commons-lang3, as defined in flink-runtime/pom.xml) - Upgrade maven failsafe to 2.17 so that tests also fail if the test errors (exceptions during execution) - added new build profile "docs-and-source" that creates javadocs and source attachments. These are usually only needed when releasing (--> speed up the build) Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/4ce973f6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/4ce973f6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/4ce973f6 Branch: refs/heads/master Commit: 4ce973f61688cb9bd10611caa5c4aa3469ee11e8 Parents: e8056af Author: Robert Metzger <[email protected]> Authored: Wed Aug 13 21:21:14 2014 +0200 Committer: Robert Metzger <[email protected]> Committed: Wed Aug 13 23:17:15 2014 +0200 ---------------------------------------------------------------------- flink-addons/flink-avro/pom.xml | 17 +++++ flink-addons/flink-spargel/pom.xml | 22 +++++++ flink-clients/pom.xml | 6 -- .../org/apache/flink/client/program/Client.java | 6 +- flink-quickstart/pom.xml | 68 -------------------- .../runtime/jobgraph/AbstractJobVertex.java | 2 +- .../operators/util/OutputEmitterTest.java | 29 ++++----- flink-test-utils/pom.xml | 5 ++ pom.xml | 68 +++++++++++--------- tools/deploy_to_maven.sh | 5 +- 10 files changed, 102 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-addons/flink-avro/pom.xml ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/pom.xml b/flink-addons/flink-avro/pom.xml index 36ad445..b95ca59 100644 --- a/flink-addons/flink-avro/pom.xml +++ b/flink-addons/flink-avro/pom.xml @@ -183,6 +183,23 @@ under the License. <maven.javadoc.skip>true</maven.javadoc.skip> </properties> </profile> + <profile> + <id>cdh4</id> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.cdh4.hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.cdh4.hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-addons/flink-spargel/pom.xml ---------------------------------------------------------------------- diff --git a/flink-addons/flink-spargel/pom.xml b/flink-addons/flink-spargel/pom.xml index ee39654..64d3877 100644 --- a/flink-addons/flink-spargel/pom.xml +++ b/flink-addons/flink-spargel/pom.xml @@ -57,4 +57,26 @@ under the License. <scope>test</scope> </dependency> </dependencies> + + <profiles> + + <profile> + <!-- Needed for the tests --> + <id>cdh4</id> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.cdh4.hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.cdh4.hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-clients/pom.xml ---------------------------------------------------------------------- diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml index b035c19..7456209 100644 --- a/flink-clients/pom.xml +++ b/flink-clients/pom.xml @@ -95,12 +95,6 @@ under the License. <version>2.4</version> <scope>compile</scope> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.4</version> - <scope>compile</scope> - </dependency> </dependencies> <!-- More information on this: http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-clients/src/main/java/org/apache/flink/client/program/Client.java ---------------------------------------------------------------------- diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java index 4485de1..9c71ef1 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java @@ -26,7 +26,7 @@ import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.List; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.flink.api.common.JobExecutionResult; @@ -174,8 +174,8 @@ public class Client { throw new ProgramInvocationException( "The program plan could not be fetched. The program silently swallowed the control flow exceptions.\n" - + "System.err: "+StringEscapeUtils.escapeHtml(baes.toString())+" \n" - + "System.out: "+StringEscapeUtils.escapeHtml(baos.toString())+" \n" ); + + "System.err: "+StringEscapeUtils.escapeHtml4(baes.toString())+" \n" + + "System.out: "+StringEscapeUtils.escapeHtml4(baos.toString())+" \n" ); } else { throw new RuntimeException(); http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-quickstart/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml index 4f5083a..de82453 100644 --- a/flink-quickstart/pom.xml +++ b/flink-quickstart/pom.xml @@ -64,72 +64,4 @@ under the License. </plugin> </plugins> </build> - - - <profiles> - <profile> - <id>release</id> - <build> - <plugins> - <!-- source attachment --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>2.2.1</version> - <executions> - <execution> - <id>attach-sources</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> - <!-- Javadocs --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <version>2.9.1</version> - <executions> - <execution> - <id>attach-javadocs</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> - <!-- signing --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-gpg-plugin</artifactId> - <version>1.4</version> - <executions> - <execution> - <id>sign-artifacts</id> - <phase>verify</phase> - <goals> - <goal>sign</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - <version>2.1</version> - <configuration> - <mavenExecutorId>forked-path</mavenExecutorId> - <useReleaseProfile>false</useReleaseProfile> - <arguments>${arguments} -Psonatype-oss-release</arguments> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - </profile> - </profiles> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java ---------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java index e4bcb4e..08a9567 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java @@ -22,7 +22,7 @@ package org.apache.flink.runtime.jobgraph; import java.io.IOException; import java.util.ArrayList; -import org.apache.commons.lang.Validate; +import org.apache.commons.lang3.Validate; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.core.io.StringRecord; http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java ---------------------------------------------------------------------- diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java index 94414c9..f109ca9 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java @@ -28,7 +28,6 @@ import java.io.PipedOutputStream; import junit.framework.TestCase; import org.junit.Assert; -import org.apache.commons.lang.NotImplementedException; import org.apache.flink.api.common.typeutils.TypeComparator; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.typeutils.runtime.record.RecordComparatorFactory; @@ -389,56 +388,56 @@ public class OutputEmitterTest extends TestCase { } @Override - public void setReference(Integer toCompare) { throw new NotImplementedException(); } + public void setReference(Integer toCompare) { throw new UnsupportedOperationException(); } @Override - public boolean equalToReference(Integer candidate) { throw new NotImplementedException(); } + public boolean equalToReference(Integer candidate) { throw new UnsupportedOperationException(); } @Override public int compareToReference( TypeComparator<Integer> referencedComparator) { - throw new NotImplementedException(); + throw new UnsupportedOperationException(); } @Override - public int compare(Integer first, Integer second) { throw new NotImplementedException(); } + public int compare(Integer first, Integer second) { throw new UnsupportedOperationException(); } @Override public int compare(DataInputView firstSource, DataInputView secondSource) { - throw new NotImplementedException(); + throw new UnsupportedOperationException(); } @Override - public boolean supportsNormalizedKey() { throw new NotImplementedException(); } + public boolean supportsNormalizedKey() { throw new UnsupportedOperationException(); } @Override - public boolean supportsSerializationWithKeyNormalization() { throw new NotImplementedException(); } + public boolean supportsSerializationWithKeyNormalization() { throw new UnsupportedOperationException(); } @Override - public int getNormalizeKeyLen() { throw new NotImplementedException(); } + public int getNormalizeKeyLen() { throw new UnsupportedOperationException(); } @Override - public boolean isNormalizedKeyPrefixOnly(int keyBytes) { throw new NotImplementedException(); } + public boolean isNormalizedKeyPrefixOnly(int keyBytes) { throw new UnsupportedOperationException(); } @Override public void putNormalizedKey(Integer record, MemorySegment target, int offset, int numBytes) { - throw new NotImplementedException(); + throw new UnsupportedOperationException(); } @Override public void writeWithKeyNormalization(Integer record, DataOutputView target) throws IOException { - throw new NotImplementedException(); + throw new UnsupportedOperationException(); } @Override public Integer readWithKeyDenormalization(Integer reuse, DataInputView source) throws IOException { - throw new NotImplementedException(); + throw new UnsupportedOperationException(); } @Override - public boolean invertNormalizedKey() { throw new NotImplementedException(); } + public boolean invertNormalizedKey() { throw new UnsupportedOperationException(); } @Override - public TypeComparator<Integer> duplicate() { throw new NotImplementedException(); } + public TypeComparator<Integer> duplicate() { throw new UnsupportedOperationException(); } } http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/flink-test-utils/pom.xml ---------------------------------------------------------------------- diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml index a9d21ed..328d9a4 100644 --- a/flink-test-utils/pom.xml +++ b/flink-test-utils/pom.xml @@ -75,6 +75,11 @@ under the License. <artifactId>hadoop-common</artifactId> <version>${hadoop.cdh4.hadoop.version}</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.cdh4.hadoop.version}</version> + </dependency> </dependencies> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a22c7e7..60f5e4b 100644 --- a/pom.xml +++ b/pom.xml @@ -234,6 +234,43 @@ under the License. </profile> <profile> + <!-- used for SNAPSHOT and regular releases --> + <id>docs-and-source</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>2.2.1</version><!--$NO-MVN-MAN-VER$--> + <executions> + <execution> + <id>attach-sources</id> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.9.1</version><!--$NO-MVN-MAN-VER$--> + <configuration> + <quiet>true</quiet> + </configuration> + <executions> + <execution> + <id>attach-javadocs</id> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>release</id> <build> <plugins> @@ -496,24 +533,8 @@ under the License. </configuration> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <version>2.9.1</version><!--$NO-MVN-MAN-VER$--> - <configuration> - <quiet>true</quiet> - </configuration> - <executions> - <execution> - <id>attach-javadocs</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <artifactId>maven-failsafe-plugin</artifactId> - <version>2.7</version><!--$NO-MVN-MAN-VER$--> + <version>2.17</version><!--$NO-MVN-MAN-VER$--> <executions> <execution> <goals> @@ -569,19 +590,6 @@ under the License. </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>2.2.1</version><!--$NO-MVN-MAN-VER$--> - <executions> - <execution> - <id>attach-sources</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> <!-- http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4ce973f6/tools/deploy_to_maven.sh ---------------------------------------------------------------------- diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh index d8a6b48..c56a2eb 100755 --- a/tools/deploy_to_maven.sh +++ b/tools/deploy_to_maven.sh @@ -72,7 +72,7 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then # Deploy regular hadoop v1 to maven - mvn -DskipTests -Drat.ignoreErrors=true deploy --settings deploysettings.xml; + mvn -Pdocs-and-source -DskipTests -Drat.ignoreErrors=true deploy --settings deploysettings.xml; fi if [[ $TRAVIS_JOB_NUMBER == *4 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then @@ -82,8 +82,7 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then # all these tweaks assume a yarn build. # performance tweaks here: no "clean deploy" so that actually nothing is being rebuild (could cause wrong poms inside the jars?) # skip tests (they were running already) - # skip javadocs generation (already generated) - mvn -B -f pom.hadoop2.xml -DskipTests -Drat.ignoreErrors=true -Dmaven.javadoc.skip=true deploy --settings deploysettings.xml; + mvn -B -f pom.hadoop2.xml -DskipTests -Pdocs-and-source -Drat.ignoreErrors=true deploy --settings deploysettings.xml; fi if [[ $TRAVIS_JOB_NUMBER == *5 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
