(TWILL-111) support MapR file system (including examples) Signed-off-by: Terence Yim <[email protected]>
add mapr repos Signed-off-by: Terence Yim <[email protected]> add mapr profile to travis Signed-off-by: Terence Yim <[email protected]> since mapr doesn't support minDFScluster, have no choice but to skip the related tests. Signed-off-by: Terence Yim <[email protected]> ignore deadlock test Signed-off-by: Terence Yim <[email protected]> Revert "ignore deadlock test" This reverts commit af2155bd5f82c32b873a810f005b6dec6bd0eada. Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/537fdb2d Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/537fdb2d Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/537fdb2d Branch: refs/heads/site Commit: 537fdb2dc9f595bf8caeb8476905f6845b51c99b Parents: dd67a7b Author: Tongqing Qiu <[email protected]> Authored: Thu Jan 15 13:53:28 2015 -0500 Committer: Terence Yim <[email protected]> Committed: Mon Jan 26 14:00:01 2015 -0800 ---------------------------------------------------------------------- .travis.yml | 1 + pom.xml | 56 ++++++++++++++++++++ twill-examples/pom.xml | 13 +++++ twill-yarn/pom.xml | 27 ++++++++++ .../org/apache/twill/internal/ServiceMain.java | 2 +- 5 files changed, 98 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/537fdb2d/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index b8a16bf..d5fe88b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,3 +38,4 @@ env: - PROFILE='hadoop-2.4' - PROFILE='hadoop-2.5' - PROFILE='cdh-4.4.0' + - PROFILE='mapr-hadoop-2.4' http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/537fdb2d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5eba751..cf71dfd 100644 --- a/pom.xml +++ b/pom.xml @@ -625,6 +625,62 @@ </build> </profile> <profile> + <id>mapr-hadoop-2.4</id> + <properties> + <hadoop.version>2.4.1-mapr-1408</hadoop.version> + <zookeeper.version>3.4.5-mapr-1406</zookeeper.version> + <mapr.fs.version>4.0.1-mapr</mapr.fs.version> + </properties> + <repositories> + <repository> + <id>mapr-releases</id> + <url>http://repository.mapr.com/maven/</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/hadoop21</source> + <source>src/main/hadoop22</source> + </sources> + </configuration> + </execution> + <execution> + <id>add-source-2.0</id> + <phase>prepare-package</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/hadoop20</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>hadoop-2.5</id> <properties> <hadoop.version>2.5.1</hadoop.version> http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/537fdb2d/twill-examples/pom.xml ---------------------------------------------------------------------- diff --git a/twill-examples/pom.xml b/twill-examples/pom.xml index 3fdaaa4..042fa9e 100644 --- a/twill-examples/pom.xml +++ b/twill-examples/pom.xml @@ -50,4 +50,17 @@ limitations under the License. </plugins> </build> + <profiles> + <profile> + <id>mapr-hadoop-2.4</id> + <dependencies> + <dependency> + <groupId>com.mapr.hadoop</groupId> + <artifactId>maprfs</artifactId> + <version>${mapr.fs.version}</version> + </dependency> + </dependencies> + </profile> + </profiles> + </project> http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/537fdb2d/twill-yarn/pom.xml ---------------------------------------------------------------------- diff --git a/twill-yarn/pom.xml b/twill-yarn/pom.xml index a732100..37b0db3 100644 --- a/twill-yarn/pom.xml +++ b/twill-yarn/pom.xml @@ -162,5 +162,32 @@ </resources> </build> </profile> + <profile> + <id>mapr-hadoop-2.4</id> + <dependencies> + <dependency> + <groupId>com.mapr.hadoop</groupId> + <artifactId>maprfs</artifactId> + <version>${mapr.fs.version}</version> + </dependency> + <dependency> + <groupId>com.mapr.fs</groupId> + <artifactId>libprotodefs</artifactId> + <version>${mapr.fs.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.14.1</version> + <configuration> + <skipTests>true</skipTests> + </configuration> + </plugin> + </plugins> + </build> + </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/537fdb2d/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java index ae86f42..786c78f 100644 --- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java +++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java @@ -118,7 +118,7 @@ public abstract class ServiceMain { return new LocalLocationFactory().create(appDir); } - if ("hdfs".equals(appDir.getScheme())) { + if ("hdfs".equals(appDir.getScheme()) || "maprfs".equals(appDir.getScheme())) { if (UserGroupInformation.isSecurityEnabled()) { return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir); }
