This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 1570840 SUBMARINE-404. Fixed fasterxml package conflict
1570840 is described below
commit 1570840a421370789f5fdd90c15f47ed68ea9d6c
Author: Xun Liu <liu...@apache.org>
AuthorDate: Thu Mar 5 19:53:13 2020 +0800
SUBMARINE-404. Fixed fasterxml package conflict
### What is this PR for?
When submitting a job through REST, an exception was found, and after
inspection, it was found that it was caused by a `com.fasterxml.jackson`
package conflict.
The avatica package contains the `com.fasterxml.jackson` class, which
conflicts with the package in submarine-server-core, so it needs to be
shielded. If you need to use this package later, you can rename it through
relocation.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-404
### How should this be tested?
* https://travis-ci.org/liuxunorg/submarine/builds/659136191
### Screenshots (if appropriate)
```
Caused by: java.lang.NoSuchMethodError:
com.fasterxml.jackson.databind.introspect.AnnotatedMethod.getParameterType(I)Lcom/fasterxml/jackson/databind/JavaType;
at
com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector._fullDeserializationType(JaxbAnnotationIntrospector.java:1627)
at
com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector._findContentAdapter(JaxbAnnotationIntrospector.java:1591)
at
com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector.findDeserializationContentConverter(JaxbAnnotationIntrospector.java:1267)
at
com.fasterxml.jackson.databind.introspect.AnnotationIntrospectorPair.findDeserializationContentConverter(AnnotationIntrospectorPair.java:583)
at
com.fasterxml.jackson.databind.deser.std.StdDeserializer.findConvertingContentDeserializer(StdDeserializer.java:996)
at
com.fasterxml.jackson.databind.deser.std.MapDeserializer.createContextual(MapDeserializer.java:240)
at
com.fasterxml.jackson.databind.DeserializationContext.handleSecondaryContextualization(DeserializationContext.java:669)
at
com.fasterxml.jackson.databind.DeserializationContext.findContextualValueDeserializer(DeserializationContext.java:430)
at
com.fasterxml.jackson.databind.deser.std.StdDeserializer.findDeserializer(StdDeserializer.java:947)
at
com.fasterxml.jackson.databind.deser.BeanDeserializerBase.resolve(BeanDeserializerBase.java:439)
at
com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCache2(DeserializerCache.java:296)
at
com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCacheValueDeserializer(DeserializerCache.java:244)
at
com.fasterxml.jackson.databind.deser.DeserializerCache.findValueDeserializer(DeserializerCache.java:142)
at
com.fasterxml.jackson.databind.DeserializationContext.findContextualValueDeserializer(DeserializationContext.java:428)
at
com.fasterxml.jackson.databind.deser.std.StdDeserializer.findDeserializer(StdDeserializer.java:947)
at
com.fasterxml.jackson.databind.deser.BeanDeserializerBase.resolve(BeanDeserializerBase.java:439)
at
com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCache2(DeserializerCache.java:296)
at
com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCacheValueDeserializer(DeserializerCache.java:244)
at
com.fasterxml.jackson.databind.deser.DeserializerCache.findValueDeserializer(DeserializerCache.java:142)
at
com.fasterxml.jackson.databind.DeserializationContext.findRootValueDeserializer(DeserializationContext.java:461)
at
com.fasterxml.jackson.databind.ObjectReader._prefetchRootDeserializer(ObjectReader.java:1791)
at
com.fasterxml.jackson.databind.ObjectReader.forType(ObjectReader.java:626)
at
org.glassfish.jersey.jackson.internal.jackson.jaxrs.base.ProviderBase.readFrom(ProviderBase.java:826)
at
org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$TerminalReaderInterceptor.invokeReadFrom(ReaderInterceptorExecutor.java:257)
at
org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$TerminalReaderInterceptor.aroundReadFrom(ReaderInterceptorExecutor.java:236)
at
org.glassfish.jersey.message.internal.ReaderInterceptorExecutor.proceed(ReaderInterceptorExecutor.java:156)
at
org.glassfish.jersey.server.internal.MappableExceptionWrapperInterceptor.aroundReadFrom(MappableExceptionWrapperInterceptor.java:73)
at
org.glassfish.jersey.message.internal.ReaderInterceptorExecutor.proceed(ReaderInterceptorExecutor.java:156)
at
org.glassfish.jersey.message.internal.MessageBodyFactory.readFrom(MessageBodyFactory.java:1091)
at
org.glassfish.jersey.message.internal.InboundMessageContext.readEntity(InboundMessageContext.java:874)
at
org.glassfish.jersey.server.ContainerRequest.readEntity(ContainerRequest.java:271)
at
org.glassfish.jersey.server.internal.inject.EntityParamValueParamProvider$EntityValueSupplier.apply(EntityParamValueParamProvider.java:97)
at
org.glassfish.jersey.server.internal.inject.EntityParamValueParamProvider$EntityValueSupplier.apply(EntityParamValueParamProvider.java:80)
at
org.glassfish.jersey.server.spi.internal.ParamValueFactoryWithSource.apply(ParamValueFactoryWithSource.java:74)
at
org.glassfish.jersey.server.spi.internal.ParameterValueHelper.getParameterValues(ParameterValueHelper.java:92)
at
org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$AbstractMethodParamInvoker.getParamValues(JavaResourceMethodDispatcherProvider.java:133)
at
org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:200)
at
org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:103)
at
org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:493)
at
org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:415)
at
org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:104)
at
org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:277)
... 39 more
```
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Xun Liu <liu...@apache.org>
Closes #204 from liuxunorg/SUBMARINE-404 and squashes the following commits:
d7db635 [Xun Liu] SUBMARINE-404. Fixed fasterxml package conflict
---
.travis.yml | 8 +-
dev-support/docker-images/operator/Dockerfile | 7 +-
dev-support/docker-images/submarine/Dockerfile | 5 +
pom.xml | 23 ++-
submarine-cloud/hack/integration-test.sh | 2 +-
.../submarine-cluster/submarine-server.yaml | 2 +-
submarine-commons/commons-cluster/pom.xml | 14 +-
submarine-commons/commons-metastore/pom.xml | 17 +-
submarine-commons/commons-rpc/pom.xml | 2 +-
submarine-commons/commons-runtime/pom.xml | 2 +-
submarine-commons/commons-utils/pom.xml | 2 +-
submarine-server/pom.xml | 1 -
submarine-server/server-core/pom.xml | 7 +
.../server/AbstractSubmarineServerTest.java | 2 +
.../server-submitter/submitter-yarn/pom.xml | 9 +
submarine-test/test-k8s/pom.xml | 39 ++++-
.../rest/AbstractSubmarineServerTest.java | 181 ---------------------
.../rest/{ClusterIT.java => ClusterRestApiIT.java} | 10 +-
.../apache/submarine/rest/JobManagerRestApiIT.java | 66 ++++++++
.../test-k8s/src/test/resources/tf-mnist-req.json | 31 ++++
.../interpreter/python-interpreter/pom.xml | 1 -
.../interpreter/spark-interpreter/pom.xml | 1 -
22 files changed, 218 insertions(+), 214 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index f6f7b4b..698ce07 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,6 +48,7 @@ env:
- EXCLUDE_INTERPRETER="" # Template disable by SUBMARINE-381
"!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter
- EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
- EXCLUDE_COMMON_RPC="!submarine-commons/commons-rpc"
+ -
EXCLUDE_COMMONS="${EXCLUDE_COMMON_RPC},!submarine-commons/commons-utils,!submarine-commons/commons-cluster,!submarine-commons/commons-runtime,!submarine-commons/commons-metastore"
- EXCLUDE_CLOUD="!org.apache.submarine:submarine-cloud"
- MOZ_HEADLESS=1
@@ -174,7 +175,8 @@ matrix:
jdk: openjdk8
addons:
chrome: stable
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install
-DskipTests" TEST_FLAG="verify -DskipRat -am" MODULES="" TEST_MODULES="-pl
submarine-test/test-k8s" TEST_PROJECTS=""
+ # Because submarine on k8s uses port 80, it needs to be set
`-Durl=http://127.0.0.1`
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install
-DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1"; MODULES=""
TEST_MODULES="-pl
${EXCLUDE_COMMONS},!org.apache.submarine:submarine-client,!org.apache.submarine:submitter-yarn,!org.apache.submarine:server-api,!org.apache.submarine:server-rpc,!org.apache.submarine:server-core,submarine-test/test-k8s"
TEST_PROJECTS=""
# Template disable by SUBMARINE-381
#- name: Test submarine interpreter
@@ -227,7 +229,7 @@ matrix:
install:
- mvn --version
- - echo "mvn $BUILD_FLAG $MODULES $PROFILE -B"
+ - echo "install-> mvn $BUILD_FLAG $MODULES $PROFILE -B"
- mvn $BUILD_FLAG $MODULES $PROFILE -B
before_script:
@@ -236,7 +238,7 @@ before_script:
script:
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
- - echo "mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
+ - echo "script-> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
- mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS
after_success:
diff --git a/dev-support/docker-images/operator/Dockerfile
b/dev-support/docker-images/operator/Dockerfile
index 4a5b14d..d1a0080 100644
--- a/dev-support/docker-images/operator/Dockerfile
+++ b/dev-support/docker-images/operator/Dockerfile
@@ -16,9 +16,10 @@
FROM alpine:3.10
MAINTAINER Apache Software Foundation <dev@submarine.apache.org>
-# Update apk repositories
-RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/main"; > /etc/apk/repositories
-RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/community"; >>
/etc/apk/repositories
+# If you are in China, enabling the following two lines of code can speed up
the build of the image, but it may cause failure in travis.
+# So when submitting the code, please make sure the following 2 lines of code
are in a commented state
+# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/main"; >
/etc/apk/repositories
+# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/community"; >>
/etc/apk/repositories
RUN apk add tzdata --no-cache
ADD ./tmp/submarine-operator /usr/local/bin/submarine-operator
diff --git a/dev-support/docker-images/submarine/Dockerfile
b/dev-support/docker-images/submarine/Dockerfile
index d26c392..792080a 100644
--- a/dev-support/docker-images/submarine/Dockerfile
+++ b/dev-support/docker-images/submarine/Dockerfile
@@ -16,6 +16,11 @@
FROM alpine:3.10
MAINTAINER Apache Software Foundation <dev@submarine.apache.org>
+# If you are in China, enabling the following two lines of code can speed up
the build of the image, but it may cause failure in travis.
+# So when submitting the code, please make sure the following 2 lines of code
are in a commented state
+# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/main"; >
/etc/apk/repositories
+# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/community"; >>
/etc/apk/repositories
+
# INSTALL openjdk
RUN apk update && \
apk add --no-cache openjdk8 tzdata bash tini && \
diff --git a/pom.xml b/pom.xml
index 6770a99..a5269e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,7 +72,7 @@
<javax.ws.rsapi.version>2.1</javax.ws.rsapi.version>
<pagehelper.version>5.1.10</pagehelper.version>
- <gson.version>2.8.1</gson.version>
+ <gson.version>2.8.5</gson.version>
<jackson-databind.version>2.9.10</jackson-databind.version>
<jackson-annotations.version>2.9.10</jackson-annotations.version>
<jackson-module-jaxb-annotations.version>2.9.10</jackson-module-jaxb-annotations.version>
@@ -158,6 +158,11 @@
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
<!-- Unit Tests -->
<dependency>
@@ -314,6 +319,21 @@
<artifactId>grpc-stub</artifactId>
<version>${grpc.verison}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons.logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -400,7 +420,6 @@
</dependencies>
</profile>
-
<profile>
<id>rat</id>
<activation>
diff --git a/submarine-cloud/hack/integration-test.sh
b/submarine-cloud/hack/integration-test.sh
index b9925f5..d4da608 100755
--- a/submarine-cloud/hack/integration-test.sh
+++ b/submarine-cloud/hack/integration-test.sh
@@ -63,7 +63,7 @@ function start() {
# $KUBECTL_BIN get pods | grep submarinecluster-submarine | awk '{print $1}'
| xargs -I {} $KUBECTL_BIN exec {} -- bash -c "tail -500
/opt/submarine-current/logs/submarine.log"
# kubectl get pods -n operations | grep operations | awk '{print $1}' |
xargs -I {} kubectl exec -it -n operations {} cat /tmp/operations-server.INFO
############################################################
- echo "Stop submarine on k8s failure!"
+ echo "Start submarine on k8s failure!"
}
function stop() {
diff --git a/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
b/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
index bf3dbb1..b6cf9a0 100644
--- a/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
+++ b/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
@@ -36,7 +36,7 @@ spec:
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
- name: submarine
+ name: submarine
- containerPort: 8081
name: cluster
env:
diff --git a/submarine-commons/commons-cluster/pom.xml
b/submarine-commons/commons-cluster/pom.xml
index eea8bc9..dcd6abd 100644
--- a/submarine-commons/commons-cluster/pom.xml
+++ b/submarine-commons/commons-cluster/pom.xml
@@ -94,6 +94,10 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -112,7 +116,11 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
</dependency>
<!-- Test libraries -->
@@ -132,7 +140,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -145,7 +153,7 @@
<goal>shade</goal>
</goals>
<configuration>
-
<outputFile>target/submarine-${artifactId}-${project.version}-shade.jar</outputFile>
+
<outputFile>target/submarine-${project.artifactId}-${project.version}-shade.jar</outputFile>
<filters>
<filter>
<artifact>*:*</artifact>
diff --git a/submarine-commons/commons-metastore/pom.xml
b/submarine-commons/commons-metastore/pom.xml
index 78ded92..9534180 100644
--- a/submarine-commons/commons-metastore/pom.xml
+++ b/submarine-commons/commons-metastore/pom.xml
@@ -134,6 +134,10 @@
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -276,6 +280,12 @@
</dependency>
<dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
@@ -301,7 +311,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -315,6 +325,7 @@
</goals>
<configuration>
<outputFile>target/submarine-${project.artifactId}-${project.version}-shade.jar</outputFile>
+ <minimizeJar>true</minimizeJar>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -329,6 +340,10 @@
<excludes>
<exclude>org.slf4j:*</exclude>
<exclude>log4j:log4j</exclude>
+ <exclude>org.apache.submarine:commons-utils:*</exclude>
+ <!-- The avatica package contains the com.fasterxml.jackson
class, which conflicts with the package in submarine-server-core,
+ so it needs to be shielded. If you need to use this package
later, you can rename it through relocation. -->
+ <exclude>org.apache.calcite.avatica:avatica:*</exclude>
</excludes>
</artifactSet>
<relocations>
diff --git a/submarine-commons/commons-rpc/pom.xml
b/submarine-commons/commons-rpc/pom.xml
index e043278..fffa799 100644
--- a/submarine-commons/commons-rpc/pom.xml
+++ b/submarine-commons/commons-rpc/pom.xml
@@ -68,7 +68,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
diff --git a/submarine-commons/commons-runtime/pom.xml
b/submarine-commons/commons-runtime/pom.xml
index 1a672c6..d66774e 100644
--- a/submarine-commons/commons-runtime/pom.xml
+++ b/submarine-commons/commons-runtime/pom.xml
@@ -194,7 +194,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/submarine-commons/commons-utils/pom.xml
b/submarine-commons/commons-utils/pom.xml
index e0030a9..1d98acf 100644
--- a/submarine-commons/commons-utils/pom.xml
+++ b/submarine-commons/commons-utils/pom.xml
@@ -73,7 +73,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/submarine-server/pom.xml b/submarine-server/pom.xml
index 1728d1b..f590319 100644
--- a/submarine-server/pom.xml
+++ b/submarine-server/pom.xml
@@ -100,7 +100,6 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>${gson.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/submarine-server/server-core/pom.xml
b/submarine-server/server-core/pom.xml
index 0add62c..4cd27b3 100644
--- a/submarine-server/server-core/pom.xml
+++ b/submarine-server/server-core/pom.xml
@@ -439,6 +439,13 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${plugin.maven.jar.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
<configuration>
<excludes>
<!-- Because SubmarineConfiguration class will load the
submarine-site.xml file from the jar package,
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
index 4b414ad..5741278 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
@@ -62,6 +62,7 @@ public abstract class AbstractSubmarineServerTest {
if (System.getProperty("websocketUrl") != null) {
websocketUrl = System.getProperty("websocketurl");
}
+ LOG.debug("getWebsocketApiUrlToTest = {}", websocketUrl);
return websocketUrl;
}
@@ -70,6 +71,7 @@ public abstract class AbstractSubmarineServerTest {
if (System.getProperty("url") != null) {
url = System.getProperty("url");
}
+ LOG.debug("getUrlToTest = {}", url);
return url;
}
diff --git a/submarine-server/server-submitter/submitter-yarn/pom.xml
b/submarine-server/server-submitter/submitter-yarn/pom.xml
index e92d357..d1b90e8 100644
--- a/submarine-server/server-submitter/submitter-yarn/pom.xml
+++ b/submarine-server/server-submitter/submitter-yarn/pom.xml
@@ -239,9 +239,18 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
diff --git a/submarine-test/test-k8s/pom.xml b/submarine-test/test-k8s/pom.xml
index f9bac95..f7a4159 100644
--- a/submarine-test/test-k8s/pom.xml
+++ b/submarine-test/test-k8s/pom.xml
@@ -38,6 +38,21 @@
</properties>
<dependencies>
<dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>server-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>server-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
@@ -47,23 +62,19 @@
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<scope>test</scope>
@@ -73,6 +84,16 @@
<artifactId>javax.ws.rs-api</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/AbstractSubmarineServerTest.java
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/AbstractSubmarineServerTest.java
deleted file mode 100644
index b4aa70d..0000000
---
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/AbstractSubmarineServerTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.submarine.rest;
-
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
-import org.apache.commons.httpclient.methods.DeleteMethod;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.commons.httpclient.methods.RequestEntity;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.util.regex.Pattern;
-
-public abstract class AbstractSubmarineServerTest {
- protected static final Logger LOG =
LoggerFactory.getLogger(AbstractSubmarineServerTest.class);
-
- static final String URL = getUrlToTest();
-
- public static String getUrlToTest() {
- // The submarine server deployed in k8s uses the default port of 8080 in
submarine-site.xml.
- // However, port 80 is exposed through k8s Ingress.
- String url = "http://localhost:80";;
- if (System.getProperty("url") != null) {
- url = System.getProperty("url");
- }
- return url;
- }
-
- protected static PostMethod httpPost(String path, String body) throws
IOException {
- return httpPost(path, body, StringUtils.EMPTY, StringUtils.EMPTY);
- }
-
- protected static PostMethod httpPost(String path, String request, String
user, String pwd)
- throws IOException {
- LOG.info("Connecting to {}", URL + path);
-
- HttpClient httpClient = new HttpClient();
- PostMethod postMethod = new PostMethod(URL + path);
- postMethod.setRequestBody(request);
- postMethod.setRequestHeader("Content-type", MediaType.APPLICATION_JSON);
- postMethod.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
-
- if (userAndPasswordAreNotBlank(user, pwd)) {
- postMethod.setRequestHeader("Cookie", "JSESSIONID=" + getCookie(user,
pwd));
- }
-
- httpClient.executeMethod(postMethod);
-
- LOG.info("{} - {}", postMethod.getStatusCode(),
postMethod.getStatusText());
-
- return postMethod;
- }
-
- protected static PutMethod httpPut(String path, String body) throws
IOException {
- return httpPut(path, body, StringUtils.EMPTY, StringUtils.EMPTY);
- }
-
- protected static PutMethod httpPut(String path, String body, String user,
String pwd) throws IOException {
- LOG.info("Connecting to {}", URL + path);
- HttpClient httpClient = new HttpClient();
- PutMethod putMethod = new PutMethod(URL + path);
- putMethod.addRequestHeader("Origin", URL);
- putMethod.setRequestHeader("Content-type", "application/yaml");
- RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8"));
- putMethod.setRequestEntity(entity);
- if (userAndPasswordAreNotBlank(user, pwd)) {
- putMethod.setRequestHeader("Cookie", "JSESSIONID=" + getCookie(user,
pwd));
- }
- httpClient.executeMethod(putMethod);
- LOG.info("{} - {}", putMethod.getStatusCode(), putMethod.getStatusText());
- return putMethod;
- }
-
- protected static DeleteMethod httpDelete(String path) throws IOException {
- return httpDelete(path, StringUtils.EMPTY, StringUtils.EMPTY);
- }
-
- protected static DeleteMethod httpDelete(String path, String user, String
pwd) throws IOException {
- LOG.info("Connecting to {}", URL + path);
- HttpClient httpClient = new HttpClient();
- DeleteMethod deleteMethod = new DeleteMethod(URL + path);
- deleteMethod.addRequestHeader("Origin", URL);
- if (userAndPasswordAreNotBlank(user, pwd)) {
- deleteMethod.setRequestHeader("Cookie", "JSESSIONID=" + getCookie(user,
pwd));
- }
- httpClient.executeMethod(deleteMethod);
- LOG.info("{} - {}", deleteMethod.getStatusCode(),
deleteMethod.getStatusText());
- return deleteMethod;
- }
-
- protected static GetMethod httpGet(String path) throws IOException {
- return httpGet(path, "", "");
- }
-
- protected static GetMethod httpGet(String path, String user, String pwd)
throws IOException {
- return httpGet(path, user, pwd, "");
- }
-
- protected static GetMethod httpGet(String path, String user, String pwd,
String cookies)
- throws IOException {
- LOG.info("Connecting to {}", URL + path);
- HttpClient httpClient = new HttpClient();
- GetMethod getMethod = new GetMethod(URL + path);
- getMethod.addRequestHeader("Origin", URL);
- httpClient.executeMethod(getMethod);
- LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText());
- return getMethod;
- }
-
- protected static boolean checkIfServerIsRunning() {
- GetMethod request = null;
- boolean isRunning = false;
- try {
- request = httpGet("/");
- isRunning = request.getStatusCode() == 200;
- } catch (IOException e) {
- LOG.warn("AbstractTestRestApi.checkIfServerIsRunning() fails .. " +
- "Submarine server is not running");
- isRunning = false;
- } finally {
- if (request != null) {
- request.releaseConnection();
- }
- }
- return isRunning;
- }
-
- private static String getCookie(String user, String password) throws
IOException {
- HttpClient httpClient = new HttpClient();
- PostMethod postMethod = new PostMethod(URL + "/login");
- postMethod.addRequestHeader("Origin", URL);
- postMethod.setParameter("password", password);
- postMethod.setParameter("userName", user);
- httpClient.executeMethod(postMethod);
- LOG.info("{} - {}", postMethod.getStatusCode(),
postMethod.getStatusText());
- Pattern pattern = Pattern.compile("JSESSIONID=([a-zA-Z0-9-]*)");
- Header[] setCookieHeaders = postMethod.getResponseHeaders("Set-Cookie");
- String jsessionId = null;
- for (Header setCookie : setCookieHeaders) {
- java.util.regex.Matcher matcher = pattern.matcher(setCookie.toString());
- if (matcher.find()) {
- jsessionId = matcher.group(1);
- }
- }
-
- if (jsessionId != null) {
- return jsessionId;
- } else {
- return StringUtils.EMPTY;
- }
- }
-
- protected static boolean userAndPasswordAreNotBlank(String user, String pwd)
{
- if (StringUtils.isBlank(user) && StringUtils.isBlank(pwd)) {
- return false;
- }
- return true;
- }
-}
diff --git
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterIT.java
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterRestApiIT.java
similarity index 88%
rename from
submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterIT.java
rename to
submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterRestApiIT.java
index b3798d4..7348a7d 100644
---
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterIT.java
+++
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ClusterRestApiIT.java
@@ -18,14 +18,16 @@
package org.apache.submarine.rest;
import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.submarine.server.AbstractSubmarineServerTest;
+
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
+import org.junit.BeforeClass;
+import org.junit.Test;
-public class ClusterIT extends AbstractSubmarineServerTest {
- public final static Logger LOG = LoggerFactory.getLogger(ClusterIT.class);
+public class ClusterRestApiIT extends AbstractSubmarineServerTest {
+ public final static Logger LOG =
LoggerFactory.getLogger(ClusterRestApiIT.class);
@BeforeClass
public static void startUp(){
diff --git
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
new file mode 100644
index 0000000..ee0972c
--- /dev/null
+++
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.rest;
+
+import com.google.gson.Gson;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.io.FileUtils;
+import org.apache.submarine.server.AbstractSubmarineServerTest;
+import org.apache.submarine.server.response.JsonResponse;
+import org.apache.submarine.server.rest.RestConstants;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.BeforeClass;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+
+public class JobManagerRestApiIT extends AbstractSubmarineServerTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(JobManagerRestApiIT.class);
+
+ @BeforeClass
+ public static void startUp(){
+ Assert.assertTrue(checkIfServerIsRunning());
+ }
+
+ // Test job created with correct JSON input
+ @Test
+ public void testCreateJob() throws Exception {
+ URL fileUrl = this.getClass().getResource("/tf-mnist-req.json");
+ String jobSpec = FileUtils.readFileToString(new File(fileUrl.toURI()),
StandardCharsets.UTF_8);
+
+ PostMethod response = httpPost("/api/" + RestConstants.V1 + "/" +
RestConstants.JOBS, jobSpec);
+ LOG.debug(response.toString());
+
+ String responseBodyAsString = response.getResponseBodyAsString();
+ LOG.debug(responseBodyAsString);
+
+ Gson gson = new Gson();
+ JsonResponse jsonResponse = gson.fromJson(responseBodyAsString,
JsonResponse.class);
+ assertEquals("Response code should be 200 ",
+ Response.Status.ACCEPTED.getStatusCode(), jsonResponse.getCode());
+ }
+}
diff --git a/submarine-test/test-k8s/src/test/resources/tf-mnist-req.json
b/submarine-test/test-k8s/src/test/resources/tf-mnist-req.json
new file mode 100644
index 0000000..0850e54
--- /dev/null
+++ b/submarine-test/test-k8s/src/test/resources/tf-mnist-req.json
@@ -0,0 +1,31 @@
+{
+ "name": "mnist",
+ "librarySpec": {
+ "name": "TensorFlow",
+ "version": "2.1.0",
+ "image": "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0",
+ "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150",
+ "envVars": {
+ "ENV_1": "ENV1"
+ }
+ },
+ "submitterSpec": {
+ "type": "k8s",
+ "configPath": null,
+ "namespace": "submarine",
+ "kind": "TFJob",
+ "apiVersion": "kubeflow.org/v1"
+ },
+ "taskSpecs": {
+ "Ps": {
+ "name": "tensorflow",
+ "replicas": 2,
+ "resources": "cpu=4,memory=2048M,nvidia.com/gpu=1"
+ },
+ "Worker": {
+ "name": "tensorflow",
+ "replicas": 2,
+ "resources": "cpu=4,memory=2048M,nvidia.com/gpu=1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/submarine-workbench/interpreter/python-interpreter/pom.xml
b/submarine-workbench/interpreter/python-interpreter/pom.xml
index a6ba493..e954ed9 100644
--- a/submarine-workbench/interpreter/python-interpreter/pom.xml
+++ b/submarine-workbench/interpreter/python-interpreter/pom.xml
@@ -177,7 +177,6 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/submarine-workbench/interpreter/spark-interpreter/pom.xml
b/submarine-workbench/interpreter/spark-interpreter/pom.xml
index d0839e5..807e190 100644
--- a/submarine-workbench/interpreter/spark-interpreter/pom.xml
+++ b/submarine-workbench/interpreter/spark-interpreter/pom.xml
@@ -135,7 +135,6 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@submarine.apache.org
For additional commands, e-mail: dev-h...@submarine.apache.org
