Ian Maxon has submitted this change and it was merged. Change subject: [ASTERIXDB-2512][TEST] Adds SpiderSilk Integration ......................................................................
[ASTERIXDB-2512][TEST] Adds SpiderSilk Integration - user model changes: no - storage format changes: no - interface changes: no details: - Adds minimal SpiderSilk integration by adding the asterix-spidersilk module, reuqired maven dependencies, sample Dockerfile for node's containers, simple config file for 3 node AsterixDB cluster, a TestUtility to make working with SpiderSilk easier and a sample test case that uses all of these together. Change-Id: Ifd64aad26bd5cd3b6040fa53e87ae0d0d0c0a562 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3053 Reviewed-by: Ian Maxon <[email protected]> Tested-by: Ian Maxon <[email protected]> --- M .gitignore A asterixdb/asterix-spidersilk/config/cc.conf A asterixdb/asterix-spidersilk/docker/Dockerfile A asterixdb/asterix-spidersilk/pom.xml A asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java A asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java A asterixdb/asterix-spidersilk/src/test/resources/logback.xml M asterixdb/pom.xml 8 files changed, 379 insertions(+), 0 deletions(-) Approvals: Ian Maxon: Looks good to me, approved; Verified diff --git a/.gitignore b/.gitignore index 569eb3d..1dcc3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ .m2* ß +.SpiderSilkWorkingDirectory diff --git a/asterixdb/asterix-spidersilk/config/cc.conf b/asterixdb/asterix-spidersilk/config/cc.conf new file mode 100644 index 0000000..3212003 --- /dev/null +++ b/asterixdb/asterix-spidersilk/config/cc.conf @@ -0,0 +1,35 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[nc/nc1] +txn.log.dir=/data/txnlog +core.dump.dir=/data/coredump +iodevices=/data +address=nc1 + +[nc/nc2] +txn.log.dir=/data/txnlog +core.dump.dir=/data/coredump +iodevices=/data +address=nc2 + +[nc] +app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint +command=asterixnc + +[cc] +address=cc \ No newline at end of file diff --git a/asterixdb/asterix-spidersilk/docker/Dockerfile b/asterixdb/asterix-spidersilk/docker/Dockerfile new file mode 100644 index 0000000..3f7f6bb --- /dev/null +++ b/asterixdb/asterix-spidersilk/docker/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM java:8-jre + +RUN apt update && apt install -y iptables \ No newline at end of file diff --git a/asterixdb/asterix-spidersilk/pom.xml b/asterixdb/asterix-spidersilk/pom.xml new file mode 100644 index 0000000..a6bb2af --- /dev/null +++ b/asterixdb/asterix-spidersilk/pom.xml @@ -0,0 +1,125 @@ +<!-- + ! Licensed to the Apache Software Foundation (ASF) under one + ! or more contributor license agreements. See the NOTICE file + ! distributed with this work for additional information + ! regarding copyright ownership. The ASF licenses this file + ! to you under the Apache License, Version 2.0 (the + ! "License"); you may not use this file except in compliance + ! with the License. You may obtain a copy of the License at + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, + ! software distributed under the License is distributed on an + ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ! KIND, either express or implied. See the License for the + ! specific language governing permissions and limitations + ! under the License. + !--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>asterix-spidersilk</artifactId> + <name>asterix-spidersilk</name> + + <parent> + <groupId>org.apache.asterix</groupId> + <artifactId>apache-asterixdb</artifactId> + <version>0.9.5-SNAPSHOT</version> + </parent> + + <licenses> + <license> + <name>Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + <comments>A business-friendly OSS license</comments> + </license> + </licenses> + + <properties> + <root.dir>${basedir}/..</root.dir> + </properties> + + <repositories> + <repository> + <id>snapshots-repo</id> + <url>https://oss.sonatype.org/content/repositories/snapshots</url> + <releases><enabled>false</enabled></releases> + <snapshots><enabled>true</enabled></snapshots> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>me.arminb.spidersilk</groupId> + <artifactId>spidersilk</artifactId> + <version>0.3.2-SNAPSHOT</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.25</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.9.7</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>1.2.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-app</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-test-framework</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-app</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-server</artifactId> + <version>${project.version}</version> + </dependency> + + </dependencies> + + <!-- While these dependencies are declared and being used, the maven dependency plugin detects them as declared and + not used. To make this right, it is needed to force these dependencies as used --> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <usedDependencies> + <usedDependency>ch.qos.logback:logback-classic</usedDependency> + <usedDependency>org.apache.asterix:asterix-app</usedDependency> + <usedDependency>org.apache.asterix:asterix-server</usedDependency> + </usedDependencies> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java new file mode 100644 index 0000000..49f35a0 --- /dev/null +++ b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.spidersilk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import me.arminb.spidersilk.SpiderSilkRunner; +import me.arminb.spidersilk.dsl.entities.Deployment; +import me.arminb.spidersilk.exceptions.RuntimeEngineException; +import me.arminb.spidersilk.execution.single_node.SingleNodeRuntimeEngine; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; + +import java.util.List; +import java.util.Map; + +public class SampleTestIT { + private static final Logger logger = LoggerFactory.getLogger(SampleTestIT.class); + + protected static SpiderSilkRunner runner; + + @BeforeClass + public static void before() throws RuntimeEngineException { + Deployment deployment = TestUtil.getSimpleClusterDeployment(); + runner = SpiderSilkRunner.run(deployment, new SingleNodeRuntimeEngine(deployment)); + TestUtil.waitForClusterToBeUp(runner); + logger.info("The cluster is UP!"); + } + + @AfterClass + public static void after() { + if (runner != null) { + runner.stop(); + } + } + + @Test + public void sampleTest() throws Exception { + + TestExecutor testExecutor = TestUtil.getTestExecutor(runner); + String ddl = "drop dataverse company if exists;" + "create dataverse company;" + "use company;" + + "create type Emp as open {" + " id : int32," + " name : string" + "};" + + "create dataset Employee(Emp) primary key id;"; + + String insertStatements = "use company;" + "insert into Employee({ \"id\":123,\"name\":\"John Doe\"});"; + + String query = "use company;" + "select value emp from Employee emp;"; + + testExecutor.executeSqlppUpdateOrDdl(ddl, TestCaseContext.OutputFormat.CLEAN_JSON); + logger.info("Company dataverse and employee dataset are created!"); + testExecutor.executeSqlppUpdateOrDdl(insertStatements, TestCaseContext.OutputFormat.CLEAN_JSON); + logger.info("A record is inserted into employee dataset"); + InputStream resultStream = testExecutor.executeSqlppUpdateOrDdl(query, TestCaseContext.OutputFormat.CLEAN_JSON); + + ObjectMapper objectMapper = new ObjectMapper(); + List<Map<String, String>> result = objectMapper.readValue(resultStream, List.class); + + Assert.assertEquals(1, result.size()); + Assert.assertEquals(123, result.get(0).get("id")); + Assert.assertEquals("John Doe", result.get(0).get("name")); + + logger.info("The fetched record matches the inserted record"); + } +} diff --git a/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java new file mode 100644 index 0000000..5d2cc02 --- /dev/null +++ b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.spidersilk; + +import me.arminb.spidersilk.SpiderSilkRunner; +import me.arminb.spidersilk.dsl.entities.Deployment; +import me.arminb.spidersilk.dsl.entities.PortType; +import me.arminb.spidersilk.dsl.entities.ServiceType; +import me.arminb.spidersilk.exceptions.RuntimeEngineException; +import org.apache.asterix.test.common.TestExecutor; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; + +public class TestUtil { + private static String mavenVersion; + private static String asterixHome; + + public static Deployment getSimpleClusterDeployment() { + mavenVersion = getMavenArtifactVersion(); + asterixHome = "/asterix/apache-asterixdb-" + mavenVersion; + + return new Deployment.DeploymentBuilder("simpleClusterDeployment") + // Service Definitions + .withService("asterix") + .applicationPath("../asterix-server/target/asterix-server-" + mavenVersion + "-binary-assembly.zip", + "/asterix", false, true, false) + .dockerFileAddress("docker/Dockerfile", false).dockerImage("spidersilk/test-asterix") + .instrumentablePath(asterixHome + "/repo/asterix-server-" + mavenVersion + ".jar") + .libraryPath(asterixHome + "/repo/*.jar").libraryPath(asterixHome + "/lib/*.jar") + .logDirectory(asterixHome + "/logs").serviceType(ServiceType.JAVA).and() + // Node Definitions + .withNode("cc", "asterix").applicationPath("config", "/asterix/config") + .startCommand(asterixHome + "/bin/asterixcc -config-file /asterix/config/cc.conf").tcpPort(19002).and() + .withNode("nc1", "asterix").startCommand(asterixHome + "/bin/asterixncservice").and() + .withNode("nc2", "asterix").startCommand(asterixHome + "/bin/asterixncservice").and().build(); + } + + public static String getMavenArtifactVersion() { + Properties mavenProperties = new Properties(); + try { + mavenProperties.load(new FileInputStream("../asterix-server/target/maven-archiver/pom.properties")); + return mavenProperties.getProperty("version"); + } catch (IOException e) { + throw new RuntimeException("Cannot open pom.properties to get the maven version"); + } + } + + public static void waitForClusterToBeUp(SpiderSilkRunner runner) throws RuntimeEngineException { + runner.runtime().runCommandInNode("cc", asterixHome + "/bin/asterixhelper wait_for_cluster"); + } + + public static TestExecutor getTestExecutor(SpiderSilkRunner runner) { + return new TestExecutor(runner.runtime().ip("cc"), runner.runtime().portMapping("cc", 19002, PortType.TCP)); + } +} diff --git a/asterixdb/asterix-spidersilk/src/test/resources/logback.xml b/asterixdb/asterix-spidersilk/src/test/resources/logback.xml new file mode 100644 index 0000000..ca7aa79 --- /dev/null +++ b/asterixdb/asterix-spidersilk/src/test/resources/logback.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<configuration> + + <appender name="Console" class="ch.qos.logback.core.ConsoleAppender"> + <layout class="ch.qos.logback.classic.PatternLayout"> + <Pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</Pattern> + </layout> + </appender> + + <logger name="me.arminb" level="DEBUG"/> + <logger name="org.apache.asterix" level="DEBUG"/> + + <root level="ERROR"> + <appender-ref ref="Console" /> + </root> +</configuration> diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index 9bf7533..bca6b62 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -189,6 +189,7 @@ <exclude>**/*.json</exclude> <exclude>**/*.adm</exclude> <exclude>**/*.template</exclude> + <exclude>**/.SpiderSilkWorkingDirectory/**</exclude> <exclude>asterix-installer/**</exclude> <!-- in case -DskipInstaller --> </excludes> </configuration> @@ -858,6 +859,7 @@ <module>asterix-client-helper</module> <module>asterix-license</module> <module>asterix-geo</module> + <module>asterix-spidersilk</module> </modules> <dependencyManagement> -- To view, visit https://asterix-gerrit.ics.uci.edu/3053 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ifd64aad26bd5cd3b6040fa53e87ae0d0d0c0a562 Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Armin Balalaie <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
