Repository: gora Updated Branches: refs/heads/master 31af406eb -> 837b12606
Add map reduce based test cases fot aerospike module Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/837b1260 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/837b1260 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/837b1260 Branch: refs/heads/master Commit: 837b12606b3c857c1d8df4b05424c607f3580230 Parents: 31af406 Author: nishadi <ndime...@gmail.com> Authored: Mon Aug 21 22:08:25 2017 +0530 Committer: nishadi <ndime...@gmail.com> Committed: Mon Aug 21 22:08:25 2017 +0530 ---------------------------------------------------------------------- .../gora/aerospike/query/AerospikeQuery.java | 3 + .../aerospike/store/AerospikeParameters.java | 12 +++- .../gora/aerospike/store/AerospikeStore.java | 2 +- gora-aerospike/src/test/conf/gora.properties | 3 +- .../gora/aerospike/GoraAerospikeTestDriver.java | 9 +-- .../mapreduce/TestAerospikeStoreCountQuery.java | 68 ++++++++++++++++++ ...estAerospikeStoreMapReduceSerialization.java | 69 ++++++++++++++++++ .../mapreduce/TestAerospikeStoreWordCount.java | 74 ++++++++++++++++++++ .../gora/aerospike/mapreduce/package-info.java | 21 ++++++ 9 files changed, 252 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/main/java/org/apache/gora/aerospike/query/AerospikeQuery.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/main/java/org/apache/gora/aerospike/query/AerospikeQuery.java b/gora-aerospike/src/main/java/org/apache/gora/aerospike/query/AerospikeQuery.java index 2f97f5f..9629b67 100644 --- a/gora-aerospike/src/main/java/org/apache/gora/aerospike/query/AerospikeQuery.java +++ b/gora-aerospike/src/main/java/org/apache/gora/aerospike/query/AerospikeQuery.java @@ -26,6 +26,9 @@ import org.apache.gora.store.DataStore; * Aerospike specific implementation of the {@link Query} interface. */ public class AerospikeQuery<K, T extends PersistentBase> extends QueryBase<K, T> { + public AerospikeQuery() { + super(null); + } public AerospikeQuery(DataStore<K, T> dataStore) { super(dataStore); } http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeParameters.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeParameters.java b/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeParameters.java index e4aab26..2ea4d57 100644 --- a/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeParameters.java +++ b/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeParameters.java @@ -19,6 +19,7 @@ package org.apache.gora.aerospike.store; import com.aerospike.client.AerospikeClient; import com.aerospike.client.Info; import com.aerospike.client.cluster.Node; +import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,8 @@ public class AerospikeParameters { private static final String AS_SERVER_PASSWORD = "gora.aerospikestore.server.password"; + private static final String AS_PROP_OVERRIDING = "gora.aerospikestore.override.hadoop.configuration"; + // Default property values private static final String DEFAULT_SERVER_IP = "localhost"; @@ -63,12 +66,19 @@ public class AerospikeParameters { * @param aerospikeMapping aerospike mapping initialized from the mapping file * @param properties property details */ - public AerospikeParameters(AerospikeMapping aerospikeMapping, Properties properties) { + public AerospikeParameters(AerospikeMapping aerospikeMapping, Properties properties, Configuration conf) { this.aerospikeMapping = aerospikeMapping; this.host = properties.getProperty(AS_SERVER_IP, DEFAULT_SERVER_IP); this.port = Integer.parseInt(properties.getProperty(AS_SERVER_PORT, DEFAULT_SERVER_PORT)); this.username = properties.getProperty(AS_SERVER_USERNAME, null); this.password = properties.getProperty(AS_SERVER_PASSWORD, null); + String overrideHadoop = properties.getProperty(AS_PROP_OVERRIDING); + if (!Boolean.parseBoolean(overrideHadoop)) { + this.host = conf.get(AS_SERVER_IP, DEFAULT_SERVER_IP); + this.port = Integer.parseInt(conf.get(AS_SERVER_PORT, DEFAULT_SERVER_PORT)); + this.username = conf.get(AS_SERVER_USERNAME, null); + this.password = conf.get(AS_SERVER_PASSWORD, null); + } } public String getHost() { http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeStore.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeStore.java b/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeStore.java index 53545f1..b456cf7 100644 --- a/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeStore.java +++ b/gora-aerospike/src/main/java/org/apache/gora/aerospike/store/AerospikeStore.java @@ -87,7 +87,7 @@ public class AerospikeStore<K, T extends PersistentBase> extends DataStoreBase<K .readMappingFile(getConf().get(PARSE_MAPPING_FILE_KEY, DEFAULT_MAPPING_FILE), keyClass, persistentClass); aerospikeParameters = new AerospikeParameters(aerospikeMappingBuilder.getAerospikeMapping(), - properties); + properties, getConf()); ClientPolicy policy = new ClientPolicy(); policy.writePolicyDefault = aerospikeParameters.getAerospikeMapping().getWritePolicy(); policy.readPolicyDefault = aerospikeParameters.getAerospikeMapping().getReadPolicy(); http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/conf/gora.properties ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/conf/gora.properties b/gora-aerospike/src/test/conf/gora.properties index c0230ec..00134c7 100644 --- a/gora-aerospike/src/test/conf/gora.properties +++ b/gora-aerospike/src/test/conf/gora.properties @@ -14,4 +14,5 @@ # limitations under the License. gora.datastore.default=org.apache.gora.aerospike.store.AerospikeStore -gora.datastore.mapping.file=gora-aerospike-mapping.xml \ No newline at end of file +gora.datastore.mapping.file=gora-aerospike-mapping.xml +gora.aerospikestore.override.hadoop.configuration=false \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/java/org/apache/gora/aerospike/GoraAerospikeTestDriver.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/GoraAerospikeTestDriver.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/GoraAerospikeTestDriver.java index 6fc120c..cc11347 100644 --- a/gora-aerospike/src/test/java/org/apache/gora/aerospike/GoraAerospikeTestDriver.java +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/GoraAerospikeTestDriver.java @@ -37,8 +37,6 @@ public class GoraAerospikeTestDriver extends GoraTestDriver { private final GenericContainer aerospikeContainer; - private final Properties properties = DataStoreFactory.createProps(); - public GoraAerospikeTestDriver(GenericContainer aerospikeContainer) { super(AerospikeStore.class); this.aerospikeContainer = aerospikeContainer; @@ -47,9 +45,8 @@ public class GoraAerospikeTestDriver extends GoraTestDriver { @Override public void setUpClass() throws Exception { log.info("Setting up Aerospike test driver"); - properties.setProperty("gora.aerospikestore.server.ip", "localhost"); - properties.setProperty("gora.aerospikestore.server.port", - aerospikeContainer.getMappedPort(3000).toString()); + conf.set("gora.aerospikestore.server.ip", "localhost"); + conf.set("gora.aerospikestore.server.port", aerospikeContainer.getMappedPort(3000).toString()); } @Override @@ -71,7 +68,7 @@ public class GoraAerospikeTestDriver extends GoraTestDriver { final DataStore<K, T> dataStore = DataStoreFactory .createDataStore((Class<? extends DataStore<K, T>>) dataStoreClass, keyClass, - persistentClass, conf, properties); + persistentClass, conf); dataStores.add(dataStore); log.info("Datastore for {} was added.", persistentClass); return dataStore; http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreCountQuery.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreCountQuery.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreCountQuery.java new file mode 100644 index 0000000..51d115e --- /dev/null +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreCountQuery.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.aerospike.mapreduce; + +import org.apache.gora.aerospike.store.AerospikeStartupLogWaitStrategy; +import org.apache.gora.aerospike.store.AerospikeStore; +import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.mapreduce.MapReduceTestUtils; +import org.apache.gora.store.DataStoreFactory; +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +import java.time.Duration; + +/** + * Aerospike Mapreduce test case for the count query + */ +public class TestAerospikeStoreCountQuery { + + private static final String DOCKER_CONTAINER_NAME = "aerospike/aerospike-server:latest"; + + @ClassRule + public static GenericContainer aerospikeContainer = new GenericContainer(DOCKER_CONTAINER_NAME) + .withExposedPorts(3000).waitingFor(new AerospikeStartupLogWaitStrategy()) + .withStartupTimeout(Duration.ofSeconds(240)); + + private AerospikeStore<String, WebPage> webPageStore; + + private Configuration conf = new Configuration(); + + @Before + public void setUp() throws Exception { + + conf.set("gora.aerospikestore.server.ip", "localhost"); + conf.set("gora.aerospikestore.server.port", aerospikeContainer.getMappedPort(3000).toString()); + + webPageStore = DataStoreFactory + .createDataStore(AerospikeStore.class, String.class, WebPage.class, conf); + } + + @After + public void tearDown() throws Exception { + webPageStore.close(); + } + + @Test + public void testCountQuery() throws Exception { + MapReduceTestUtils.testCountQuery(webPageStore, conf); + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java new file mode 100644 index 0000000..276aefd --- /dev/null +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.aerospike.mapreduce; + +import org.apache.gora.aerospike.store.AerospikeStartupLogWaitStrategy; +import org.apache.gora.aerospike.store.AerospikeStore; +import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.mapreduce.MapReduceTestUtils; +import org.apache.gora.store.DataStoreFactory; +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; +import org.junit.Before; +import org.junit.After; +import org.junit.ClassRule; + +import org.testcontainers.containers.GenericContainer; + +import java.time.Duration; + +/** + * Aerospike Mapreduce test case for map reduce serialization + */ +public class TestAerospikeStoreMapReduceSerialization { + + private static final String DOCKER_CONTAINER_NAME = "aerospike/aerospike-server:latest"; + + @ClassRule + public static GenericContainer aerospikeContainer = new GenericContainer(DOCKER_CONTAINER_NAME) + .withExposedPorts(3000).waitingFor(new AerospikeStartupLogWaitStrategy()) + .withStartupTimeout(Duration.ofSeconds(240)); + + private AerospikeStore<String, WebPage> webPageStore; + + private Configuration conf = new Configuration(); + + @Before + public void setUp() throws Exception { + + conf.set("gora.aerospikestore.server.ip", "localhost"); + conf.set("gora.aerospikestore.server.port", aerospikeContainer.getMappedPort(3000).toString()); + + webPageStore = DataStoreFactory + .createDataStore(AerospikeStore.class, String.class, WebPage.class, conf); + } + + @After + public void tearDown() throws Exception { + webPageStore.close(); + } + + @Test + public void testMapReduceSerialization() throws Exception { + MapReduceTestUtils.testMapReduceSerialization(conf, webPageStore, webPageStore); + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreWordCount.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreWordCount.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreWordCount.java new file mode 100644 index 0000000..3e24469 --- /dev/null +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreWordCount.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.aerospike.mapreduce; + +import org.apache.gora.aerospike.store.AerospikeStartupLogWaitStrategy; +import org.apache.gora.aerospike.store.AerospikeStore; +import org.apache.gora.examples.generated.TokenDatum; +import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.mapreduce.MapReduceTestUtils; +import org.apache.gora.store.DataStoreFactory; +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +import java.time.Duration; + +/** + * Aerospike Mapreduce test case for the word count + */ +public class TestAerospikeStoreWordCount { + + private static final String DOCKER_CONTAINER_NAME = "aerospike/aerospike-server:latest"; + + @ClassRule + public static GenericContainer aerospikeContainer = new GenericContainer(DOCKER_CONTAINER_NAME) + .withExposedPorts(3000).waitingFor(new AerospikeStartupLogWaitStrategy()) + .withStartupTimeout(Duration.ofSeconds(240)); + + private AerospikeStore<String, WebPage> webPageStore; + + private AerospikeStore<String, TokenDatum> tokenStore; + + private Configuration conf = new Configuration(); + + @Before + public void setUp() throws Exception { + + conf.set("gora.aerospikestore.server.ip", "localhost"); + conf.set("gora.aerospikestore.server.port", aerospikeContainer.getMappedPort(3000).toString()); + + webPageStore = DataStoreFactory + .createDataStore(AerospikeStore.class, String.class, WebPage.class, conf); + tokenStore = DataStoreFactory + .createDataStore(AerospikeStore.class, String.class, TokenDatum.class, conf); + } + + @After + public void tearDown() throws Exception { + webPageStore.close(); + tokenStore.close(); + } + + @Test + public void testWordCount() throws Exception { + MapReduceTestUtils.testWordCount(conf, webPageStore, tokenStore); + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/837b1260/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/package-info.java ---------------------------------------------------------------------- diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/package-info.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/package-info.java new file mode 100644 index 0000000..749847b --- /dev/null +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * This package contains all the unit tests for basic CRUD operations + * functionality of the Aerospike dataStore. + */ +package org.apache.gora.aerospike.mapreduce; \ No newline at end of file
