lewismc closed pull request #12: SDAP-13 Rearchitect MUDROD storage
URL: https://github.com/apache/incubator-sdap-mudrod/pull/12
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index 3009cbf..56afcc1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,9 @@ core/.externalToolBuilders/Maven_Ant_Builder.launch
core/maven-eclipse.xml
service/.classpath
web/.classpath
+storage/lib
+storage/.classpath
+storage/.externalToolBuilders/Maven_Ant_Builder.launch
+storage/.gitignore
+storage/maven-eclipse.xml
+storage/target
diff --git a/core/pom.xml b/core/pom.xml
index 2725e87..93d5b34 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../</relativePath>
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
index 2b29c03..662f8d6 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
@@ -33,13 +33,11 @@
import java.util.*;
/**
- * Recommend metadata using combination all two methods, including
content-based
- * similarity and session-level similarity
+ * Recommend metadata using combination of content-based
+ * similarity and session-level similarity.
*/
public class HybridRecommendation extends DiscoveryStepAbstract {
- /**
- *
- */
+
private static final long serialVersionUID = 1L;
// recommended metadata list
protected transient List<LinkedTerm> termList = new ArrayList<>();
@@ -50,17 +48,17 @@
private static final String WEIGHT = "weight";
/**
- * recommended data class Date: Sep 12, 2016 2:25:28 AM
+ * recommended data class
*/
class LinkedTerm {
- public String term = null;
- public double weight = 0;
- public String model = null;
+ private String term;
+ private double weight = 0;
+ private String model;
public LinkedTerm(String str, double w, String m) {
- term = str;
- weight = w;
- model = m;
+ this.term = str;
+ this.weight = w;
+ this.model = m;
}
}
@@ -79,7 +77,7 @@ public Object execute(Object o) {
}
/**
- * Get recommended data for a giving dataset
+ * Get recommended data for a given dataset
*
* @param input: a giving dataset
* @param num: the number of recommended dataset
@@ -104,15 +102,15 @@ public JsonObject getRecomDataInJson(String input, int
num) {
JsonElement sessionSimJson = mapToJson(sortedSessionSimMap, num);
resultJson.add("sessionSim", sessionSimJson);
- Map<String, Double> hybirdSimMap = new HashMap<String, Double>();
+ Map<String, Double> hybirdSimMap = new HashMap<>();
for (String name : sortedAbstractSimMap.keySet()) {
- hybirdSimMap.put(name, sortedAbstractSimMap.get(name) /** 0.4 */);
+ hybirdSimMap.put(name, sortedAbstractSimMap.get(name));
}
for (String name : sortedVariableSimMap.keySet()) {
if (hybirdSimMap.get(name) != null) {
- double sim = hybirdSimMap.get(name) + sortedVariableSimMap.get(name)
/** 0.3 */;
+ double sim = hybirdSimMap.get(name) + sortedVariableSimMap.get(name);
hybirdSimMap.put(name, Double.parseDouble(df.format(sim)));
} else {
double sim = sortedVariableSimMap.get(name);
@@ -122,7 +120,7 @@ public JsonObject getRecomDataInJson(String input, int num)
{
for (String name : sortedSessionSimMap.keySet()) {
if (hybirdSimMap.get(name) != null) {
- double sim = hybirdSimMap.get(name) + sortedSessionSimMap.get(name)
/** 0.1 */;
+ double sim = hybirdSimMap.get(name) + sortedSessionSimMap.get(name);
hybirdSimMap.put(name, Double.parseDouble(df.format(sim)));
} else {
double sim = sortedSessionSimMap.get(name);
@@ -164,13 +162,11 @@ protected JsonElement mapToJson(Map<String, Double>
wordweights, int num) {
}
String nodesJson = gson.toJson(nodes);
- JsonElement nodesElement = gson.fromJson(nodesJson, JsonElement.class);
-
- return nodesElement;
+ return gson.fromJson(nodesJson, JsonElement.class);
}
/**
- * Get recommend dataset for a giving dataset
+ * Get recommend dataset for a given dataset
*
* @param type recommend method
* @param input a giving dataset
@@ -206,8 +202,12 @@ protected JsonElement mapToJson(Map<String, Double>
wordweights, int num) {
*/
public List<LinkedTerm> getRelatedDataFromES(String type, String input, int
num) {
- SearchRequestBuilder builder =
es.getClient().prepareSearch(props.getProperty(INDEX_NAME)).setTypes(type).setQuery(QueryBuilders.termQuery("concept_A",
input)).addSort(WEIGHT, SortOrder.DESC)
- .setSize(num);
+ SearchRequestBuilder builder = es.getClient()
+ .prepareSearch(props.getProperty(INDEX_NAME))
+ .setTypes(type)
+ .setQuery(QueryBuilders.termQuery("concept_A", input))
+ .addSort(WEIGHT, SortOrder.DESC)
+ .setSize(num);
SearchResponse usrhis = builder.execute().actionGet();
@@ -266,7 +266,6 @@ public static void main(String[] args) throws IOException {
ESDriver es = new ESDriver(me.getConfig());
HybridRecommendation test = new HybridRecommendation(props, es, null);
- // String input = "NSCAT_LEVEL_1.7_V2";
String input = "AQUARIUS_L3_SSS_SMIA_MONTHLY-CLIMATOLOGY_V4";
JsonObject json = test.getRecomDataInJson(input, 10);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/utils/ClassLoadingUtils.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/ClassLoadingUtils.java
new file mode 100644
index 0000000..bda78a5
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/ClassLoadingUtils.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.utils;
+
+public class ClassLoadingUtils {
+
+ private ClassLoadingUtils() {
+ //Utility Class
+ }
+
+ /**
+ * Loads a class using the class loader.
+ * 1. The class loader of the current class is being used.
+ * 2. The thread context class loader is being used.
+ * If both approaches fail, returns null.
+ *
+ * @param className The name of the class to load.
+ * @return The class or null if no class loader could load the class.
+ * @throws ClassNotFoundException if and only if no definition for the class
with the specified name could be found.
+ */
+ public static Class<?> loadClass(String className) throws
ClassNotFoundException {
+ return ClassLoadingUtils.loadClass(ClassLoadingUtils.class,className);
+ }
+
+ /**
+ * Loads a class using the class loader.
+ * 1. The class loader of the context class is being used.
+ * 2. The thread context class loader is being used.
+ * If both approaches fail, returns null.
+ *
+ * @param contextClass The name of a context class to use.
+ * @param className The name of the class to load.
+ * @return The class or null if no class loader could load the class.
+ * @throws ClassNotFoundException Aif and only if no definition for the
class with the specified name could be found.
+ */
+ public static Class<?> loadClass(Class<?> contextClass, String className)
throws ClassNotFoundException {
+ Class<?> clazz = null;
+ if (contextClass.getClassLoader() != null) {
+ clazz = loadClass(className, contextClass.getClassLoader());
+ }
+ if (clazz == null && Thread.currentThread().getContextClassLoader() !=
null) {
+ clazz = loadClass(className,
Thread.currentThread().getContextClassLoader());
+ }
+ if (clazz == null) {
+ throw new ClassNotFoundException("Failed to load class" + className);
+ }
+ return clazz;
+ }
+
+ /**
+ * Loads a {@link Class} from the specified {@link ClassLoader} without
throwing {@ClassNotFoundException}.
+ *
+ * @param className The name of the class to load.
+ * @param classLoader Class loader instance where given class to be loaded.
+ * @return The class or null if no class loader could load the class.
+ */
+ private static Class<?> loadClass(String className, ClassLoader classLoader)
{
+ Class<?> clazz = null;
+ if (classLoader != null && className != null) {
+ try {
+ clazz = classLoader.loadClass(className);
+ } catch (ClassNotFoundException e) {
+ //Ignore and return null
+ }
+ }
+ return clazz;
+ }
+}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/utils/ReflectionUtils.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/ReflectionUtils.java
new file mode 100644
index 0000000..b902f7e
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/ReflectionUtils.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.utils;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+/**
+ * Utility methods related to reflection
+ */
+public class ReflectionUtils {
+
+ public static Class<?>[] EMPTY_CLASS_ARRAY = new Class<?>[0];
+ public static Object[] EMPTY_OBJECT_ARRAY = new Object[0];
+
+ /**
+ * Returns the empty argument constructor of the class.
+ *
+ * @param clazz the class reference of given type <code>T</code>.
+ * @param <T> class type variable.
+ * @return constructor for the given class type <code>T</code>.
+ * @throws SecurityException if the caller's class loader is not the same as
the
+ * class loader of above class.
+ * @throws NoSuchMethodException default construct cannot by found for given
class type.
+ */
+ public static<T> Constructor<T> getConstructor(Class<T> clazz)
+ throws SecurityException, NoSuchMethodException {
+ if(clazz == null) {
+ throw new IllegalArgumentException("class cannot be null");
+ }
+ Constructor<T> cons = clazz.getConstructor(EMPTY_CLASS_ARRAY);
+ cons.setAccessible(true);
+ return cons;
+ }
+
+ /**
+ * Returns whether the class defines an empty argument constructor.
+ *
+ * @param clazz class reference of given type <code>T</code>.
+ * @return boolean indicating constructor for the given class type
<code>T</code> exist.
+ * @throws SecurityException if the caller's class loader is not the same as
the
+ * class loader of above class.
+ * @throws NoSuchMethodException default construct cannot by found for given
class type.
+ */
+ public static boolean hasConstructor(Class<?> clazz)
+ throws SecurityException, NoSuchMethodException {
+ if(clazz == null) {
+ throw new IllegalArgumentException("class cannot be null");
+ }
+ Constructor<?>[] consts = clazz.getConstructors();
+
+ boolean found = false;
+ for(Constructor<?> cons : consts) {
+ if(cons.getParameterTypes().length == 0) {
+ found = true;
+ }
+ }
+
+ return found;
+ }
+
+ /**
+ * Constructs a new instance of the class using the no-arg constructor.
+ *
+ * @param clazz the class of the object.
+ * @param <T> class type variable.
+ * @return a new instance of the object.
+ * @throws SecurityException if the caller's class loader is not the same as
the
+ * class loader of above class.
+ * @throws IllegalArgumentException this will not be thrown since
<code>field.get(obj)</code> passing obj is null
+ * since the field is a static class level variable inside the class.
+ * @throws IllegalAccessException if the field is inaccessible due to java
language access control.
+ * @throws InstantiationException could not be instantiated from the given
constructor.
+ * @throws NoSuchMethodException default construct cannot by found for given
class type.
+ * @throws InvocationTargetException if the underlying constructor throws an
exception.
+ */
+ public static <T> T newInstance(Class<T> clazz)
+ throws InstantiationException, IllegalAccessException,
+ SecurityException, NoSuchMethodException, IllegalArgumentException,
+ InvocationTargetException {
+
+ Constructor<T> cons = getConstructor(clazz);
+
+ return cons.newInstance(EMPTY_OBJECT_ARRAY);
+ }
+
+ /**
+ * Constructs a new instance of the class using the no-arg constructor.
+ *
+ * @param classStr the class name of the object.
+ * @return a new instance of the object.
+ * @throws SecurityException if the caller's class loader is not the same as
the
+ * class loader of above class.
+ * @throws IllegalArgumentException this will not be thrown since
<code>field.get(obj)</code> passing obj is null.
+ * since the field is a static class level variable inside the class.
+ * @throws IllegalAccessException if the field is inaccessible due to java
language access control.
+ * @throws ClassNotFoundException class definition cannot be found for the
class type.
+ * @throws InstantiationException could not be instantiated from the given
constructor.
+ * @throws NoSuchMethodException default construct cannot by found for given
class type.
+ * @throws InvocationTargetException if the underlying constructor throws an
exception.
+ */
+ public static Object newInstance(String classStr)
+ throws InstantiationException, IllegalAccessException,
+ ClassNotFoundException, SecurityException, IllegalArgumentException,
+ NoSuchMethodException, InvocationTargetException {
+ if(classStr == null) {
+ throw new IllegalArgumentException("class cannot be null");
+ }
+ Class<?> clazz = ClassLoadingUtils.loadClass(classStr);
+ return newInstance(clazz);
+ }
+
+ /**
+ * Returns the value of a named static field.
+ *
+ * @param clazz the class of the object.
+ * @param fieldName field name of the instance which value is required.
+ * @return a new instance of the object.
+ * @throws SecurityException if the caller's class loader is not the same as
the
+ * class loader of above class.
+ * @throws NoSuchFieldException if a field with the specified name is not
found.
+ * @throws IllegalArgumentException this will not be thrown since
<code>field.get(obj)</code> passing obj is null
+ * since the field is a static class level variable inside the class.
+ * @throws IllegalAccessException if the field is inaccessible due to java
language access control.
+ */
+ public static Object getStaticField(Class<?> clazz, String fieldName)
+ throws IllegalArgumentException, SecurityException,
+ IllegalAccessException, NoSuchFieldException {
+
+ return clazz.getField(fieldName).get(null);
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 6d1414d..e371fe4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,7 @@
<version>10</version>
</parent>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>pom</packaging>
@@ -135,8 +135,10 @@
</properties>
<modules>
+ <module>analysis</module>
<module>core</module>
<module>service</module>
+ <module>storage</module>
<module>web</module>
</modules>
@@ -416,6 +418,14 @@
</reporting>
<dependencyManagement>
<dependencies>
+ <!-- Internal Dependencies -->
+ <dependency>
+ <groupId>org.apache.sdap</groupId>
+ <artifactId>mudrod-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- End Internal Dependencies -->
+
<!-- Elasticsearch dependencies -->
<dependency>
<groupId>org.elasticsearch</groupId>
diff --git a/service/pom.xml b/service/pom.xml
index 8dd59a9..3d98af5 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../</relativePath>
@@ -40,14 +40,14 @@
<dependencies>
<!-- Core Module -->
<dependency>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-core</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Static web resources -->
<dependency>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-web</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/storage/pom.xml b/storage/pom.xml
new file mode 100644
index 0000000..448315b
--- /dev/null
+++ b/storage/pom.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0 Unless
+
+ required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS"
+ BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ express or implied. See the License for the specific language
+ governing permissions and limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.sdap</groupId>
+ <artifactId>mudrod-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
+
+ <artifactId>mudrod-storage</artifactId>
+
+ <name>Mudrod :: Storage</name>
+ <description>Mudrod storage logic.</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sdap</groupId>
+ <artifactId>mudrod-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jdom</groupId>
+ <artifactId>jdom</artifactId>
+ </dependency>
+
+ <!-- Elasticsearch dependencies -->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>transport-netty4-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-spark-20_2.11</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ </dependency>
+ <!-- End of Elasticsearch dependencies -->
+
+ <!-- Logging dependencies -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+ <!-- End of Logging dependencies -->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <resources>
+
+ <resource>
+ <directory>${basedir}/../</directory>
+ <targetPath>META-INF</targetPath>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>NOTICE.txt</include>
+ </includes>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>release</id>
+ <build>
+ <resources>
+ <resource>
+ <directory>${basedir}/../</directory>
+ <targetPath>
+ ${project.build.directory}/apidocs/META-INF
+ </targetPath>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>NOTICE.txt</include>
+ </includes>
+ </resource>
+ </resources>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriver.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriver.java
new file mode 100644
index 0000000..9a9d7cb
--- /dev/null
+++ b/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriver.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.sdap.mudrod.storage.elasticsearch.ElasticSearchDriver;
+import org.apache.sdap.mudrod.storage.solr.SolrDriver;
+
+/**
+ * Core storage datum from which all concrete storage-related
+ * implementations should extend.
+ * @since v0.0.1-SNAPSHOT
+ */
+public interface StorageDriver {
+
+ /**
+ *
+ * @return an initialized {@link
org.apache.sdap.mudrod.storage.StorageDriver} implementation.
+ */
+ default StorageDriver initialize(Properties props) {
+ StorageDriver sDriver = null;
+ if (props != null) {
+ switch (props.getProperty("mudrod.storage.driver", "elasticsearch")) {
+ case "elasticsearch":
+ sDriver = new ElasticSearchDriver(props);
+ break;
+ default:
+ sDriver = new SolrDriver(props);
+ break;
+ }
+ } else {
+
+ }
+ return sDriver;
+ }
+
+ abstract void createBulkProcessor();
+
+ abstract void destroyBulkProcessor();
+
+ abstract void putMapping(String indexName, String settingsJson, String
mappingJson) throws IOException;
+
+ abstract String customAnalyzing(String indexName, String str) throws
InterruptedException, ExecutionException;
+
+ abstract String customAnalyzing(String indexName, String analyzer, String
str) throws InterruptedException, ExecutionException;
+
+ abstract List<String> customAnalyzing(String indexName, List<String> list)
throws InterruptedException, ExecutionException;
+
+ //abstract void deleteAllByQuery(String index, String type, QueryBuilder
query);
+
+ abstract void deleteType(String index, String type);
+
+ abstract List<String> getTypeListWithPrefix(Object object, Object object2);
+
+ abstract List<String> getIndexListWithPrefix(Object object);
+
+ abstract String searchByQuery(String index, String type, String query);
+
+ abstract String searchByQuery(String index, String type, String query,
Boolean bDetail);
+
+ abstract List<List<String>> buildMeasurementHierarchies(
+ List<String> topics, List<String> terms, List<String> variables,
+ List<String> variableDetails);
+
+ abstract List<String> autoComplete(String index, String term);
+
+ abstract void close();
+
+ abstract void refreshIndex();
+
+ //abstract Client makeClient(Properties props);
+
+ //abstract UpdateRequest generateUpdateRequest(String index,
+ // String type, String id, String field1, Object value1);
+
+ //UpdateRequest generateUpdateRequest(String index, String type, String id,
+ // Map<String, Object> filedValueMap);
+
+ abstract int getDocCount(String index, String... type);
+
+ abstract int getDocCount(String[] index, String[] type);
+
+ //public int getDocCount(String[] index, String[] type, QueryBuilder
filterSearch);
+
+}
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriverFactory.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriverFactory.java
new file mode 100644
index 0000000..ef3c80b
--- /dev/null
+++
b/storage/src/main/java/org/apache/sdap/mudrod/storage/StorageDriverFactory.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.apache.sdap.mudrod.utils.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class StorageDriverFactory {
+
+ public static final Logger log =
LoggerFactory.getLogger(StorageDriverFactory.class);
+
+ public static final String MUDROD_DEFAULT_PROPERTIES_FILE =
"config.properties";
+
+ public static final String MUDROD_DEFAULT_DATASTORE_KEY =
"mudrod.datastore.default";
+
+ private StorageDriverFactory() { }
+
+ /**
+ * Creates a new {@link Properties}. It adds the default gora configuration
+ * resources. This properties object can be modified and used to instantiate
+ * store instances. It is recommended to use a properties object for a single
+ * store, because the properties object is passed on to store initialization
+ * methods that are able to store the properties as a field.
+ * @return The new properties object.
+ */
+ public static Properties createProps() {
+ try {
+ Properties properties = new Properties();
+ InputStream stream = StorageDriverFactory.class.getClassLoader()
+ .getResourceAsStream(MUDROD_DEFAULT_PROPERTIES_FILE);
+ if(stream != null) {
+ try {
+ properties.load(stream);
+ return properties;
+ } finally {
+ stream.close();
+ }
+ } else {
+ log.warn(MUDROD_DEFAULT_PROPERTIES_FILE + " not found, properties will
be empty.");
+ }
+ return properties;
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void initializeStorageDriver(
+ StorageDriver storageDriver, Properties properties) throws
IOException {
+ storageDriver.initialize(properties);
+ }
+
+ /**
+ * Instantiate a new {@link DataStore}.
+ *
+ * @param storageDriverClass The datastore implementation class.
+ * @param properties The properties to be used in the store.
+ * @return A new {@link org.apache.sdap.mudrod.storage.StorageDriver}
instance.
+ * @throws Exception
+ */
+ public static StorageDriver createDataStore(Class<?> storageDriverClass,
Properties properties) throws Exception{
+ try {
+ StorageDriver storageDriver =
+ (StorageDriver) ReflectionUtils.newInstance(storageDriverClass);
+ initializeStorageDriver(storageDriver, properties);
+ return storageDriver;
+ } catch(Exception ex) {
+ throw new Exception(ex);
+ }
+ }
+}
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/ElasticSearchDriver.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/ElasticSearchDriver.java
new file mode 100644
index 0000000..886070d
--- /dev/null
+++
b/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/ElasticSearchDriver.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage.elasticsearch;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.storage.StorageDriver;
+import org.apache.sdap.mudrod.utils.ESTransportClient;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
+import
org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.SuggestBuilders;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import com.google.gson.GsonBuilder;
+
+/**
+ *
+ */
+public class ElasticSearchDriver implements StorageDriver {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ElasticSearchDriver.class);
+ private transient Client client = null;
+ private transient Node node = null;
+ private transient BulkProcessor bulkProcessor = null;
+
+ /**
+ * @param props
+ *
+ */
+ public ElasticSearchDriver(Properties props) {
+ try {
+ setClient(makeClient(props));
+ } catch (IOException e) {
+ LOG.error("Error whilst constructing Elastcisearch client.", e);
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#createBulkProcessor()
+ */
+ @Override
+ public void createBulkProcessor() {
+ LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}",
1000, 2500500);
+ setBulkProcessor(BulkProcessor.builder(getClient(), new
BulkProcessor.Listener() {
+ @Override
+ public void beforeBulk(long executionId, BulkRequest request) {
+ LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not
implemented yet!");
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
+ LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not
implemented yet!");
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request, Throwable
failure) {
+ LOG.error("Bulk request has failed!");
+ throw new RuntimeException("Caught exception in bulk: " +
request.getDescription() + ", failure: " + failure, failure);
+ }
+ }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500,
ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100),
10)).setConcurrentRequests(1)
+ .build());
+
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#destroyBulkProcessor()
+ */
+ @Override
+ public void destroyBulkProcessor() {
+ try {
+ getBulkProcessor().awaitClose(20, TimeUnit.MINUTES);
+ setBulkProcessor(null);
+ refreshIndex();
+ } catch (InterruptedException e) {
+ LOG.error("Error destroying the Bulk Processor.", e);
+ }
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#putMapping(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public void putMapping(String indexName, String settingsJson, String
mappingJson) throws IOException {
+ boolean exists = getClient()
+ .admin()
+ .indices()
+ .prepareExists(indexName)
+ .execute()
+ .actionGet()
+ .isExists();
+ if (exists) {
+ return;
+ }
+
+
getClient().admin().indices().prepareCreate(indexName).setSettings(Settings.builder().loadFromSource(settingsJson)).execute().actionGet();
+
getClient().admin().indices().preparePutMapping(indexName).setType("_default_").setSource(mappingJson).execute().actionGet();
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public String customAnalyzing(String indexName, String str) throws
InterruptedException, ExecutionException {
+ return this.customAnalyzing(indexName, "cody", str);
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public String customAnalyzing(String indexName, String analyzer, String str)
throws InterruptedException, ExecutionException {
+ String[] strList = str.toLowerCase().split(",");
+ for (int i = 0; i < strList.length; i++) {
+ String tmp = "";
+ AnalyzeResponse r =
client.admin().indices().prepareAnalyze(strList[i]).setIndex(indexName).setAnalyzer(analyzer).execute().get();
+ for (AnalyzeToken token : r.getTokens()) {
+ tmp += token.getTerm() + " ";
+ }
+ strList[i] = tmp.trim();
+ }
+ return String.join(",", strList);
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.util.List)
+ */
+ @Override
+ public List<String> customAnalyzing(String indexName, List<String> list)
throws InterruptedException, ExecutionException {
+ if (list == null) {
+ return list;
+ }
+ List<String> customlist = new ArrayList<>();
+ for (String aList : list) {
+ customlist.add(this.customAnalyzing(indexName, aList));
+ }
+
+ return customlist;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#deleteType(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public void deleteType(String index, String type) {
+ this.deleteAllByQuery(index, type, QueryBuilders.matchAllQuery());
+
+ }
+
+ public void deleteAllByQuery(String index, String type, QueryBuilder query) {
+ ImmutableOpenMap<String, MappingMetaData> mappings = getClient().admin()
+ .cluster().prepareState().execute().actionGet()
+ .getState().metaData().index(index).getMappings();
+
+ //check if the type exists
+ if (!mappings.containsKey(type))
+ return;
+
+ createBulkProcessor();
+ SearchResponse scrollResp = getClient().prepareSearch(index).setSearchType(
+ SearchType.QUERY_AND_FETCH).setTypes(type).setScroll(
+ new
TimeValue(60000)).setQuery(query).setSize(10000).execute().actionGet();
+
+ while (true) {
+ for (SearchHit hit : scrollResp.getHits().getHits()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, type,
hit.getId());
+ getBulkProcessor().add(deleteRequest);
+ }
+
+ scrollResp = getClient().prepareSearchScroll(scrollResp.getScrollId())
+ .setScroll(new TimeValue(600000)).execute().actionGet();
+ if (scrollResp.getHits().getHits().length == 0) {
+ break;
+ }
+
+ }
+ destroyBulkProcessor();
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getTypeListWithPrefix(java.lang.Object,
java.lang.Object)
+ */
+ @Override
+ public List<String> getTypeListWithPrefix(Object object, Object object2) {
+ ArrayList<String> typeList = new ArrayList<>();
+ GetMappingsResponse res;
+ try {
+ res = getClient().admin().indices().getMappings(new
GetMappingsRequest().indices(object.toString())).get();
+ ImmutableOpenMap<String, MappingMetaData> mapping =
res.mappings().get(object.toString());
+ for (ObjectObjectCursor<String, MappingMetaData> c : mapping) {
+ if (c.key.startsWith(object2.toString())) {
+ typeList.add(c.key);
+ }
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ LOG.error("Error whilst obtaining type list from Elasticsearch
mappings.", e);
+ }
+ return typeList;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getIndexListWithPrefix(java.lang.Object)
+ */
+ @Override
+ public List<String> getIndexListWithPrefix(Object object) {
+ LOG.info("Retrieving index list with prefix: {}", object.toString());
+ String[] indices = client.admin().indices().getIndex(new
GetIndexRequest()).actionGet().getIndices();
+
+ ArrayList<String> indexList = new ArrayList<>();
+ for (String indexName : indices) {
+ if (indexName.startsWith(object.toString())) {
+ indexList.add(indexName);
+ }
+ }
+
+ return indexList;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#searchByQuery(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public String searchByQuery(String index, String type, String query) {
+ return searchByQuery(index, type, query, false);
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#searchByQuery(java.lang.String,
java.lang.String, java.lang.String, java.lang.Boolean)
+ */
+ @Override
+ public String searchByQuery(String index, String type, String query, Boolean
bDetail) {
+ boolean exists =
getClient().admin().indices().prepareExists(index).execute().actionGet().isExists();
+ if (!exists) {
+ return null;
+ }
+
+ QueryBuilder qb = QueryBuilders.queryStringQuery(query);
+ SearchResponse response =
getClient().prepareSearch(index).setTypes(type).setQuery(qb).setSize(500).execute().actionGet();
+
+ // Map of K,V pairs where key is the field name from search result and
value is the that should be returned for that field. Not always the same.
+ Map<String, String> fieldsToReturn = new HashMap<>();
+
+ fieldsToReturn.put("Dataset-ShortName", "Short Name");
+ fieldsToReturn.put("Dataset-LongName", "Long Name");
+ fieldsToReturn.put("DatasetParameter-Topic", "Topic");
+ fieldsToReturn.put("Dataset-Description", "Dataset-Description");
+ fieldsToReturn.put("DatasetCitation-ReleaseDateLong", "Release Date");
+
+ if (bDetail) {
+ fieldsToReturn.put("DatasetPolicy-DataFormat", "DataFormat");
+ fieldsToReturn.put("Dataset-Doi", "Dataset-Doi");
+ fieldsToReturn.put("Dataset-ProcessingLevel", "Processing Level");
+ fieldsToReturn.put("DatasetCitation-Version", "Version");
+ fieldsToReturn.put("DatasetSource-Sensor-ShortName",
"DatasetSource-Sensor-ShortName");
+ fieldsToReturn.put("DatasetProject-Project-ShortName",
"DatasetProject-Project-ShortName");
+ fieldsToReturn.put("DatasetParameter-Category",
"DatasetParameter-Category");
+ fieldsToReturn.put("DatasetLocationPolicy-BasePath",
"DatasetLocationPolicy-BasePath");
+ fieldsToReturn.put("DatasetParameter-Variable-Full",
"DatasetParameter-Variable-Full");
+ fieldsToReturn.put("DatasetParameter-Term-Full",
"DatasetParameter-Term-Full");
+ fieldsToReturn.put("DatasetParameter-VariableDetail",
"DatasetParameter-VariableDetail");
+
+ fieldsToReturn.put("DatasetRegion-Region", "Region");
+ fieldsToReturn.put("DatasetCoverage-NorthLat", "NorthLat");
+ fieldsToReturn.put("DatasetCoverage-SouthLat", "SouthLat");
+ fieldsToReturn.put("DatasetCoverage-WestLon", "WestLon");
+ fieldsToReturn.put("DatasetCoverage-EastLon", "EastLon");
+ fieldsToReturn.put("DatasetCoverage-StartTimeLong-Long",
"DatasetCoverage-StartTimeLong-Long");
+ fieldsToReturn.put("Dataset-DatasetCoverage-StopTimeLong",
"Dataset-DatasetCoverage-StopTimeLong");
+
+ fieldsToReturn.put("Dataset-TemporalResolution",
"Dataset-TemporalResolution");
+ fieldsToReturn.put("Dataset-TemporalRepeat", "Dataset-TemporalRepeat");
+ fieldsToReturn.put("Dataset-LatitudeResolution",
"Dataset-LatitudeResolution");
+ fieldsToReturn.put("Dataset-LongitudeResolution",
"Dataset-LongitudeResolution");
+ fieldsToReturn.put("Dataset-AcrossTrackResolution",
"Dataset-AcrossTrackResolution");
+ fieldsToReturn.put("Dataset-AlongTrackResolution",
"Dataset-AlongTrackResolution");
+ }
+
+ List<Map<String, Object>> searchResults = new ArrayList<>();
+
+ for (SearchHit hit : response.getHits().getHits()) {
+ Map<String, Object> source = hit.getSource();
+
+ Map<String, Object> searchResult =
source.entrySet().stream().filter(entry ->
fieldsToReturn.keySet().contains(entry.getKey()))
+ .collect(Collectors.toMap(entry ->
fieldsToReturn.get(entry.getKey()), Entry::getValue));
+
+ // searchResult is now a map where the key = value from fieldsToReturn
and the value = value from search result
+
+ // Some results require special handling/formatting:
+ // Release Date formatting
+ LocalDate releaseDate =
Instant.ofEpochMilli(Long.parseLong(((ArrayList<String>)
searchResult.get("Release Date")).get(0))).atZone(ZoneId.of("Z")).toLocalDate();
+ searchResult.put("Release Date",
releaseDate.format(DateTimeFormatter.ISO_DATE));
+
+ if (bDetail) {
+
+ // DataFormat value, translate RAW to BINARY
+ if ("RAW".equals(searchResult.get("DataFormat"))) {
+ searchResult.put("DataFormat", "BINARY");
+ }
+
+ // DatasetLocationPolicy-BasePath Should only contain ftp, http, or
https URLs
+ List<String> urls = ((List<String>)
searchResult.get("DatasetLocationPolicy-BasePath")).stream().filter(url ->
url.startsWith("ftp") || url.startsWith("http")).collect(Collectors.toList());
+ searchResult.put("DatasetLocationPolicy-BasePath", urls);
+
+ // Time Span Formatting
+ LocalDate startDate = Instant.ofEpochMilli((Long)
searchResult.get("DatasetCoverage-StartTimeLong-Long")).atZone(ZoneId.of("Z")).toLocalDate();
+ LocalDate endDate =
"".equals(searchResult.get("Dataset-DatasetCoverage-StopTimeLong")) ?
+ null :
+
Instant.ofEpochMilli(Long.parseLong(searchResult.get("Dataset-DatasetCoverage-StopTimeLong").toString())).atZone(ZoneId.of("Z")).toLocalDate();
+ searchResult.put("Time Span",
startDate.format(DateTimeFormatter.ISO_DATE) + " to " + (endDate == null ?
"Present" : endDate.format(DateTimeFormatter.ISO_DATE)));
+
+ // Temporal resolution can come from one of two fields
+ searchResult.put("TemporalResolution",
"".equals(searchResult.get("Dataset-TemporalResolution")) ?
searchResult.get("Dataset-TemporalRepeat") :
searchResult.get("Dataset-TemporalResolution"));
+
+ // Special formatting for spatial resolution
+ String latResolution = (String)
searchResult.get("Dataset-LatitudeResolution");
+ String lonResolution = (String)
searchResult.get("Dataset-LongitudeResolution");
+ if (!latResolution.isEmpty() && !lonResolution.isEmpty()) {
+ searchResult.put("SpatialResolution", latResolution + " degrees
(latitude) x " + lonResolution + " degrees (longitude)");
+ } else {
+ String acrossResolution = (String)
searchResult.get("Dataset-AcrossTrackResolution");
+ String alonResolution = (String)
searchResult.get("Dataset-AlongTrackResolution");
+ double dAcrossResolution = Double.parseDouble(acrossResolution) /
1000;
+ double dAlonResolution = Double.parseDouble(alonResolution) / 1000;
+ searchResult.put("SpatialResolution", dAlonResolution + " km (Along)
x " + dAcrossResolution + " km (Across)");
+ }
+
+ // Measurement is a list of hierarchies that goes Topic -> Term ->
Variable -> Variable Detail. Need to construct these hierarchies.
+ List<List<String>> measurements =
buildMeasurementHierarchies((List<String>) searchResult.get("Topic"),
(List<String>) searchResult.get("DatasetParameter-Term-Full"),
+ (List<String>)
searchResult.get("DatasetParameter-Variable-Full"), (List<String>)
searchResult.get("DatasetParameter-VariableDetail"));
+
+ searchResult.put("Measurements", measurements);
+
+ }
+
+ searchResults.add(searchResult);
+ }
+
+ Map<String, List<?>> pdResults = new HashMap<>();
+ pdResults.put("PDResults", searchResults);
+
+ return new GsonBuilder().create().toJson(pdResults);
+ }
+
+ /**
+ * Builds a List of Measurement Hierarchies given the individual source
lists.
+ * The hierarchy is built from the element in the same position from each
input list in the order: Topic -> Term -> Variable -> VariableDetail
+ * "None" and blank strings are ignored. If, at any level, an element does
not exist for that position or it is "None" or blank, that hierarchy is
considered complete.
+ *
+ * For example, if the input is:
+ * <pre>
+ * topics = ["Oceans", "Oceans"]
+ * terms = ["Sea Surface Topography", "Ocean Waves"]
+ * variables = ["Sea Surface Height", "Significant Wave Height"]
+ * variableDetails = ["None", "None"]
+ * </pre>
+ *
+ * The output would be:
+ * <pre>
+ * [
+ * ["Oceans", "Sea Surface Topography", "Sea Surface Height"],
+ * ["Oceans", "Ocean Waves", "Significant Wave Height"]
+ * ]
+ * </pre>
+ * Oceans > Sea Surface Topography > Sea Surface Height
+ * Oceans > Ocean Waves > Significant Wave Height
+ *
+ * @param topics List of topics, the first element of a measurement
+ * @param terms List of terms, the second element of a measurement
+ * @param variables List of variables, the third element of a measurement
+ * @param variableDetails List of variable details, the fourth element of a
measurement
+ *
+ * @return A List where each element is a single hierarchy (as a List) built
from the provided input lists.
+ */
+ @Override
+ public List<List<String>> buildMeasurementHierarchies(List<String> topics,
List<String> terms, List<String> variables, List<String> variableDetails) {
+ List<List<String>> measurements = new ArrayList<>();
+
+ for (int x = 0; x < topics.size(); x++) {
+ measurements.add(new ArrayList<>());
+ measurements.get(x).add(topics.get(x));
+ // Only add the next 'level' if we can
+ if (x < terms.size() && !"None".equalsIgnoreCase(terms.get(x)) &&
StringUtils.isNotBlank(terms.get(x))) {
+ measurements.get(x).add(terms.get(x));
+ if (x < variables.size() && !"None".equalsIgnoreCase(variables.get(x))
&& StringUtils.isNotBlank(variables.get(x))) {
+ measurements.get(x).add(variables.get(x));
+ if (x < variableDetails.size() &&
!"None".equalsIgnoreCase(variableDetails.get(x)) &&
StringUtils.isNotBlank(variableDetails.get(x))) {
+ measurements.get(x).add(variableDetails.get(x));
+ }
+ }
+ }
+ }
+
+ return measurements;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#autoComplete(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public List<String> autoComplete(String index, String term) {
+ boolean exists =
this.getClient().admin().indices().prepareExists(index).execute().actionGet().isExists();
+ if (!exists) {
+ return new ArrayList<>();
+ }
+
+ Set<String> suggestHS = new HashSet<>();
+ List<String> suggestList = new ArrayList<>();
+
+ // please make sure that the completion field is configured in the ES
mapping
+ CompletionSuggestionBuilder suggestionsBuilder =
SuggestBuilders.completionSuggestion("Dataset-Metadata").prefix(term,
Fuzziness.fromEdits(2)).size(100);
+ SearchRequestBuilder suggestRequestBuilder =
getClient().prepareSearch(index).suggest(new
SuggestBuilder().addSuggestion("completeMe", suggestionsBuilder));
+ SearchResponse sr =
suggestRequestBuilder.setFetchSource(false).execute().actionGet();
+
+ Iterator<? extends Suggest.Suggestion.Entry.Option> iterator =
sr.getSuggest().getSuggestion("completeMe").iterator().next().getOptions().iterator();
+
+ while (iterator.hasNext()) {
+ Suggest.Suggestion.Entry.Option next = iterator.next();
+ String suggest = next.getText().string().toLowerCase();
+ suggestList.add(suggest);
+ }
+
+ suggestHS.addAll(suggestList);
+ suggestList.clear();
+ suggestList.addAll(suggestHS);
+ return suggestList;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#close()
+ */
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#refreshIndex()
+ */
+ @Override
+ public void refreshIndex() {
+ client.admin().indices().prepareRefresh().execute().actionGet();
+ }
+
+ /**
+ * Generates a TransportClient or NodeClient
+ *
+ * @param props a populated {@link java.util.Properties} object
+ * @return a constructed {@link org.elasticsearch.client.Client}
+ * @throws IOException if there is an error building the
+ * {@link org.elasticsearch.client.Client}
+ */
+ protected Client makeClient(Properties props) throws IOException {
+ String clusterName = props.getProperty(MudrodConstants.ES_CLUSTER);
+ String hostsString = props.getProperty(MudrodConstants.ES_UNICAST_HOSTS);
+ String[] hosts = hostsString.split(",");
+ String portStr = props.getProperty(MudrodConstants.ES_TRANSPORT_TCP_PORT);
+ int port = Integer.parseInt(portStr);
+
+ Settings.Builder settingsBuilder = Settings.builder();
+
+ // Set the cluster name and build the settings
+ if (!clusterName.isEmpty())
+ settingsBuilder.put("cluster.name", clusterName);
+
+ settingsBuilder.put("http.type", "netty3");
+ settingsBuilder.put("transport.type", "netty3");
+
+ Settings settings = settingsBuilder.build();
+
+ Client client = null;
+
+ // Prefer TransportClient
+ if (hosts != null && port > 1) {
+ TransportClient transportClient = new ESTransportClient(settings);
+ for (String host : hosts)
+ transportClient.addTransportAddress(new
InetSocketTransportAddress(InetAddress.getByName(host), port));
+ client = transportClient;
+ } else if (clusterName != null) {
+ node = new Node(settings);
+ client = node.client();
+ }
+
+ return client;
+ }
+
+ /**
+ * @return the client
+ */
+ public Client getClient() {
+ return client;
+ }
+
+ /**
+ * @param client the client to set
+ */
+ public void setClient(Client client) {
+ this.client = client;
+ }
+
+ /**
+ * @return the bulkProcessor
+ */
+ public BulkProcessor getBulkProcessor() {
+ return bulkProcessor;
+ }
+
+ /**
+ * @param bulkProcessor the bulkProcessor to set
+ */
+ public void setBulkProcessor(BulkProcessor bulkProcessor) {
+ this.bulkProcessor = bulkProcessor;
+ }
+
+ public UpdateRequest generateUpdateRequest(String index, String type, String
id, String field1, Object value1) {
+
+ UpdateRequest ur = null;
+ try {
+ ur = new UpdateRequest(index, type,
id).doc(jsonBuilder().startObject().field(field1, value1).endObject());
+ } catch (IOException e) {
+ LOG.error("Error whilst attempting to generate a new Update Request.",
e);
+ }
+
+ return ur;
+ }
+
+ public UpdateRequest generateUpdateRequest(String index, String type, String
id, Map<String, Object> filedValueMap) {
+
+ UpdateRequest ur = null;
+ try {
+ XContentBuilder builder = jsonBuilder().startObject();
+ for (Entry<String, Object> entry : filedValueMap.entrySet()) {
+ String key = entry.getKey();
+ builder.field(key, filedValueMap.get(key));
+ }
+ builder.endObject();
+ ur = new UpdateRequest(index, type, id).doc(builder);
+ } catch (IOException e) {
+ LOG.error("Error whilst attempting to generate a new Update Request.",
e);
+ }
+
+ return ur;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getDocCount(java.lang.String,
java.lang.String[])
+ */
+ @Override
+ public int getDocCount(String index, String... type) {
+ MatchAllQueryBuilder search = QueryBuilders.matchAllQuery();
+ String[] indexArr = new String[] { index };
+ return this.getDocCount(indexArr, type, search);
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getDocCount(java.lang.String[],
java.lang.String[])
+ */
+ @Override
+ public int getDocCount(String[] index, String[] type) {
+ MatchAllQueryBuilder search = QueryBuilders.matchAllQuery();
+ return this.getDocCount(index, type, search);
+ }
+
+ public int getDocCount(String[] index, String[] type, QueryBuilder
filterSearch) {
+ SearchRequestBuilder countSrBuilder =
getClient().prepareSearch(index).setTypes(type).setQuery(filterSearch).setSize(0);
+ SearchResponse countSr = countSrBuilder.execute().actionGet();
+ int docCount = (int) countSr.getHits().getTotalHits();
+ return docCount;
+ }
+
+}
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/package-info.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/package-info.java
new file mode 100644
index 0000000..107ff51
--- /dev/null
+++
b/storage/src/main/java/org/apache/sdap/mudrod/storage/elasticsearch/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage.elasticsearch;
\ No newline at end of file
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/package-info.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/package-info.java
new file mode 100644
index 0000000..af07a01
--- /dev/null
+++ b/storage/src/main/java/org/apache/sdap/mudrod/storage/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage;
\ No newline at end of file
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/SolrDriver.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/SolrDriver.java
new file mode 100644
index 0000000..58fac37
--- /dev/null
+++ b/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/SolrDriver.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage.solr;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.sdap.mudrod.storage.StorageDriver;
+
+/**
+ *
+ */
+public class SolrDriver implements StorageDriver {
+
+ /**
+ * @param props
+ *
+ */
+ public SolrDriver(Properties props) {
+ // TODO Auto-generated constructor stub
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#createBulkProcessor()
+ */
+ @Override
+ public void createBulkProcessor() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#destroyBulkProcessor()
+ */
+ @Override
+ public void destroyBulkProcessor() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#putMapping(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public void putMapping(String indexName, String settingsJson, String
mappingJson) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public String customAnalyzing(String indexName, String str) throws
InterruptedException, ExecutionException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public String customAnalyzing(String indexName, String analyzer, String str)
throws InterruptedException, ExecutionException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#customAnalyzing(java.lang.String,
java.util.List)
+ */
+ @Override
+ public List<String> customAnalyzing(String indexName, List<String> list)
throws InterruptedException, ExecutionException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#deleteType(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public void deleteType(String index, String type) {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getTypeListWithPrefix(java.lang.Object,
java.lang.Object)
+ */
+ @Override
+ public List<String> getTypeListWithPrefix(Object object, Object object2) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getIndexListWithPrefix(java.lang.Object)
+ */
+ @Override
+ public List<String> getIndexListWithPrefix(Object object) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#searchByQuery(java.lang.String,
java.lang.String, java.lang.String)
+ */
+ @Override
+ public String searchByQuery(String index, String type, String query) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#searchByQuery(java.lang.String,
java.lang.String, java.lang.String, java.lang.Boolean)
+ */
+ @Override
+ public String searchByQuery(String index, String type, String query, Boolean
bDetail) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#buildMeasurementHierarchies(java.util.List,
java.util.List, java.util.List, java.util.List)
+ */
+ @Override
+ public List<List<String>> buildMeasurementHierarchies(List<String> topics,
List<String> terms, List<String> variables, List<String> variableDetails) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#autoComplete(java.lang.String,
java.lang.String)
+ */
+ @Override
+ public List<String> autoComplete(String index, String term) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#close()
+ */
+ @Override
+ public void close() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.sdap.mudrod.storage.StorageDriver#refreshIndex()
+ */
+ @Override
+ public void refreshIndex() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getDocCount(java.lang.String,
java.lang.String[])
+ */
+ @Override
+ public int getDocCount(String index, String... type) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.sdap.mudrod.storage.StorageDriver#getDocCount(java.lang.String[],
java.lang.String[])
+ */
+ @Override
+ public int getDocCount(String[] index, String[] type) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+}
diff --git
a/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/package-info.java
b/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/package-info.java
new file mode 100644
index 0000000..1c48a7e
--- /dev/null
+++
b/storage/src/main/java/org/apache/sdap/mudrod/storage/solr/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage.solr;
\ No newline at end of file
diff --git a/web/pom.xml b/web/pom.xml
index 3b637b0..e6f58da 100644
--- a/web/pom.xml
+++ b/web/pom.xml
@@ -17,7 +17,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.sdap.mudrod</groupId>
+ <groupId>org.apache.sdap</groupId>
<artifactId>mudrod-web</artifactId>
<version>0.0.1-SNAPSHOT</version>
<build>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
