Author: ivol37 at gmail.com
Date: Tue Jan 25 17:27:49 2011
New Revision: 707
Log:
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/NutchIndex.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_schema.xml
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_solrconfig.xml
Removed:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrCoreProvider.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/schema.xml
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/solrconfig.xml
Modified:
sandbox/ivol/amdatu-searchandindex/solr/pom.xml
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
Modified: sandbox/ivol/amdatu-searchandindex/solr/pom.xml
==============================================================================
--- sandbox/ivol/amdatu-searchandindex/solr/pom.xml (original)
+++ sandbox/ivol/amdatu-searchandindex/solr/pom.xml Tue Jan 25 17:27:49 2011
@@ -88,7 +88,37 @@
<scope>provided</scope>
<type>bundle</type>
</dependency>
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <artifactId>solr-solrj</artifactId>
+ <groupId>org.apache.solr</groupId>
+ <version>1.4.0</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nutch</groupId>
+ <artifactId>nutch</artifactId>
+ <version>2.0-dev</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
+
+
+
+ <repositories>
+ <repository>
+ <id>apacherepo</id>
+ <name>Apache Repository</name>
+ <url>https://repository.apache.org/content/groups/public</url>
+ </repository>
+ </repositories>
+
<build>
<plugins>
<plugin>
@@ -102,7 +132,7 @@
<Embed-Dependency>*;scope=compile</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
<Import-Package>
- !org.mortbay.*,,
+ !org.mortbay.*,
!com.sun.*,
!javax.mail.*,
!javax.jmdns,
@@ -113,6 +143,7 @@
!com.ibm.*,
!org.apache.avalon.*,
!org.apache.log,
+ org.codehaus.stax2.*,
*
</Import-Package>
<Export-Package>
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,5 @@
+package org.amdatu.searchandindex.solr;
+
+public interface SolrRestService {
+
+}
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/NutchIndex.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/NutchIndex.java
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,5 @@
+package org.amdatu.searchandindex.solr.impl;
+
+public class NutchIndex {
+
+}
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,139 @@
+/*
+ Copyright (C) 2010 Amdatu.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.amdatu.searchandindex.solr.impl;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLStreamException;
+
+import org.amdatu.core.config.templates.ConfigTemplateManager;
+import org.amdatu.core.tenant.Tenant;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.NakamuraSolrConfig;
+import org.apache.solr.core.NakamuraSolrResourceLoader;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.schema.IndexSchema;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.service.log.LogService;
+import org.xml.sax.SAXException;
+
+public class SolrApi {
+ // Statics
+ private static final String DEFAULT_SCHEMA = "default_schema.xml";
+ private static final String DEFAULT_SOLRCONFIG = "default_solrconfig.xml";
+
+ // Service dependencies injected by the dependency manager
+ private volatile LogService m_logService;
+ private volatile BundleContext m_bundleContext;
+ private volatile ConfigTemplateManager m_configTemplateManager;
+
+ // Private members
+ private File m_workDir;
+ private CoreContainer m_coreContainer;
+
+ public SolrApi(File workDir, CoreContainer coreContainer) {
+ m_workDir = workDir;
+ m_coreContainer = coreContainer;
+ }
+
+ public void createIndex(Tenant tenant, String index) {
+ // Verify if a core for this index already exists
+ String coreDirName = tenant.getId() + "/" + index;
+ String coreName = tenant.getId() + "_" + index;
+ if (m_coreContainer.getCore(coreName) == null) {
+ try {
+ m_logService.log(LogService.LOG_INFO, "Creating new Solr index
'" + index + "' for tenant '" + tenant.getId() + "'");
+
+ // Create a new core. The name of that core will be <tenant
id>/<index name>
+ createCore(coreName, coreDirName);
+ }
+ catch (IOException e) {
+ m_logService.log(LogService.LOG_ERROR, "Could not create Solr
core configuration for '" + coreName + "'", e);
+ }
+ catch (ParserConfigurationException e) {
+ m_logService.log(LogService.LOG_ERROR, "Could not create Solr
core configuration for '" + coreName + "'", e);
+ }
+ catch (SAXException e) {
+ m_logService.log(LogService.LOG_ERROR, "Could not create Solr
core configuration for '" + coreName + "'", e);
+ }
+ } else {
+ m_logService.log(LogService.LOG_INFO, "Solr index '" + index + "'
for tenant '" + tenant.getId() + "' not created since it already exists");
+ }
+ }
+
+ public void updateIndex(Tenant tenant, String index, InputStream is)
throws XMLStreamException, SolrServerException, IOException {
+ String coreName = tenant.getId() + "_" + index;
+ SolrServer server = new EmbeddedSolrServer(m_coreContainer, coreName);
+ SolrInputDocument solrDocument = SolrUtil.fromXMLStream(is);
+ server.add(solrDocument);
+ }
+
+ private void createCore(String coreName, String coreDirName) throws
IOException, ParserConfigurationException, SAXException {
+ // Save the current context classloader
+ final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
+ try {
+ // Classloader switcheroo
+ final ClassLoader classLoader = this.getClass().getClassLoader();
+ Thread.currentThread().setContextClassLoader(classLoader);
+
+ // Prepare /conf directory for this core
+ File coreDir = new File(m_workDir, coreDirName);
+ File coreConfDir = new File(m_workDir, coreDirName + "/conf");
+ if (coreConfDir.exists()) {
+ coreConfDir.delete();
+ }
+ coreConfDir.mkdirs();
+ File coresConfigFile = new File(m_workDir,
SolrDaemonServiceImpl.SOLR_CONFIG_XML);
+
+ Bundle bundle = m_bundleContext.getBundle();
+
+ // Write the schema
+ URL schema = bundle.getResource(DEFAULT_SCHEMA);
+ File targetSchema = new File(m_workDir + "/" + coreDirName +
"/conf/schema.xml");
+ m_configTemplateManager.writeConfiguration(schema, targetSchema);
+
+ // Write the solrConfig
+ URL solrConfig = bundle.getResource(DEFAULT_SOLRCONFIG);
+ File targetSolrConfig = new File(m_workDir + "/" + coreDirName +
"/conf/solrconfig.xml");
+ m_configTemplateManager.writeConfiguration(solrConfig,
targetSolrConfig);
+
+ SolrResourceLoader loader = new
NakamuraSolrResourceLoader(m_workDir.getAbsolutePath(), classLoader);
+ SolrConfig config = new NakamuraSolrConfig(loader,
"solrconfig.xml", new FileInputStream(targetSolrConfig));
+ IndexSchema indexSchema = new IndexSchema(config, null, new
FileInputStream(targetSchema));
+ CoreDescriptor desc = new CoreDescriptor(m_coreContainer,
coreName, coreDir.getAbsolutePath());
+ SolrCore nakamuraCore = new SolrCore(coreName,
coreDir.getAbsolutePath(), config, indexSchema, desc);
+ m_coreContainer.register(coreName, nakamuraCore, false);
+ m_coreContainer.persistFile(coresConfigFile);
+ } finally {
+ // Restore classloader
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
Tue Jan 25 17:27:49 2011
@@ -21,25 +21,20 @@
import java.net.URL;
import java.util.Dictionary;
import java.util.Enumeration;
-import java.util.List;
import org.amdatu.core.config.templates.ConfigTemplateManager;
-import org.amdatu.core.tenant.Tenant;
import org.amdatu.core.tenant.TenantManagementService;
-import org.amdatu.searchandindex.solr.SolrCoreProvider;
+import org.amdatu.searchandindex.solr.SolrRestService;
import org.amdatu.searchandindex.solr.SolrService;
import org.amdatu.web.httpcontext.HttpContextServiceFactory;
import org.amdatu.web.httpcontext.ResourceProvider;
import org.apache.felix.dm.DependencyManager;
import org.apache.felix.dm.ServiceDependency;
import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.NakamuraSolrResourceLoader;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
-import org.osgi.framework.ServiceReference;
import org.osgi.service.cm.ConfigurationException;
import org.osgi.service.cm.ManagedService;
import org.osgi.service.log.LogService;
@@ -49,21 +44,18 @@
*/
public class SolrDaemonServiceImpl implements SolrService, ManagedService {
// Statics
- private static final String SOLR_CONFIG_XML = "solr.xml";
+ public static final String SOLR_CONFIG_XML = "solr.xml";
private static final String CONFIG_DIR = "conf";
// Services injected by the Felix dependency manager
private volatile LogService m_logService;
private volatile BundleContext m_bundleContext;
private volatile DependencyManager m_dependencyManager;
- private volatile TenantManagementService m_tenantService;
private volatile ConfigTemplateManager m_configTemplateManager;
// Private members
private CoreContainer m_coreContainer;
-
private File m_workDir;
- private List<Tenant> m_tenants;
/**
* The init() method is invoked by the Felix dependency manager.
@@ -77,36 +69,37 @@
// Load the URL of the storage-conf.xml and write it file using
the config template
// manager, which automatically replaces configuration entries in
that file
File storageConfigFile = new File(m_workDir, SOLR_CONFIG_XML);
- m_workDir.mkdirs();
- Bundle bundle = m_bundleContext.getBundle();
- URL url = bundle.getResource(SOLR_CONFIG_XML);
- try {
- // Replace placeholders in the solr.xml config file
- m_configTemplateManager.writeConfiguration(url,
storageConfigFile);
-
- // Solr uses this system property to find its storage location.
- System.setProperty("solr.solr.home",
storageConfigFile.getParentFile().getAbsolutePath());
-
- // Update the main config
- File mainConfigDir = new File(m_workDir, "/" + CONFIG_DIR);
- mainConfigDir.mkdirs();
-
- // Find all entries in our 'conf' directory.
- final Enumeration<URL> resources =
bundle.findEntries(CONFIG_DIR, "*.*", true);
- if (resources != null) {
- while (resources.hasMoreElements()) {
- final URL resource = resources.nextElement();
- File coreConfFile = new File(mainConfigDir,
resource.getFile().replace(CONFIG_DIR + "/", ""));
- if (!coreConfFile.exists()) {
- // Only write this file if it does not yet exist
-
m_configTemplateManager.writeConfiguration(resource, coreConfFile);
+ if (!storageConfigFile.exists()) {
+ m_workDir.mkdirs();
+ Bundle bundle = m_bundleContext.getBundle();
+ URL url = bundle.getResource(SOLR_CONFIG_XML);
+ try {
+ // Replace placeholders in the solr.xml config file
+ m_configTemplateManager.writeConfiguration(url,
storageConfigFile);
+
+ // Solr uses this system property to find its storage
location.
+ System.setProperty("solr.solr.home",
storageConfigFile.getParentFile().getAbsolutePath());
+
+ // Update the main config
+ File mainConfigDir = new File(m_workDir, "/" + CONFIG_DIR);
+ mainConfigDir.mkdirs();
+
+ // Find all entries in our 'conf' directory.
+ final Enumeration<URL> resources =
bundle.findEntries(CONFIG_DIR, "*.*", true);
+ if (resources != null) {
+ while (resources.hasMoreElements()) {
+ final URL resource = resources.nextElement();
+ File coreConfFile = new File(mainConfigDir,
resource.getFile().replace(CONFIG_DIR + "/", ""));
+ if (!coreConfFile.exists()) {
+ // Only write this file if it does not yet
exist
+
m_configTemplateManager.writeConfiguration(resource, coreConfFile);
+ }
}
}
+ } catch (IOException e) {
+ m_logService.log(LogService.LOG_ERROR, "Could not replace
configuration entries in storage-conf.xml", e);
}
- } catch (IOException e) {
- m_logService.log(LogService.LOG_ERROR, "Could not replace
configuration entries in storage-conf.xml", e);
}
-
} catch (Exception e) {
m_logService.log(LogService.LOG_ERROR, "Could not create context
component", e);
}
@@ -117,55 +110,17 @@
// Start the Solr daemon
startSolrDaemon();
+ // Register the Solr API service
+ SolrApi solrApi = registerSolrApi();
+
// Register the Solr dispatch filter
- registerRestAPI();
+ registerRestAPI(solrApi);
}
catch (Exception e) {
m_logService.log(LogService.LOG_ERROR, "Failed to start Solr
Daemon", e);
}
}
- // Invoked when a new SolrIndexProvider comes available.
- public synchronized void onIndexAdded(ServiceReference ref, Object
service) {
- if (service instanceof SolrCoreProvider) {
- SolrCoreProvider indexProvider = (SolrCoreProvider) service;
- m_logService.log(LogService.LOG_INFO, "New Solr index available: "
+ indexProvider.getCoreName());
-
- // For each tenant, prepare a Solr index (if it doesn't exist yet)
- // First create the core configuration. The name of that core will
be
- // <tenant id>/<index name>
- for (Tenant tenant : m_tenants) {
- String id = indexProvider.getCoreName();
- String core = tenant.getId() + "/" + id;
- try {
- createCoreConfiguration(core);
- }
- catch (IOException e) {
- m_logService.log(LogService.LOG_ERROR, "Could not create
Solr core configuration for '" + core + "'", e);
- }
- }
- }
-
- }
-
- public synchronized void onIndexRemoved(ServiceReference ref, Object
service) {
- }
-
- // Invoked when a new SolrIndexProvider comes available.
- public synchronized void onTenantAdded(ServiceReference ref, Object
service) {
- if (service instanceof Tenant) {
- Tenant tenant = (Tenant) service;
- m_tenants.add(tenant);
- }
- }
-
- public synchronized void onTenantRemoved(ServiceReference ref, Object
service) {
- if (service instanceof Tenant) {
- Tenant tenant = (Tenant) service;
- m_tenants.remove(tenant);
- }
- }
-
@SuppressWarnings("unchecked")
public void updated(Dictionary dictionary) throws ConfigurationException {
if (dictionary != null) {
@@ -177,29 +132,6 @@
}
}
- @SuppressWarnings("unchecked")
- private void createCoreConfiguration(String core) throws IOException {
- File coreconfdir = new File(m_workDir, core + "/conf");
- if (coreconfdir.exists()) {
- coreconfdir.delete();
- }
- coreconfdir.mkdirs();
- Bundle bundle = m_bundleContext.getBundle();
-
- // Find all entries in our 'conf' directory.
- final Enumeration<URL> resources =
bundle.findEntries("conf_core_profile", "*.*", true);
- if (resources != null) {
- while (resources.hasMoreElements()) {
- final URL resource = resources.nextElement();
- File coreconffile = new File(coreconfdir,
resource.getFile().replace("conf_core_profile/", ""));
- if (!coreconffile.exists()) {
- // Only write this file if it does not yet exist
- m_configTemplateManager.writeConfiguration(resource,
coreconffile);
- }
- }
- }
- }
-
private void startSolrDaemon() throws Exception {
// Save the current context classloader first
final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
@@ -209,14 +141,10 @@
final ClassLoader classLoader = this.getClass().getClassLoader();
Thread.currentThread().setContextClassLoader(classLoader);
- // Create the resource loader and core container
- SolrResourceLoader loader = new
NakamuraSolrResourceLoader(m_workDir.getAbsolutePath(), classLoader);
- m_coreContainer = new CoreContainer(loader);
-
// Create the core initializer
boolean abortOnConfigurationError = true;
CoreContainer.Initializer init = new CoreContainer.Initializer();
- init.initialize();
+ m_coreContainer = init.initialize();
abortOnConfigurationError = init.isAbortOnConfigurationError();
// Optionally abort if we found a sever error
@@ -243,13 +171,24 @@
}
}
+ // Register the Solr API
+ private SolrApi registerSolrApi() {
+ SolrApi solrApiService = new SolrApi(m_workDir, m_coreContainer);
+ m_dependencyManager.add(m_dependencyManager.createComponent()
+ .setImplementation(solrApiService)
+ .add(createServiceDependency(ConfigTemplateManager.class))
+ .add(createServiceDependency(LogService.class)));
+ return solrApiService;
+ }
+
// Register the REST service
- private void registerRestAPI() {
- SolrRestServiceImpl restService = new SolrRestServiceImpl(m_workDir);
+ private void registerRestAPI(SolrApi solrApi) {
+ SolrRestServiceImpl restService = new SolrRestServiceImpl(m_workDir,
solrApi);
m_dependencyManager.add(m_dependencyManager.createComponent()
.setImplementation(restService)
- .setInterface(ResourceProvider.class.getName(), null)
+ .setInterface(new String[]{SolrRestService.class.getName(),
ResourceProvider.class.getName()}, null)
.add(createServiceDependency(LogService.class))
+ .add(createServiceDependency(TenantManagementService.class))
.add(createServiceDependency(HttpContextServiceFactory.class)));
}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
Tue Jan 25 17:27:49 2011
@@ -16,25 +16,43 @@
*/
package org.amdatu.searchandindex.solr.impl;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringWriter;
+import java.io.Writer;
import java.net.URL;
import java.util.Dictionary;
import java.util.Hashtable;
import javax.servlet.Filter;
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.stream.XMLStreamException;
+import org.amdatu.core.tenant.Tenant;
+import org.amdatu.core.tenant.TenantException;
+import org.amdatu.core.tenant.TenantManagementService;
+import org.amdatu.searchandindex.solr.SolrRestService;
import org.amdatu.searchandindex.solr.osgi.Activator;
import org.amdatu.web.httpcontext.HttpContextServiceFactory;
import org.amdatu.web.httpcontext.ResourceProvider;
import org.apache.felix.dm.Component;
import org.apache.felix.dm.DependencyManager;
+import org.apache.solr.client.solrj.SolrServerException;
import org.osgi.framework.BundleContext;
+import org.osgi.framework.Constants;
import org.osgi.service.log.LogService;
-
/**
* This class provides a REST interface on the Solr index service. The REST
API can be used to create, update
* and remove indices from Solr as well as performing index queries.
@@ -42,19 +60,22 @@
* @author ivol
*/
@Path("index")
-public class SolrRestServiceImpl implements ResourceProvider {
+public class SolrRestServiceImpl implements ResourceProvider, SolrRestService {
// Services injected by the Felix dependency manager
private volatile LogService m_logService;
private volatile Component m_contextComponent;
private volatile HttpContextServiceFactory m_httpContextServiceFactory;
private volatile BundleContext m_bundleContext;
private volatile DependencyManager m_dependencyManager;
+ private volatile TenantManagementService m_tenantService;
+ private volatile SolrApi m_solrApi;
// Private members
private File m_workDir;
- public SolrRestServiceImpl(File workDir) {
+ public SolrRestServiceImpl(File workDir, SolrApi solrApi) {
m_workDir = workDir;
+ m_solrApi = solrApi;
}
public void init() {
@@ -81,9 +102,9 @@
// Registers the Solr dispatch filter
private void registerSolrDispatchFilter() {
// Create and register a http servlet filter. This filter is mapped on
.*
- Dictionary<String, String> filterProperties = new Hashtable<String,
String>();
+ Dictionary<String, Object> filterProperties = new Hashtable<String,
Object>();
filterProperties.put("pattern", "/" + Activator.RESOURCE_ID +
"(|\\?.*|/.*)");
- filterProperties.put("service.ranking", "10");
+ filterProperties.put(Constants.SERVICE_RANKING, 10);
filterProperties.put("contextId", Activator.RESOURCE_ID);
SolrDispatchFilter filter = new SolrDispatchFilter("/" +
Activator.RESOURCE_ID, m_workDir.getAbsolutePath());
@@ -102,4 +123,65 @@
public String status() {
return "Index service online";
}
+
+ @PUT
+ @Path("{index}")
+ @Produces({MediaType.APPLICATION_XML})
+ public Response createIndex(@PathParam("index") final String index,
@Context final HttpServletRequest request, InputStream is) {
+ // Resolve tenant from hostname
+ String serverName = request.getServerName();
+ try {
+ for (Tenant tenant : m_tenantService.getTenants()) {
+ if (serverName.equals(tenant.getProperties().get("hostname")))
{
+ // This is the tenant for which the index should be created
+ m_solrApi.createIndex(tenant, index);
+ if (is.available() > 0) {
+ // When more then 0 bytes are available, a document is
send along with the PUT request
+ // that should be added to this index
+ m_solrApi.updateIndex(tenant, index, is);
+ }
+ }
+ }
+ }
+ catch (TenantException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (XMLStreamException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (SolrServerException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return Response.status(Response.Status.OK).build();
+ }
+
+
+ public String convertStreamToString(InputStream is)
+ throws IOException {
+ if (is != null) {
+ Writer writer = new StringWriter();
+ char[] buffer = new char[1024];
+ try {
+ Reader reader = new BufferedReader(
+ new InputStreamReader(is, "UTF-8"));
+ int n;
+ while ((n = reader.read(buffer)) != -1) {
+ writer.write(buffer, 0, n);
+ }
+ } finally {
+ is.close();
+ }
+ return writer.toString();
+ } else {
+ return "";
+ }
+ }
+
}
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,174 @@
+package org.amdatu.searchandindex.solr.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.amdatu.core.tenant.TenantEntity;
+import org.amdatu.core.tenant.TenantException;
+import org.amdatu.core.tenant.TenantManagementService;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpException;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.solr.common.SolrInputDocument;
+
+/**
+ * Problems found with invoking REST from a service;
+ *
+ * - If the REST performs authorization checks, how do I provide login
credentials? Should I post to some
+ * login REST service first before invoking a REST API?
+ * - How do I ensure that the REST service is online? Even providing a
required service dependency on an
+ * interface implemented by the REST service won't help, as you can never tell
when exactly the REST service
+ * is picked up by Wink. In fact, this is a general problem using the
whiteboard pattern. In this case, the
+ * REST service will NEVER be available when this service is started.
Therefore a separate thread is created.
+ *
+ */
+public class SolrTest {
+ private volatile TenantManagementService m_tenantService;
+
+ public void start() {
+ try {
+ TenantEntity tenant1 = m_tenantService.getTenantById("school_1");
+ if (tenant1 == null) {
+ tenant1 = m_tenantService.createTenant("school_1", "School 1
(localhost)");
+ tenant1.putProperty("hostname", "localhost");
+ m_tenantService.updateTenant(tenant1);
+ } else {
+ tenant1.putProperty("hostname", "localhost");
+ m_tenantService.updateTenant(tenant1);
+ }
+
+ TenantEntity tenant2 = m_tenantService.getTenantById("school_2");
+ if (tenant2 == null) {
+ tenant2 = m_tenantService.createTenant("school_2", "School 2
(127.0.0.1)");
+ tenant2.putProperty("hostname", "127.0.0.1");
+ } else {
+ tenant2.putProperty("hostname", "127.0.0.1");
+ m_tenantService.updateTenant(tenant2);
+ }
+
+ // Create new thread and start it
+ new TestThread().start();
+ } catch (TenantException e) {
+ e.printStackTrace();
+ }
+ }
+
+ class TestThread extends Thread {
+ public void run() {
+ try {
+ String baseUrl1 = "http://localhost:2237/rest/index";;
+ String baseUrl2 = "http://127.0.0.1:2237/rest/index";;
+
+ boolean available = false;
+ while (!available) {
+ Thread.sleep(1000);
+ available = (invokeGET(baseUrl1).statuscode ==
HttpStatus.SC_OK);
+ }
+
+ // Create indices "course" for both tenants
+ invokePUT(baseUrl1 + "/course");
+ invokePUT(baseUrl2 + "/course");
+
+ // Index a document
+ SolrInputDocument document = new SolrInputDocument();
+ document.addField( "id", "id1", 1.0f );
+ document.addField( "name", "doc1", 1.0f );
+ document.addField( "price", 10 );
+
+ streamPUT(baseUrl1 + "/course",
SolrUtil.toXMLStream(document));
+ }
+ catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private RESTResult invokeGET(String url) {
+ return invoke(url, javax.ws.rs.HttpMethod.GET);
+ }
+
+ private RESTResult invokePUT(String url) {
+ return invoke(url, javax.ws.rs.HttpMethod.PUT);
+ }
+
+ private RESTResult invoke(String url, String httpMethod) {
+ HttpClient httpClient = new HttpClient();
+ HttpMethod method = null;
+ if (httpMethod.equals(javax.ws.rs.HttpMethod.GET)) {
+ method = new GetMethod(url);
+ } else if (httpMethod.equals(javax.ws.rs.HttpMethod.PUT)) {
+ method = new PutMethod(url);
+ } else if (httpMethod.equals(javax.ws.rs.HttpMethod.DELETE)) {
+ method = new DeleteMethod(url);
+ } else if (httpMethod.equals(javax.ws.rs.HttpMethod.POST)) {
+ method = new PostMethod(url);
+ }
+ try {
+ // Execute the method, this should return a 200
+ int statusCode = httpClient.executeMethod(method);
+ RESTResult result = new RESTResult();
+ result.statuscode = statusCode;
+ result.response = new String(method.getResponseBody(), "UTF-8");
+ return result;
+ }
+ catch (HttpException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ finally {
+ // Release the connection.
+ method.releaseConnection();
+ }
+ return null;
+ }
+
+ private RESTResult streamPUT(String url, InputStream is) {
+ HttpClient httpClient = new HttpClient();
+ PutMethod method = new PutMethod(url);
+
+ try {
+ // Execute the method, this should return a 200
+ RequestEntity requestEntity = new InputStreamRequestEntity(is);
+ method.setRequestEntity(requestEntity);
+ int statusCode = httpClient.executeMethod(method);
+ RESTResult result = new RESTResult();
+ result.statuscode = statusCode;
+ result.response = new String(method.getResponseBody(), "UTF-8");
+ return result;
+ }
+ catch (HttpException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ finally {
+ // Release the connection.
+ method.releaseConnection();
+ }
+ return null;
+ }
+
+ class RESTResult {
+ public int statuscode;
+ public String response;
+ }
+}
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,146 @@
+package org.amdatu.searchandindex.solr.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.StrUtils;
+
+public class SolrUtil {
+ public static InputStream toXMLStream(SolrInputDocument document) throws
UnsupportedEncodingException {
+ String xml = ClientUtils.toXML(document);
+ InputStream is = new ByteArrayInputStream(xml.getBytes("UTF-8"));
+ return is;
+ }
+
+ @SuppressWarnings("restriction")
+ public static SolrInputDocument fromXMLStream(InputStream is) throws
UnsupportedEncodingException, XMLStreamException {
+ // Save the current context classloader first
+ final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
+ try {
+ // overwrite the context classloader, restore it later
+ final ClassLoader classLoader = SolrUtil.class.getClassLoader();
+ Thread.currentThread().setContextClassLoader(classLoader);
+
+ XMLInputFactory factory = XMLInputFactory.newInstance();
+ XMLStreamReader parser = factory.createXMLStreamReader(is);
+ SolrInputDocument doc = null;
+ while (true) {
+ int event = parser.next();
+ switch (event) {
+ case XMLStreamConstants.END_DOCUMENT:
+ parser.close();
+ return doc;
+
+ case XMLStreamConstants.START_ELEMENT:
+ String currTag = parser.getLocalName();
+ if ("doc".equals(currTag)) {
+ doc = readDoc(parser);
+ }
+ break;
+ }
+ }
+ } finally {
+ // Restore classloader
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+
+ }
+
+
+ @SuppressWarnings("restriction")
+ private static SolrInputDocument readDoc(XMLStreamReader parser) throws
XMLStreamException {
+ SolrInputDocument doc = new SolrInputDocument();
+
+ String attrName = "";
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ attrName = parser.getAttributeLocalName(i);
+ if ("boost".equals(attrName)) {
+ doc.setDocumentBoost(
Float.parseFloat(parser.getAttributeValue(i)) );
+ } else {
+ //log.warning("Unknown attribute doc/@" + attrName);
+ }
+ }
+
+ StringBuilder text = new StringBuilder();
+ String name = null;
+ float boost = 1.0f;
+ boolean isNull = false;
+ while (true) {
+ int event = parser.next();
+ switch (event) {
+ // Add everything to the text
+ case XMLStreamConstants.SPACE:
+ case XMLStreamConstants.CDATA:
+ case XMLStreamConstants.CHARACTERS:
+ text.append( parser.getText() );
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ if ("doc".equals(parser.getLocalName())) {
+ return doc;
+ }
+ else if ("field".equals(parser.getLocalName())) {
+ if (!isNull) {
+ doc.addField(name, text.toString(), boost );
+ boost = 1.0f;
+ }
+ }
+ break;
+
+ case XMLStreamConstants.START_ELEMENT:
+ text.setLength(0);
+ String localName = parser.getLocalName();
+ if (!"field".equals(localName)) {
+ //log.warning("unexpected XML tag doc/" + localName);
+ throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "unexpected XML tag doc/" + localName);
+ }
+ boost = 1.0f;
+ String attrVal = "";
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ attrName = parser.getAttributeLocalName(i);
+ attrVal = parser.getAttributeValue(i);
+ if ("name".equals(attrName)) {
+ name = attrVal;
+ } else if ("boost".equals(attrName)) {
+ boost = Float.parseFloat(attrVal);
+ } else if ("null".equals(attrName)) {
+ isNull = StrUtils.parseBoolean(attrVal);
+ } else {
+ // log.warning("Unknown attribute doc/field/@" +
attrName);
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ public static void executeInContextClassLoader(ContextExecutor executor) {
+ // Save the current context classloader first
+ final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
+ try {
+ // overwrite the context classloader, restore it later
+ final ClassLoader classLoader = SolrUtil.class.getClassLoader();
+ Thread.currentThread().setContextClassLoader(classLoader);
+
+ // Execute stuff
+ executor.execute();
+ } finally {
+ // Restore classloader
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ interface ContextExecutor {
+ void execute();
+ }
+}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
Tue Jan 25 17:27:49 2011
@@ -17,10 +17,12 @@
package org.amdatu.searchandindex.solr.osgi;
import org.amdatu.core.config.templates.ConfigTemplateManager;
-import org.amdatu.core.tenant.Tenant;
import org.amdatu.core.tenant.TenantManagementService;
+import org.amdatu.searchandindex.solr.SolrRestService;
import org.amdatu.searchandindex.solr.SolrService;
+import org.amdatu.searchandindex.solr.impl.NutchIndex;
import org.amdatu.searchandindex.solr.impl.SolrDaemonServiceImpl;
+import org.amdatu.searchandindex.solr.impl.SolrTest;
import org.apache.felix.dm.DependencyActivatorBase;
import org.apache.felix.dm.DependencyManager;
import org.osgi.framework.BundleContext;
@@ -41,13 +43,24 @@
public void init(BundleContext context, DependencyManager manager) throws
Exception {
// Create and register the Solr service.
manager.add(createComponent()
+ .setImplementation(SolrTest.class)
+
.add(createServiceDependency().setService(SolrRestService.class).setRequired(true))
+
.add(createServiceDependency().setService(TenantManagementService.class).setRequired(true)));
+
+ // Create and register the Solr service.
+ manager.add(createComponent()
+ .setImplementation(NutchIndex.class)
+
.add(createServiceDependency().setService(SolrRestService.class).setRequired(true))
+
.add(createServiceDependency().setService(TenantManagementService.class).setRequired(true)));
+
+ // Create and register the Solr service.
+ manager.add(createComponent()
.setInterface(SolrService.class.getName(), null)
.setImplementation(SolrDaemonServiceImpl.class)
.add(createServiceDependency().setService(LogService.class).setRequired(true))
.add(createServiceDependency().setService(ConfigTemplateManager.class).setRequired(true))
.add(createServiceDependency().setService(TenantManagementService.class).setRequired(true))
- .add(createConfigurationDependency().setPid(SolrService.PID))
-
.add(createServiceDependency().setService(Tenant.class).setCallbacks("onTenantAdded",
"onTenantRemoved")));
+ .add(createConfigurationDependency().setPid(SolrService.PID)));
}
public void destroy(BundleContext context, DependencyManager manager)
throws Exception {
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_schema.xml
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_schema.xml
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default)
+ or located where the classloader for the Solr webapp can find it.
+
+ This example schema is the recommended starting point for users.
+ It should be kept correct and concise, usable out-of-the-box.
+
+ For more information, on how to customize this file, please see
+ http://wiki.apache.org/solr/SchemaXml
+
+ PERFORMANCE NOTE: this schema includes many optional features and should not
+ be used for benchmarking. To improve performance one could
+ - set stored="false" for all fields possible (esp large fields) when you
+ only need to search on the field but don't need to return the original
+ value.
+ - set indexed="false" if you don't need to search on the field, but only
+ return the field as a result of searching on other indexed fields.
+ - remove all unneeded copyField statements
+ - for best index size and searching performance, set "index" to false
+ for all general text fields, use copyField to copy them to the
+ catchall "text" field, and use that for searching.
+ - For maximum indexing performance, use the StreamingUpdateSolrServer
+ java client.
+ - Remember to run the JVM in server mode, and use a higher logging level
+ that avoids logging every request
+-->
+
+<schema name="blueconic" version="1.2">
+ <!-- attribute "name" is the name of this schema and is only used for
display purposes.
+ Applications should change this to reflect the nature of the search
collection.
+ version="1.2" is Solr's version number for the schema syntax and
semantics. It should
+ not normally be changed by applications.
+ 1.0: multiValued attribute did not exist, all fields are multiValued by
nature
+ 1.1: multiValued attribute introduced, false by default
+ 1.2: omitTermFreqAndPositions attribute introduced, true by default
except for text fields.
+ -->
+
+ <types>
+ <!-- field type definitions. The "name" attribute is
+ just a label to be used by field definitions. The "class"
+ attribute and any other attributes determine the real
+ behavior of the fieldType.
+ Class names starting with "solr" refer to java classes in the
+ org.apache.solr.analysis package.
+ -->
+
+ <!-- The StrField type is not analyzed, but indexed/stored verbatim.
+ - StrField and TextField support an optional compressThreshold which
+ limits compression (if enabled in the derived fields) to values which
+ exceed a certain size (in characters).
+ -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"
omitNorms="true"/>
+
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"
omitNorms="true"/>
+ <!--Binary data type. The data should be sent/retrieved in as Base64
encoded Strings -->
+ <fieldtype name="binary" class="solr.BinaryField"/>
+
+ <!-- The optional sortMissingLast and sortMissingFirst attributes are
+ currently supported on types that are sorted internally as strings.
+ This includes
"string","boolean","sint","slong","sfloat","sdouble","pdate"
+ - If sortMissingLast="true", then a sort on this field will cause
documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause
documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
+
+ <!--
+ Default numeric field types. For faster range queries, consider the
tint/tfloat/tlong/tdouble types.
+ -->
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
+
+ <!--
+ Numeric field types that index each value at various levels of precision
+ to accelerate range queries when the number of values between the range
+ endpoints is large. See the javadoc for NumericRangeQuery for internal
+ implementation details.
+
+ Smaller precisionStep values (specified in bits) will lead to more tokens
+ indexed per value, slightly larger index size, and faster range queries.
+ A precisionStep of 0 disables indexing at different precision levels.
+ -->
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
+
+ <!-- The format for this date field is of the form 1995-12-31T23:59:59Z,
and
+ is a more restricted form of the canonical representation of dateTime
+ http://www.w3.org/TR/xmlschema-2/#dateTime
+ The trailing "Z" designates UTC time and is mandatory.
+ Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+ All other components are mandatory.
+
+ Expressions can also be used to denote calculations that should be
+ performed relative to "NOW" to determine the value, ie...
+
+ NOW/HOUR
+ ... Round to the start of the current hour
+ NOW-1DAY
+ ... Exactly 1 day prior to now
+ NOW/DAY+6MONTHS+3DAYS
+ ... 6 months and 3 days in the future from the start of
+ the current day
+
+ Consult the DateField javadocs for more information.
+
+ Note: For faster range queries, consider the tdate type
+ -->
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true"
precisionStep="0" positionIncrementGap="0"/>
+
+ <!-- A Trie based date field for faster date range queries and date
faceting. -->
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true"
precisionStep="6" positionIncrementGap="0"/>
+
+
+ <!--
+ Note:
+ These should only be used for compatibility with existing indexes
(created with older Solr versions)
+ or if "sortMissingFirst" or "sortMissingLast" functionality is needed.
Use Trie based fields instead.
+
+ Plain numeric field types that store and index the text
+ value verbatim (and hence don't support range queries, since the
+ lexicographic ordering isn't equal to the numeric ordering)
+ -->
+ <fieldType name="pint" class="solr.IntField" omitNorms="true"/>
+ <fieldType name="plong" class="solr.LongField" omitNorms="true"/>
+ <fieldType name="pfloat" class="solr.FloatField" omitNorms="true"/>
+ <fieldType name="pdouble" class="solr.DoubleField" omitNorms="true"/>
+ <fieldType name="pdate" class="solr.DateField" sortMissingLast="true"
omitNorms="true"/>
+
+
+ <!--
+ Note:
+ These should only be used for compatibility with existing indexes
(created with older Solr versions)
+ or if "sortMissingFirst" or "sortMissingLast" functionality is needed.
Use Trie based fields instead.
+
+ Numeric field types that manipulate the value into
+ a string value that isn't human-readable in its internal form,
+ but with a lexicographic ordering the same as the numeric ordering,
+ so that range queries work correctly.
+ -->
+ <fieldType name="sint" class="solr.SortableIntField"
sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="slong" class="solr.SortableLongField"
sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sfloat" class="solr.SortableFloatField"
sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sdouble" class="solr.SortableDoubleField"
sortMissingLast="true" omitNorms="true"/>
+
+
+ <!-- The "RandomSortField" is not used to store or search any
+ data. You can declare fields of this type it in your schema
+ to generate pseudo-random orderings of your docs for sorting
+ purposes. The ordering is generated based on the field name
+ and the version of the index, As long as the index version
+ remains unchanged, and the same field name is reused,
+ the ordering of the docs will be consistent.
+ If you want different psuedo-random orderings of documents,
+ for the same version of the index, use a dynamicField and
+ change the name
+ -->
+ <fieldType name="random" class="solr.RandomSortField" indexed="true" />
+
+ <!-- solr.TextField allows the specification of custom text analyzers
+ specified as a tokenizer and a list of token filters. Different
+ analyzers may be specified for indexing and querying.
+
+ The optional positionIncrementGap puts space between multiple fields
of
+ this type on the same document, with the purpose of preventing false
phrase
+ matching across fields.
+
+ For more info on customizing your analyzer chain, please see
+ http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+ -->
+
+ <!-- One can also specify an existing Analyzer class that has a
+ default constructor via the class attribute on the analyzer element
+ <fieldType name="text_greek" class="solr.TextField">
+ <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
+ </fieldType>
+ -->
+
+ <!-- A text field that only splits on whitespace for exact matching of
words -->
+ <fieldType name="text_ws" class="solr.TextField"
positionIncrementGap="100">
+ </fieldType>
+
+ <!-- A text field that uses WordDelimiterFilter to enable splitting and
matching of
+ words on case-change, alpha numeric boundaries, and non-alphanumeric
chars,
+ so that a query of "wifi" or "wi fi" could match a document containing
"Wi-Fi".
+ Synonyms and stopwords are customized by external files, and stemming
is enabled.
+ -->
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ </fieldType>
+
+ <!-- A general unstemmed text field - good if one does not know the
language of the field -->
+ <fieldType name="textgen" class="solr.TextField"
positionIncrementGap="100">
+ </fieldType>
+
+ <!-- A general unstemmed text field that indexes tokens normally and also
+ reversed (via ReversedWildcardFilterFactory), to enable more
efficient
+ leading wildcard queries. -->
+ <fieldType name="text_rev" class="solr.TextField"
positionIncrementGap="100">
+ </fieldType>
+
+ <!-- since fields of this type are by default not stored or indexed,
+ any data added to them will be ignored outright. -->
+ <fieldtype name="ignored" stored="false" indexed="false"
multiValued="true" class="solr.StrField" />
+ </types>
+
+
+ <fields>
+ <!-- Valid attributes for fields:
+ name: mandatory - the name for the field
+ type: mandatory - the name of a previously defined type from the
+ <types> section
+ indexed: true if this field should be indexed (searchable or sortable)
+ stored: true if this field should be retrievable
+ compressed: [false] if this field should be stored using gzip compression
+ (this will only apply if the field type is compressable; among
+ the standard field types, only TextField and StrField are)
+ multiValued: true if this field may contain multiple values per document
+ omitNorms: (expert) set to true to omit the norms associated with
+ this field (this disables length normalization and index-time
+ boosting for the field, and saves some memory). Only full-text
+ fields or fields that need an index-time boost need norms.
+ termVectors: [false] set to true to store the term vector for a
+ given field.
+ When using MoreLikeThis, fields used for similarity should be
+ stored for best performance.
+ termPositions: Store position information with the term vector.
+ This will increase storage costs.
+ termOffsets: Store offset information with the term vector. This
+ will increase storage costs.
+ default: a value that should be used if no value is specified
+ when adding a document.
+ -->
+
+ <field name="id" type="string" indexed="true" stored="true" required="true"
/>
+
+ <!-- catchall field, containing all other searchable text fields
(implemented
+ via copyField further on in this schema -->
+ <field name="text" type="text" indexed="true" stored="false"
multiValued="true"/>
+
+ <field name="timestamp" type="date" indexed="true" stored="false"
default="NOW" multiValued="false"/>
+
+ <dynamicField name="*_i" type="sint" indexed="true" stored="false"
omitNorms="true" multiValued="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="false"
multiValued="true"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="false"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="false"/>
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="false"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="false"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="false"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="false"/>
+
+ <!-- some trie-coded dynamic fields for faster range queries -->
+ <dynamicField name="*_ti" type="tint" indexed="true" stored="false"/>
+ <dynamicField name="*_tl" type="tlong" indexed="true" stored="false"/>
+ <dynamicField name="*_tf" type="tfloat" indexed="true" stored="false"/>
+ <dynamicField name="*_td" type="tdouble" indexed="true" stored="false"/>
+ <dynamicField name="*_tdt" type="tdate" indexed="true" stored="false"/>
+
+ <dynamicField name="*_pi" type="pint" indexed="true" stored="false"/>
+
+ <dynamicField name="ignored_*" type="ignored" multiValued="true"/>
+ <dynamicField name="attr_*" type="textgen" indexed="true" stored="false"
multiValued="true"/>
+
+ <dynamicField name="random_*" type="random" />
+
+ <!-- uncomment the following to ignore any fields that don't already match
an existing
+ field name or dynamic field, rather than reporting them as an error.
+ alternately, change the type="ignored" to some other type e.g. "text"
if you want
+ unknown fields indexed and/or stored by default -->
+ <!--dynamicField name="*" type="ignored" multiValued="true" /-->
+
+ </fields>
+
+ <!-- Field to use to determine and enforce document uniqueness.
+ Unless this field is marked with required="false", it will be a required
field
+ -->
+ <uniqueKey>id</uniqueKey>
+
+ <!-- field for the QueryParser to use when an explicit fieldname is absent -->
+ <defaultSearchField>text</defaultSearchField>
+
+ <!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
+ <solrQueryParser defaultOperator="OR"/>
+</schema>
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_solrconfig.xml
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/default_solrconfig.xml
Tue Jan 25 17:27:49 2011
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ http://wiki.apache.org/solr/SolrConfigXml
+
+-->
+
+<config>
+ <indexDefaults>
+ <!-- Sets the amount of RAM that may be used by Lucene indexing
+ for buffering added documents and deletions before they are
+ flushed to the Directory. -->
+ <ramBufferSizeMB>32</ramBufferSizeMB>
+ <lockType>simple</lockType>
+ <!--
+ Expert:
+ Controls how often Lucene loads terms into memory -->
+ <!--<termIndexInterval>256</termIndexInterval>-->
+ </indexDefaults>
+
+ <jmx />
+
+ <query>
+ <maxBooleanClauses>102400</maxBooleanClauses>
+ </query>
+
+ <updateHandler class="solr.DirectUpdateHandler2" />
+
+ <requestDispatcher handleSelect="true" >
+ <requestParsers enableRemoteStreaming="false"
multipartUploadLimitInKB="2048" />
+ </requestDispatcher>
+
+ <requestHandler name="standard" class="solr.StandardRequestHandler"
default="true">
+ <arr name="last-components">
+ </arr>
+ </requestHandler>
+
+ <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
+ <requestHandler name="/admin/"
class="org.apache.solr.handler.admin.AdminHandlers" />
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>solr</defaultQuery>
+ </admin>
+
+</config>
+
