Added:
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/DefaultSolrDirectoryManager.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/DefaultSolrDirectoryManager.java?rev=1205331&view=auto
==============================================================================
---
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/DefaultSolrDirectoryManager.java
(added)
+++
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/DefaultSolrDirectoryManager.java
Wed Nov 23 09:01:33 2011
@@ -0,0 +1,693 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.commons.solr.managed.impl;
+
+import static
org.apache.stanbol.commons.solr.managed.impl.ManagementUtils.substituteProperty;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.stanbol.commons.solr.SolrServerAdapter;
+import org.apache.stanbol.commons.solr.SolrServerAdapter.SolrCoreProperties;
+import org.apache.stanbol.commons.solr.SolrServerAdapter.SolrServerProperties;
+import org.apache.stanbol.commons.solr.managed.ManagedSolrServer;
+import
org.apache.stanbol.commons.solr.managed.standalone.StandaloneManagedSolrServer;
+import org.apache.stanbol.commons.solr.utils.ConfigUtils;
+import
org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+/**
+ * Historical. Just kept to be able to lookup the original code while debugging
+ * the new {@link ManagedSolrServerImpl} and {@link
StandaloneManagedSolrServer}
+ * implementations.<p>
+ * Hopefully I do never commit this class again. If you find it feel free to
+ * create a JIRA issue for its removal!
+ *
+ * @author Rupert Westenthaler
+ * @deprecated replaced by {@link ManagedSolrServerImpl} when running within an
+ * OSGI environment and the {@link StandaloneManagedSolrServer} if outside
+ * of an OSGI environment
+ */
+//@Component(immediate = true, metatype = true)
+//@Service
+//@Properties(value = {
+// @Property(name = ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY,
+// value = ManagedSolrServer.DEFAULT_SOLR_DATA_DIR)
+//})
+@Deprecated
+public class DefaultSolrDirectoryManager {//implements ManagedSolrServer {
+ /**
+ * The logger
+ */
+ private final Logger log =
LoggerFactory.getLogger(DefaultSolrDirectoryManager.class);
+
+ /**
+ * This key is used to store the file name of the archive supposed to
provide the data for the
+ * uninitialised index within the configuration the configuration
+ */
+ private static final String UNINITIALISED_INDEX_ARCHIVE_NAME_KEY =
"Uninitialised-Index-Archive-Name";
+
+ /**
+ * The dataFileProvider used to lookup index data.
+ */
+ @Reference
+ private DataFileProvider dataFileProvider;
+ /**
+ * The directory used by the internally managed embedded solr server. Use
{@link #lookupManagedSolrDir()}
+ * instead of using this member, because this member is not initialised
within the constructor or the
+ * {@link #activate(ComponentContext)} method.
+ */
+ private File managedSolrDir;
+
+ /**
+ * The component context. Only available when running within an OSGI
Environment and the component is
+ * active.
+ */
+ private ComponentContext componentContext;
+ /**
+ * For some functionality within this component it is important to track
if this instance operates within
+ * or outside of an OSGI environment. because of this this boolean is set
to true as soon as the first
+ * time {@link #activate(ComponentContext)} or {@link
#deactivate(ComponentContext)} is called. If someone
+ * knows a better method to check that feel free to change!
+ */
+ private boolean withinOSGI = false;
+ /**
+ * Initialising Solr Indexes with a lot of data may take some time.
Especially if the data need to be
+ * copied to the managed directory. Therefore it is important to wait for
the initialisation to be
+ * complete before opening an Solr Index on it.
+ * <p>
+ * To this set all cores that are currently initialised are added. As soon
as an initialisation completed
+ * this set is notified.
+ */
+ private Set<String> initCores = new HashSet<String>();
+ /**
+ * Holds the list of cores that where installed by using
+ * {@link #createSolrIndex(String, String, java.util.Properties)} but the
{@link DataFileProvider}
+ * could not yet provide the necessary data for the initialisation.
+ * <p>
+ * The list of uninitialised cores is stored within the data folder of the
bundle under
+ * {@link #UNINITIALISED_SITE_DIRECTORY_NAME} and loaded at activation.
+ *
+ */
+ private Map<String,java.util.Properties> uninitialisedCores = new
HashMap<String,java.util.Properties>();
+
+ /**
+ * OSGI {@link ServiceRegistration} for the internal {@link
DataFileProvider}
+ * used to load index configurations via classpath.<p>
+ * This service is registered/unregistered in the activate/deactivate
+ * method
+ */
+ private ServiceRegistration dfpServiceRegistration;
+ /**
+ * The internally managed Solr server
+ */
+ private SolrServerAdapter managedSolrServer;
+
+ public DefaultSolrDirectoryManager() {}
+
+ /**
+ * Internally used to lookup the {@link DataFileProvider} instance because
+ * this Method initialises an instance of the {@link
ClassPathSolrIndexConfigProvider}
+ * in case the component runs outside of an OSGI environment.
+ * @return the DatafileProvider
+ */
+ private DataFileProvider getDataFileProvider(){
+ if(dataFileProvider == null && !withinOSGI){
+ this.dataFileProvider = new ClassPathSolrIndexConfigProvider(null);
+ }
+ return dataFileProvider;
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#isSolrDir(java.lang.String)
+ */
+// @Override
+ public final boolean isManagedIndex(String solrIndexName) throws
IllegalStateException {
+ if (solrIndexName == null) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be NULL");
+ }
+ if (solrIndexName.isEmpty()) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
+ }
+ // first check if the directory for the parsed index exists
+ boolean exists = new File(lookupManagedSolrDir(componentContext),
solrIndexName).exists();
+ return !exists ? // if no directory exists
+ // check also if an uninitialised index was requested
+ uninitialisedCores.containsKey(solrIndexName)
+ : true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#getManagedIndices()
+ */
+// @Override
+ public final Map<String,File> getManagedIndices() throws
IllegalStateException {
+ File solrDir = lookupManagedSolrDir(componentContext);
+ String[] indexNames = solrDir.list(DirectoryFileFilter.INSTANCE);
+ Map<String,File> indexes = new HashMap<String,File>();
+ for (String indexName : indexNames) {
+ File coreSchema = new
File(solrDir,indexName+File.separatorChar+"conf"+File.separatorChar+"schema.xml");
+ File coreConf = new
File(solrDir,indexName+File.separatorChar+"conf"+File.separatorChar+"solrconfig.xml");
+ // TODO: validate that this is actually a SolrCore!
+ if(coreSchema.isFile() && coreConf.isFile()){
+ indexes.put(indexName, new File(solrDir, indexName));
+ } else {
+ log.debug("directory {} in managed Solr directory {} is no
Solr Core!",
+ indexName,solrDir);
+ }
+ }
+ // we need also add the uninitialised indexes (with a null as value)
+ for (String indexName : uninitialisedCores.keySet()) {
+ if (!indexes.containsKey(indexName)) {
+ indexes.put(indexName, null);
+ }
+ }
+ return indexes;
+ }
+// @Override
+ public boolean isInitialisedIndex(String solrIndexName) {
+ if (solrIndexName == null) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be NULL");
+ }
+ if (solrIndexName.isEmpty()) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
+ }
+ if (initCores.contains(solrIndexName)) { // if it is currently
initialised
+ return false; // return false
+ } else { // check if the dir is there
+ return new File(lookupManagedSolrDir(componentContext),
solrIndexName).exists();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ *
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#getSolrDirectory(java.lang.String)
+ */
+// @Override
+ public final File getSolrIndexDirectory(final String solrIndexName) throws
IllegalArgumentException {
+ return initSolrDirectory(solrIndexName, null, componentContext);
+ }
+
+// @Override
+ public final File createSolrIndex(final String solrIndexName,
ArchiveInputStream ais) {
+ return initSolrDirectory(solrIndexName, ais, componentContext);
+ }
+
+// @Override
+ public final File createSolrIndex(String solrIndexName,
+ String indexArchiveName,
+ java.util.Properties properties)
throws IllegalArgumentException,
+
IOException {
+ if(!ConfigUtils.isValidSolrIndexFileName(indexArchiveName)){
+ log.debug("add SolrIndexFileExtension to parsed indexArchive
{}",indexArchiveName);
+ indexArchiveName =
ConfigUtils.appandSolrIndexFileExtension(indexArchiveName, null);
+ }
+ // now add the index to the list of uninitialised
+ addUninitialisedIndex(solrIndexName, indexArchiveName, properties);
+ return initSolrDirectory(solrIndexName, null, componentContext);
+ }
+
+ /**
+ * Tries to get the {@link ArchiveInputStream} for the index from the
{@link DataFileProvider}.
+ *
+ * @param symbolicName
+ * the symbolic name of this bundle or <code>null</code> if
+ * not available (e.g. when running outside OSGI
+ * @param solrIndexName
+ * the name of the index to initialise
+ * @param properties
+ * the properties for this index. Must contain the
+ * {@link #UNINITIALISED_INDEX_ARCHIVE_NAME_KEY}.
+ * @return The {@link ArchiveInputStream} or <code>null</code> if the data
are still not available
+ * @throws IOException
+ * on any IO related error while initialising the index
+ * @throws IllegalStateException
+ * if the parsed configuration does not provide a value for
+ * {@link #UNINITIALISED_INDEX_ARCHIVE_NAME_KEY}.
+ */
+ private ArchiveInputStream lookupIndexArchive(String symbolicName,
+ String solrIndexName,
+ java.util.Properties
properties) throws IOException,
+
IllegalStateException {
+ // we need to copy the properties to a map
+ Map<String,String> propMap;
+ if (properties == null) {
+ properties = new java.util.Properties(); // create an empty
properties file
+ propMap = null;
+ } else {
+ propMap = new HashMap<String,String>();
+ for (Entry<Object,Object> entry : properties.entrySet()) {
+ propMap.put(entry.getKey().toString(), entry.getValue() !=
null ? entry.getValue().toString()
+ : null);
+ }
+ }
+ String archiveName =
properties.getProperty(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY);
+ if (archiveName == null) {
+ throw new IllegalStateException(
+ "Found uninitialised index config that does not contain
the required "
+ + UNINITIALISED_INDEX_ARCHIVE_NAME_KEY + "
property!");
+ }
+ propMap.remove(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY);// do not parse
this internal property
+ //we need to parse null as bundleSymbolic name, because we will accept
+ //index data from any bundle!
+ InputStream is = getDataFileProvider().getInputStream(null,
archiveName, propMap);
+ return is == null ? null :
ConfigUtils.getArchiveInputStream(archiveName, is);
+ }
+
+ private void addUninitialisedIndex(String indexName, String
sourceFileName, java.util.Properties config) throws IOException {
+ ComponentContext context = componentContext;
+ if (config == null) {
+ config = new java.util.Properties();
+ }
+ config.setProperty(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY,
sourceFileName);
+ synchronized (uninitialisedCores) {
+ if (uninitialisedCores.put(indexName, config) != null) {
+ removeUninitialisedIndexConfig(context, indexName); // remove
the old version
+ }
+ saveUninitialisedIndexConfig(context, indexName, config); // save
the new version
+ }
+ }
+
+ private void removeUninitialisedIndex(String indexName) {
+ ComponentContext context = componentContext;
+ synchronized (uninitialisedCores) {
+ if (uninitialisedCores.remove(indexName) != null) {
+ removeUninitialisedIndexConfig(context, indexName); // remove
the old version
+ }
+ }
+ }
+
+ /**
+ * Internally used to get/init the Solr directory of a SolrCore or the
root Solr directory (if
+ * <code>null</code> is parsed)
+ *
+ * @param solrIndexName
+ * the name of the Core or <code>null</code> to get/init the
root solr directory
+ * @param ais
+ * The Input stream of the Archive to load the index from or
<code>null</code> to load the
+ * default core configuration.
+ * @param context
+ * A reference to the component context or <code>null</code> if
running outside an OSGI
+ * container. This is needed to avoid that {@link
#deactivate(ComponentContext)} sets the
+ * context to <code>null</code> during this method does its
initialisation work.
+ * @return the Solr directory or <code>null</code> if the requested index
could not be created (e.g.
+ * because of <code>false</code> was parsed as create) or in case
this component is deactivated
+ * @throws IllegalStateException
+ * in case this method is called when this component is
running within an OSGI environment and
+ * it is deactivated or the initialisation for the parsed
index failed.
+ * @throws IllegalArgumentException
+ * if the parsed solrIndexName is <code>null</code> or empty.
+ */
+ private final File initSolrDirectory(final String solrIndexName,
+ ArchiveInputStream ais,
+ ComponentContext context) throws
IllegalStateException {
+ if (solrIndexName == null) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be NULL");
+ }
+ if (solrIndexName.isEmpty()) {
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
+ }
+ /*
+ * The name of the index used for the configuration may differ from the
+ * name of the target index.
+ * Therefore hold this name in an own Var and init it with the target
+ * index name (as default)
+ */
+ String indexConfigurationName = solrIndexName;
+ File managedCoreContainerDirectory = lookupManagedSolrDir(context);
+ File coreDir = new File(managedCoreContainerDirectory, solrIndexName);
+ if (!coreDir.exists()) {
+ // first add the index to the list of currently init cores
+ synchronized (initCores) {
+ log.debug(" > start initializing SolrIndex {}" +
solrIndexName);
+ initCores.add(solrIndexName);
+ }
+ // second check if the Index is an uninitialised one and if this
is the case
+ // try to get the ArchiveInputStream form the DataFileProvider
+ java.util.Properties uninitialisedProperties;
+ synchronized (uninitialisedCores) {
+ uninitialisedProperties =
uninitialisedCores.get(solrIndexName);
+ if (uninitialisedProperties != null) {
+ // NOTE: this may override an parsed ArchiveInputStream
+ // -> this is an error by the implementation of this class
+ // so throw an Exception to detect such errors early!
+ if (ais != null) {
+ throw new IllegalStateException(
+ "The parsed ArchiveInputStream is not null for
an uninitialised Index. "
+ + "Please report this error the the
stanbol-def mailing list!");
+ }
+ try {
+ String symbolicName = context != null ?
context.getBundleContext().getBundle()
+ .getSymbolicName() : null;
+ ais = lookupIndexArchive(symbolicName, solrIndexName,
uninitialisedProperties);
+ //we need to parse the name of the source index
+ String indexArchiveName =
uninitialisedProperties.getProperty(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY);
+ if(indexArchiveName != null){
+ //the name of the Archive (eluding extensions) MUST
+ //BE the same as the name of the folder within the
+ //archive that holds the data.
+ indexConfigurationName =
indexArchiveName.substring(0,
+ indexArchiveName.indexOf('.'));
+ }
+ } catch (Exception e) {
+ log.warn(
+ "The Index Archive for index {} not available (see
\"Stanbol Data File Provider\" Tab of the Apache Webconsole for details).",
+ solrIndexName);
+ }
+ }
+ }
+ // third do the actual initialisation work
+ try {
+ if (ais != null) {
+ ConfigUtils.copyCore(ais, coreDir, indexConfigurationName,
false);
+ // try to remove from uninitialised
+ removeUninitialisedIndex(solrIndexName);
+ //register the S0lrCore
+ if(withinOSGI){
+ SolrCoreProperties coreConfig = new
SolrCoreProperties(solrIndexName);
+ coreConfig.setCoreDir(coreDir);
+ managedSolrServer.registerCore(coreConfig);
+ }
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(String.format(
+ "Unable to copy default configuration for Solr Index %s to
the configured path %s",
+ solrIndexName == null ? "" : solrIndexName,
+ managedCoreContainerDirectory.getAbsoluteFile()), e);
+ } finally {
+ // regardless what happened remove the index from the
currently init
+ // indexes and notify all other waiting for the initialisation
+ synchronized (initCores) {
+ // initialisation done
+ initCores.remove(solrIndexName);
+ log.debug(" ... notify after trying to init SolrIndex
{}" + solrIndexName);
+ // notify that the initialisation completed or failed
+ initCores.notifyAll();
+ }
+ }
+ } else { // the dir exists
+ // check if still initialising ... and wait until the
initialisation
+ // is complete
+ synchronized (initCores) {
+ while (initCores.contains(solrIndexName)) {
+ log.info(" > wait for initialisation of SolrIndex " +
solrIndexName);
+ try {
+ initCores.wait();
+ } catch (InterruptedException e) {
+ // a core is initialised ... back to work
+ }
+ }
+ }
+ }
+ if(coreDir.isDirectory()){
+ return coreDir;
+ } else {
+ return null; //not initialised
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#getManagedSolrDir()
+ */
+// @Override
+ public File getManagedDirectory() {
+ return lookupManagedSolrDir(componentContext);
+ }
+
+ /**
+ * Lookup the location of the managed Solr directory. Also initialised the
default configuration if the
+ * directory does not yet exist.
+ *
+ * @param context
+ * A reference to the component context or <code>null</code> if
running outside an OSGI
+ * container. This is needed to avoid that {@link
#deactivate(ComponentContext)} sets the
+ * context to <code>null</code> during this method does its
initialisation work.
+ * @return the directory based on the current configuration
+ * @throws IllegalStateException
+ * in case this method is called when this component is
running within an OSGI environment and
+ * it is deactivated.
+ */
+ private File lookupManagedSolrDir(ComponentContext context) throws
IllegalStateException {
+ if (managedSolrDir == null) {
+ String configuredDataDir;
+ if (context == null) { // load via system properties
+ configuredDataDir =
System.getProperty(ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY,
ManagedSolrServer.DEFAULT_SOLR_DATA_DIR);
+ } else { // load via OSGI config
+ Object value =
context.getProperties().get(ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY);
+ if (value != null) {
+ configuredDataDir = value.toString();
+ } else {
+ configuredDataDir =
ManagedSolrServer.DEFAULT_SOLR_DATA_DIR;
+ }
+ }
+ // property substitution
+ configuredDataDir = FilenameUtils.separatorsToSystem(
+ substituteProperty(configuredDataDir,context != null ?
context.getBundleContext() : null));
+ // determine the directory holding the SolrIndex
+ /*
+ * NOTE: In case the configuredDataDir.isAbsolute()==false this
code will initialise the index
+ * relative to the "user.dir" of the application.
+ */
+ if (withinOSGI && context == null) {
+ // ensure to do not set an solrDataDir if this component is
+ // running within an OSGI environment and is deactivated
+ throw new IllegalStateException(
+ "Unable to lookup managed Solr directories when
component is deactivated!");
+ } else { // set the the absolute path
+ managedSolrDir = new File(configuredDataDir).getAbsoluteFile();
+ }
+ // check if the "solr.xml" file exists in the directory
+ File solrConf = new File(managedSolrDir, "solr.xml");
+ if (!solrConf.exists()) {
+ try {
+ if (context != null) { // load via bundle
+ managedSolrDir =
ConfigUtils.copyDefaultConfig(context.getBundleContext().getBundle(),
+ managedSolrDir, false);
+ } else { // load from jar
+ managedSolrDir =
ConfigUtils.copyDefaultConfig((Class<?>) null, managedSolrDir, false);
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(
+ String.format(
+ "Unable to copy default configuration for the
manages Solr Directory to the configured path '%s'!",
+ managedSolrDir.getAbsoluteFile()), e);
+ }
+ }
+ }
+ return managedSolrDir;
+ }
+
+ /*
+ * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Methods for storing and
+ * loading configurations for uninitialised indexes - - - - - - - - - - -
- - - - - - - - - - - - - - - -
+ * - - - - - - - - - -
+ */
+
+ /**
+ * The path to the directory used to store properties of uninitialised
referenced sites.
+ * <p>
+ * Such sites are one that are created by using {@link
#createSolrIndex(String, String, Map)} but the
+ * {@link DataFileProvider} does not yet provide the necessary data to
initialise the index.
+ * <p>
+ * This directory will store properties files with the indexName as name,
properties as extension and the
+ * properties as value
+ */
+ private final String UNINITIALISED_INDEX_DIRECTORY =
".config/uninitialised-index";
+
+ /**
+ * Saves the configuration of an uninitialised index
+ *
+ * @param context
+ * the context used to get the data storage
+ * @param indexName
+ * the name of the uninitialised index
+ * @param properties
+ * the properties of the uninitialised index
+ * @throws IOException
+ * on any error while saving the configuration
+ */
+ private void saveUninitialisedIndexConfig(ComponentContext context,
+ String indexName,
+ java.util.Properties
properties) throws IOException {
+ File uninstalledConfigDir = getUninitialisedSiteDirectory(context,
true);
+ File config = new File(uninstalledConfigDir, indexName + '.'
+ +
ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION + ".ref");
+ FileOutputStream out = null;
+ if (properties == null) { // if no config is provided
+ properties = new java.util.Properties();// save an empty one
+ }
+ try {
+ out = new FileOutputStream(config);
+ properties.store(out, null);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ /**
+ * Returns the directory used to store the configurations of uninitialised
Solr Indexes
+ *
+ * @return the directory
+ */
+ private File getUninitialisedSiteDirectory(ComponentContext
context,boolean init) {
+ File uninstalledConfigDir;
+ if (context == null) { //outside OSGI
+ // use config directory relative to the the Managed Solr Directory
+ uninstalledConfigDir = new
File(lookupManagedSolrDir(context),UNINITIALISED_INDEX_DIRECTORY).getAbsoluteFile();
+ } else { //whithin an OSGI environment
+ //use the DataFile directory of the bundle
+ uninstalledConfigDir =
context.getBundleContext().getDataFile(UNINITIALISED_INDEX_DIRECTORY);
+ }
+ log.info("SolrYard Config Directory: "+uninstalledConfigDir);
+ if (!uninstalledConfigDir.exists()) {
+ if(init){
+ if (!uninstalledConfigDir.mkdirs()) {
+ throw new IllegalStateException("Unable to create
Directory "
+ +
UNINITIALISED_INDEX_DIRECTORY
+ + "for storing information
of uninitialised Solr Indexes");
+ }
+ }
+ } else if (!uninstalledConfigDir.isDirectory()) {
+ throw new IllegalStateException("The directory " +
UNINITIALISED_INDEX_DIRECTORY
+ + "for storing uninitialised Solr
Indexes Information exists"
+ + "but is not a directory!");
+ } // else -> it exists and is a dir -> nothing todo
+ return uninstalledConfigDir;
+ }
+
+ /**
+ * Loads the configurations of uninitialised Solr Indexes
+ *
+ * @return the map with the index name as key and the properties as values
+ * @throws IOException
+ * on any error while loading the configurations
+ */
+ private Map<String,java.util.Properties>
loadUninitialisedIndexConfigs(ComponentContext context) throws IOException {
+ File uninstalledConfigDir = getUninitialisedSiteDirectory(context,
false);
+ Map<String,java.util.Properties> configs = new
HashMap<String,java.util.Properties>();
+ if (uninstalledConfigDir.exists()) {
+ for (String file : uninstalledConfigDir.list(new SuffixFileFilter(
+ ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION + ".ref"))) {
+ String indexName = file.substring(0, file.indexOf('.'));
+ java.util.Properties props = new java.util.Properties();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(new File(uninstalledConfigDir,
file));
+ props.load(is);
+ configs.put(indexName, props);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+ }
+ return configs;
+ }
+
+ /**
+ * Removes the configuration for the index with the parsed name form the
list if uninitialised indexes
+ *
+ * @param indexName
+ * the name of the index
+ * @return if the file was deleted.
+ */
+ private boolean removeUninitialisedIndexConfig(ComponentContext context,
String indexName) {
+ File configFile = new File(getUninitialisedSiteDirectory(context,
false),
+ indexName + '.' + ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION +
".ref");
+ return configFile.delete();
+ }
+
+ @Activate
+ protected void activate(ComponentContext context) throws IOException,
ParserConfigurationException, SAXException {
+ componentContext = context;
+ withinOSGI = true;
+ //lookup/init the managed SolrDir
+ File managedDir = lookupManagedSolrDir(componentContext);
+ log.info("... activate {} with Directory
{}",this.getClass(),managedDir.getAbsolutePath());
+ SolrServerProperties solrServerConfig = new
SolrServerProperties(managedDir);
+ solrServerConfig.setServerName("default");//TODO make configurable
+ managedSolrServer = new SolrServerAdapter(
+ context.getBundleContext(), solrServerConfig);
+ log.info("Managed Indices:");
+ for(Entry<String,File> entry : getManagedIndices().entrySet()){
+ /*
+ * NOTE: Managed SolrCores may be already registered in the
solr.xml.
+ * In that case they are initialised when creating the
ManagedSolrServer.
+ * If not we need to add them manually. The following if(..)
prevents
+ * that cores are registered (and therefore initialised) twice!.
+ */
+ if(!managedSolrServer.isCore(entry.getKey())){
+ SolrCoreProperties coreProperties = new
SolrCoreProperties(entry.getKey());
+ coreProperties.setCoreDir(entry.getValue());
+ managedSolrServer.registerCore(coreProperties);
+ }
+ log.info(" {}:
{}",entry.getKey(),entry.getValue().getAbsolutePath());
+ }
+ synchronized (uninitialisedCores) {
+
uninitialisedCores.putAll(loadUninitialisedIndexConfigs(componentContext));
+ }
+ log.info("Uninitialised Indices: {}",uninitialisedCores.keySet());
+ // Need our DataFileProvider before building the models
+ dfpServiceRegistration = context.getBundleContext().registerService(
+ DataFileProvider.class.getName(),
+ new ClassPathSolrIndexConfigProvider(
+ context.getBundleContext().getBundle().getSymbolicName()),
null);
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext context) {
+ if(dfpServiceRegistration != null) {
+ dfpServiceRegistration.unregister();
+ dfpServiceRegistration = null;
+ }
+ synchronized (uninitialisedCores) {
+ uninitialisedCores.clear();
+ }
+ managedSolrServer.shutdown();
+ managedSolrServer = null;
+ componentContext = null;
+
+ }
+}
Propchange:
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/DefaultSolrDirectoryManager.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/ManagedIndexMetadata.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/ManagedIndexMetadata.java?rev=1205331&view=auto
==============================================================================
---
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/ManagedIndexMetadata.java
(added)
+++
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/ManagedIndexMetadata.java
Wed Nov 23 09:01:33 2011
@@ -0,0 +1,483 @@
+/**
+ *
+ */
+package org.apache.stanbol.commons.solr.managed.impl;
+
+import static
org.apache.stanbol.commons.solr.SolrConstants.PROPERTY_SERVER_NAME;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.stanbol.commons.solr.SolrConstants;
+import org.apache.stanbol.commons.solr.managed.IndexMetadata;
+import org.apache.stanbol.commons.solr.managed.ManagedIndexState;
+import org.apache.stanbol.commons.solr.managed.ManagedSolrServer;
+import org.apache.stanbol.commons.solr.utils.ConfigUtils;
+import org.osgi.framework.Constants;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Helper class that manages the configuration files for SolrIndexed managed
+ * by {@link ManagedSolrServer} implementations.<p>
+ * It can both operate within and outside an OSGI environment. In case a
+ * OSGI environment is available is uses the persistent storage area of the
+ * bundle and creates a subfolder based on the {@link Constants#SERVICE_PID}
+ * of the
+ * @author westei
+ *
+ */
+public class ManagedIndexMetadata {
+
+ private Logger log = LoggerFactory.getLogger(ManagedIndexMetadata.class);
+ /**
+ * The path to the directory used to store metadata about managed indexes.
+ * <p>
+ * This directory will store properties files with the indexName as name,
properties as extension and the
+ * properties as value
+ */
+ private static final String DEFAULT_INDEX_CONFIG_DIR =
".config/index-config";
+
+ /**
+ * This map is used synchronise access to configuration files
+ *
+ */
+ //private Map<String,int[]> configDirSync = Collections.synchronizedMap(
+ // new HashMap<String,int[]>());
+
+ private final String serverName;
+ private final ComponentContext context;
+ private final String pid;
+
+ private Map<ManagedIndexState,Map<String,IndexMetadata>> managed = new
EnumMap<ManagedIndexState,Map<String,IndexMetadata>>(ManagedIndexState.class);
+// private final Map<String,IndexMetadata> uninitialised = new
HashMap<String,IndexMetadata>();
+ private final Map<String, Collection<String>> archiveName2CoreName = new
HashMap<String,Collection<String>>();
+// private final Map<String, IndexMetadata> active = new
HashMap<String,IndexMetadata>();
+ /**
+ * The {@link #managed} is used to synchronise while reading/writing the
+ * in-memory model
+ */
+ private Object inMemoryModelLock = managed;
+
+
+ private ManagedIndexMetadata(String serverName, String
pid,ComponentContext context){
+ this.serverName = serverName;
+ this.pid = pid;
+ this.context = context;
+ //init the Maps for manageing Indexes with the different states
+ for(ManagedIndexState state : ManagedIndexState.values()){
+ managed.put(state, new HashMap<String,IndexMetadata>());
+ }
+ }
+ /**
+ * Constructor to be used outside of an OSGI context
+ * @param serverName the name of the Server
+ */
+ public ManagedIndexMetadata(String serverName){
+ this(serverName,serverName,null);
+ if(serverName == null || serverName.isEmpty()){
+ throw new IllegalArgumentException("The parsed ServerName MUST NOT
be NULL nor empty!");
+ }
+ }
+ /**
+ * The constructor to be used inside an OSGI environment.
+ * The {@link #serverName} is parsed form the {@link
SolrConstants#PROPERTY_SERVER_NAME}.
+ * @param context the context of the {@link ManagedSolrServer}
implementation
+ */
+ public ManagedIndexMetadata(ComponentContext context) {
+ this((String)context.getProperties().get(PROPERTY_SERVER_NAME),
+ (String)context.getProperties().get(Constants.SERVICE_PID),
+ context);
+ if(serverName == null){
+ throw new IllegalArgumentException("The properties of the " +
+ "ComponentContext of the ManagedSolrServer '"+serverName+
+ "'MUST contain a '"+SolrConstants.PROPERTY_SERVER_NAME+"'
value!");
+ }
+ if(pid == null){
+ throw new IllegalArgumentException("The properties of the " +
+ "ComponentContext of the ManagedSolrServer
'"+serverName+
+ "'MUST contain a '"+Constants.SERVICE_PID+"' value!");
+ }
+ Map<String,IndexMetadata> indexConfigs;
+ try {
+ indexConfigs = loadIndexConfigs();
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to load information about"
+
+ "uninitialised SolrCores for managed Solr server '"+
+ serverName+"'!",e);
+ }
+ synchronized (inMemoryModelLock) { //build the in-memory model
+ for(Entry<String,IndexMetadata> entry : indexConfigs.entrySet()){
+ //read the state from the value and
+ //put the entry to the map for the state
+ managed.get(entry.getValue().getState()).put(
+ entry.getKey(), entry.getValue());
+ for(String archiveName : entry.getValue().getIndexArchives()){
+ Collection<String> indexes =
archiveName2CoreName.get(archiveName);
+ if(indexes == null){
+ indexes = new HashSet<String>();
+ archiveName2CoreName.put(archiveName, indexes);
+ }
+ indexes.add(entry.getKey());
+ }
+ }
+ }
+ }
+ public boolean isInState(ManagedIndexState state,String indexName) {
+ synchronized (inMemoryModelLock) {
+ return managed.get(state).containsKey(indexName);
+ }
+ }
+ public Collection<String> getInState(ManagedIndexState state) {
+ synchronized (inMemoryModelLock) {
+ return new HashSet<String>(managed.get(state).keySet());
+ }
+ }
+ public boolean isManaged(String indexName){
+ synchronized (inMemoryModelLock) {
+ for(Map<String,IndexMetadata> inState : managed.values()) {
+ if(inState.containsKey(indexName)){
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ public Set<String> getManaged() {
+ Set<String> names = new HashSet<String>();
+ synchronized (inMemoryModelLock) {
+ for(Map<String,IndexMetadata> inState : managed.values()) {
+ names.addAll(inState.keySet());
+ }
+ }
+ return names;
+ }
+ /**
+ * Provides the state of an index based on the managed metadata
+ * @param indexName the name of the index
+ * @return the state of <code>null</code> if the index name is not known by
+ * the in-memory model
+ */
+ public ManagedIndexState getState(String indexName){
+ synchronized (inMemoryModelLock) {
+ for(Entry<ManagedIndexState,Map<String,IndexMetadata>> entry :
managed.entrySet()) {
+ if(entry.getValue().containsKey(indexName)){
+ return entry.getKey();
+ }
+ }
+ }
+ return null;
+ }
+ /**
+ * Getter for the metadata of all indexes in a given state. Changing the
+ * returned {@link Collection} or the Entries does not affect the state
+ * of this class.
+ * @param state the state
+ * @return the metadata of all the indexes in that state (empty if none,
+ * <code>null</code> if <code>null</code> was parsed as state)
+ */
+ public Collection<IndexMetadata> getIndexMetadata(ManagedIndexState state){
+ if(state == null){
+ return null;
+ }
+ Collection<IndexMetadata> clones = new HashSet<IndexMetadata>();
+ synchronized (inMemoryModelLock) {
+ for(IndexMetadata metadata : managed.get(state).values()){
+ IndexMetadata clone = new IndexMetadata();
+ clone.putAll(metadata);
+ clones.add(clone);
+ }
+ }
+ return clones;
+
+ }
+ public IndexMetadata getIndexMetadata(String indexName){
+ IndexMetadata metadata = null;
+ synchronized (inMemoryModelLock) {
+ Iterator<Map<String,IndexMetadata>> inStateIt =
managed.values().iterator();
+ while(metadata == null && inStateIt.hasNext()) {
+ metadata = inStateIt.next().get(indexName);
+ }
+ }
+ //we need to return a clone to prevent changes by external changes to
+ //the internal state!
+ if(metadata != null){
+ IndexMetadata clone = new IndexMetadata();
+ clone.putAll(metadata);
+ return clone;
+ } else {
+ return null;
+ }
+ }
+ @SuppressWarnings("unchecked")
+ public Collection<String> getIndexNames(String archiveName){
+ synchronized (inMemoryModelLock) {
+ Collection<String> indexNames =
archiveName2CoreName.get(archiveName);
+ return indexNames == null ?
+ (Collection<String>)Collections.EMPTY_LIST :
+ Collections.unmodifiableCollection(indexNames);
+ }
+ }
+
+ public void addUninitialisedIndex(String indexName, String
indexArchiveName, Boolean sync) throws IOException {
+ IndexMetadata config = new IndexMetadata();
+ config = new IndexMetadata();
+ config.setIndexName(indexName);
+ config.setServerName(serverName);
+ config.setState(ManagedIndexState.UNINITIALISED);
+ if(sync != null){
+ config.setSynchronized(sync);
+ }
+ //no need to clone, because we have created the instance
+ updateIndexProperties(null, config, false);
+ }
+ public void store(IndexMetadata properties){
+ updateIndexProperties(null, properties, true);
+ }
+ /**
+ * Removes the metadata of the index with that name
+ * @param name the name of the index to remove
+ * @return the removed metadata of the index of <code>null</code> if no
+ * matadata for an index with that name where present.
+ */
+ public IndexMetadata remove(String name){
+ return updateIndexProperties(name, null, true);
+ }
+ /**
+ * Adds, update and deletes index metadata
+ * @param name the name of the index (can be <code>null</code> of
properties are parsed)
+ * @param properties the properties or <code>null</code> to remove
+ * @param clone If <code>true</code> the parsed properties are cloned.
Clones
+ * are required for parsed properties to prevent external changes of
properties
+ * stored in the internal lists. Only parse <code>false</code> in case the
+ * parsed properties are already a clone
+ * @return The old {@link IndexMetadata} instance (especially usefull in
case of
+ * remove operations)
+ */
+ private IndexMetadata updateIndexProperties(String name, IndexMetadata
properties, boolean clone) {
+ if(name == null && properties == null){
+ return null;
+ }
+ if(name != null && properties != null &&
!name.equals(properties.getIndexName())){
+ throw new IllegalArgumentException("The value of the Index-Name
property '"+
+ properties.getIndexName()+"' is not the same as the parsed
name '"+
+ name+"'!");
+ }
+ if(name == null) {
+ name = properties.getIndexName();
+ }
+ //first persist
+ if(properties != null){
+ try {
+ saveIndexConfig(name, properties);
+ } catch (IOException e) {
+ log.error("Unable to store Properties (see Exception below):
{}",properties.toString());
+ throw new IllegalStateException("Unable to save Index metadata
for index '"+
+ name+"'!",e);
+ }
+ } else {
+ removeIndexConfig(name);
+ }
+ //clone is meaningless if properties are NULL
+ if(clone && properties != null){
+ IndexMetadata tmp = properties;
+ properties = new IndexMetadata();
+ properties.putAll(tmp);
+ }
+ Map<String,IndexMetadata> toAdd,toRemove;
+ IndexMetadata oldMetadata = null;
+ synchronized (inMemoryModelLock ) {
+ ManagedIndexState currentState = getState(name);
+ if(currentState != null){
+ toRemove = managed.get(currentState);
+ } else {
+ toRemove = null;
+ }
+ if(properties == null){
+ toAdd = null; //remove
+ } else {
+ ManagedIndexState newState = properties.getState();
+ toAdd = managed.get(newState);
+ }
+ //now update in-memory state
+ if(toRemove != null){
+ oldMetadata = toRemove.remove(name);
+ }
+ if(toAdd != null){
+ toAdd.put(name, properties);
+ }
+ //now update the archive name to core name mappings
+ if(oldMetadata != null){
+ for(String indexArchive : oldMetadata.getIndexArchives()){
+ Collection<String> indexes =
archiveName2CoreName.get(indexArchive);
+ if(indexes.remove(name) && indexes.isEmpty()){
+ archiveName2CoreName.remove(indexArchive);
+ }
+ }
+ }
+ if(properties != null){
+ for(String indexArchive : properties.getIndexArchives()){
+ Collection<String> indexes =
archiveName2CoreName.get(indexArchive);
+ if(indexes == null){
+ indexes = new HashSet<String>();
+ archiveName2CoreName.put(indexArchive, indexes);
+ }
+ indexes.add(name);
+ }
+ }
+ }
+ return oldMetadata;
+ }
+
+
+ /**
+ * Saves the configuration of an uninitialised index
+ *
+ * @param indexName
+ * the name of the uninitialised index
+ * @param properties
+ * the properties of the uninitialised index
+ * @throws IOException
+ * on any error while saving the configuration
+ */
+ private void saveIndexConfig(String indexName, IndexMetadata properties)
throws IOException {
+ File configDir = getIndexConfigDirectory(true);
+ File config = new File(configDir, indexName + '.'
+ + ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION + ".ref");
+ synchronized (pid) {
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(config);
+ properties.store(out, null);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+ }
+
+
+ /**
+ * Returns the directory used to store the configurations of uninitialised
Solr Indexes
+ * @param init
+ * if <code>true</code> the directory is created if needed
+ * @return the directory
+ */
+ private File getIndexConfigDirectory(boolean init) {
+ File uninstalledConfigDir;
+ if (context == null) { //outside OSGI
+ // use config directory relative to the the Managed Solr Directory
+ uninstalledConfigDir = new
File(DEFAULT_INDEX_CONFIG_DIR,pid).getAbsoluteFile();
+ } else { //whithin an OSGI environment
+ //use the DataFile directory of the bundle
+ uninstalledConfigDir =
context.getBundleContext().getDataFile(DEFAULT_INDEX_CONFIG_DIR+'/'+pid);
+ }
+ log.info("SolrYard Config Directory: "+uninstalledConfigDir);
+ if (!uninstalledConfigDir.exists()) {
+ if(init) {
+ if (!uninstalledConfigDir.mkdirs()) {
+ throw new IllegalStateException("Unable to create
Directory "
+ + DEFAULT_INDEX_CONFIG_DIR
+ + "for storing information
of uninitialised Solr Indexes");
+ }
+ }
+ } else if (!uninstalledConfigDir.isDirectory()) {
+ throw new IllegalStateException("The directory " +
DEFAULT_INDEX_CONFIG_DIR
+ + "for storing uninitialised Solr
Indexes Information exists"
+ + "but is not a directory!");
+ } // else -> it exists and is a dir -> nothing todo
+ return uninstalledConfigDir;
+ }
+
+ /**
+ * Loads the configurations of uninitialised Solr Indexes
+ *
+ * @return the map with the index name as key and the properties as values
+ * @throws IOException
+ * on any error while loading the configurations
+ */
+ private Map<String,IndexMetadata> loadIndexConfigs() throws IOException {
+ File uninstalledConfigDir = getIndexConfigDirectory(false);
+ Map<String,IndexMetadata> configs = new
HashMap<String,IndexMetadata>();
+ synchronized (pid) {
+ if (uninstalledConfigDir.exists()) {
+ for (String file : uninstalledConfigDir.list(new
SuffixFileFilter(
+ ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION + ".ref"))) {
+ String indexName = file.substring(0, file.indexOf('.'));
+ File configFile = new File(uninstalledConfigDir, file);
+ IndexMetadata props = new IndexMetadata();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(configFile);
+ props.load(is);
+ //validate Index-Name and Server-Name properties!
+ if(!indexName.equals(props.getIndexName())){
+ throw new IOException("The IndexName
'"+props.getIndexName()+
+ "within the IndexConfig file does not
correspond to the file name '"+
+ file+"'!");
+ }
+ if(!serverName.equals(props.getServerName())){
+ throw new IOException("The Name of the Referenced
Solr server '"+
+ serverName+" does not correspond with the
Server-Name value '"+
+ props.getServerName()+"' within the property
file '"+
+ file+"'!");
+ }
+ configs.put(indexName, props);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+ }
+ }
+ return configs;
+ }
+
+ /**
+ * Removes the configuration for the index with the parsed name form the
list if uninitialised indexes
+ *
+ * @param context
+ * the context used to get the data storage or <code>null</code>
+ * if not available (e.g. outside an OSGI environment)
+ * @param serverName
+ * the name of the managed solr server
+ * @param indexName
+ * the name of the index
+ * @return if the file was deleted.
+ */
+ private boolean removeIndexConfig(String indexName) {
+ File configFile = new File(getIndexConfigDirectory(false),
+ indexName + '.' + ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION +
".ref");
+ synchronized (pid) {
+ return configFile.delete();
+ }
+ }
+ @Override
+ public int hashCode() {
+ return pid.hashCode();
+ }
+ @Override
+ public boolean equals(Object o) {
+ return o instanceof ManagedIndexMetadata &&
+ ((ManagedIndexMetadata)o).pid.equals(pid);
+ }
+ @Override
+ public String toString() {
+ return String.format("ManagedCores [name:%s|pid:%s|managed:%s]",
+ serverName,pid,managed);
+ }
+
+}
\ No newline at end of file
Propchange:
incubator/stanbol/trunk/commons/solr/managed/src/main/java/org/apache/stanbol/commons/solr/managed/impl/ManagedIndexMetadata.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
