Modified: oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java?rev=1051347&r1=1051346&r2=1051347&view=diff ============================================================================== --- oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java (original) +++ oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java Tue Dec 21 00:42:17 2010 @@ -23,11 +23,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Vector; +import java.util.Map.Entry; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; @@ -83,8 +85,10 @@ public class CatalogServiceLocal impleme protected File pluginStorageDir; protected boolean oneCatalogFailsAllFail; protected boolean simplifyQueries; + protected boolean disableIntersectingCrossCatalogQueries; + protected int crossCatalogResultSortingThreshold; - public CatalogServiceLocal(CatalogRepository catalogRepository, IngestMapper ingestMapper, File pluginStorageDir, TransactionIdFactory transactionIdFactory, boolean restrictQueryPermissions, boolean restrictIngestPermissions, boolean oneCatalogFailsAllFail, boolean simplifyQueries) throws InstantiationException { + public CatalogServiceLocal(CatalogRepository catalogRepository, IngestMapper ingestMapper, File pluginStorageDir, TransactionIdFactory transactionIdFactory, boolean restrictQueryPermissions, boolean restrictIngestPermissions, boolean oneCatalogFailsAllFail, boolean simplifyQueries, boolean disableIntersectingCrossCatalogQueries, int crossCatalogResultSortingThreshold) throws InstantiationException { try { this.catalogs = new HashSet<Catalog>(); this.catalogsLock = new ReentrantReadWriteLock(); @@ -97,6 +101,8 @@ public class CatalogServiceLocal impleme this.setCatalogRepository(catalogRepository); this.oneCatalogFailsAllFail = oneCatalogFailsAllFail; this.simplifyQueries = simplifyQueries; + this.disableIntersectingCrossCatalogQueries = disableIntersectingCrossCatalogQueries; + this.crossCatalogResultSortingThreshold = crossCatalogResultSortingThreshold; }catch (Exception e) { e.printStackTrace(); throw new InstantiationException(e.getMessage()); @@ -664,9 +670,68 @@ public class CatalogServiceLocal impleme } public Page getPage(PageInfo pageInfo, QueryExpression queryExpression, Set<String> catalogIds) throws CatalogServiceException { - QueryPager queryPager = new QueryPager(this._query(queryExpression, catalogIds)); - queryPager.setPageInfo(pageInfo); - return this.getPage(queryExpression, catalogIds, queryPager); + if (this.disableIntersectingCrossCatalogQueries) { + try { + int totalResults = 0; + LinkedHashMap<String, Integer> catalogToSizeOfMap = new LinkedHashMap<String, Integer>(); + for (String catalogId : catalogIds) { + Catalog catalog = this.getCatalog(catalogId); + QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression); + if (qe != null) { + int catalogResultSize = catalog.sizeOf(qe); + totalResults += catalogResultSize; + catalogToSizeOfMap.put(catalogId, catalogResultSize); + } + } + + LOG.log(Level.INFO, "Routing query to catalogs as non-cross catalog intersecting queries . . ."); + if (totalResults <= this.crossCatalogResultSortingThreshold) { + List<CatalogReceipt> catalogReceipts = new Vector<CatalogReceipt>(); + for (String catalogId : catalogToSizeOfMap.keySet()) { + Catalog catalog = this.getCatalog(catalogId); + QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression); + if (qe != null) + catalogReceipts.addAll(catalog.query(qe)); + } + List<TransactionReceipt> transactionReceipts = this.getPossiblyUnindexedTransactionReceipts(catalogReceipts); + LOG.log(Level.INFO, "Sorting Query Results . . . "); + Collections.sort(transactionReceipts, new Comparator<TransactionReceipt>() { + public int compare(TransactionReceipt o1, + TransactionReceipt o2) { + return o2.getTransactionDate().compareTo(o1.getTransactionDate()); + } + }); + QueryPager queryPager = new QueryPager(transactionReceipts); + queryPager.setPageInfo(pageInfo); + return this.getPage(queryExpression, catalogIds, queryPager); + }else { + int currentIndex = 0; + int desiredStartingIndex = pageInfo.getPageNum() * pageInfo.getPageSize(); + List<CatalogReceipt> pageOfReceipts = new Vector<CatalogReceipt>(); + for (Entry<String, Integer> entry : catalogToSizeOfMap.entrySet()) { + if (desiredStartingIndex - currentIndex <= entry.getValue()) { + Catalog catalog = this.getCatalog(entry.getKey()); + QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression); + if (qe != null) { + List<CatalogReceipt> receipts = catalog.query(qe, desiredStartingIndex - currentIndex, Math.min((desiredStartingIndex - currentIndex) + pageInfo.getPageSize(), entry.getValue())); + pageOfReceipts.addAll(receipts); + if (pageOfReceipts.size() >= pageInfo.getPageSize()) + break; + } + }else { + currentIndex += entry.getValue(); + } + } + return new Page(new ProcessedPageInfo(pageInfo.getPageSize(), pageInfo.getPageNum(), totalResults), queryExpression, catalogIds, this.indexReceipts(this.getPossiblyUnindexedTransactionReceipts(pageOfReceipts))); + } + }catch (Exception e) { + throw new CatalogServiceException(e.getMessage(), e); + } + }else { + QueryPager queryPager = new QueryPager(this._query(queryExpression, catalogIds)); + queryPager.setPageInfo(pageInfo); + return this.getPage(queryExpression, catalogIds, queryPager); + } } public QueryPager query(QueryExpression queryExpression) throws CatalogServiceException { @@ -1061,6 +1126,10 @@ TR: for (CatalogReceipt catalogReceip List<QueryExpression> restrictedExpressions = new Vector<QueryExpression>(); for (QueryExpression qe : queryLogicalGroup.getExpressions()) { QueryExpression restrictedQE = this.reduceToUnderstoodExpressions(catalog, qe); + if (restrictedQE == null && queryLogicalGroup.getOperator().equals(QueryLogicalGroup.Operator.AND) && this.disableIntersectingCrossCatalogQueries) { + restrictedExpressions.clear(); + break; + } if (restrictedQE != null) restrictedExpressions.add(restrictedQE); }
Modified: oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java?rev=1051347&r1=1051346&r2=1051347&view=diff ============================================================================== --- oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java (original) +++ oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java Tue Dec 21 00:42:17 2010 @@ -52,12 +52,14 @@ public class CatalogServiceLocalFactory protected String pluginStorageDir = null; protected boolean oneCatalogFailsAllFail = false; protected boolean simplifyQueries = false; + protected boolean disableIntersectingCrossCatalogQueries = false; + protected int crossCatalogResultSortingThreshold = 200; public CatalogServiceLocalFactory() {} public CatalogServiceLocal createCatalogService() { try { - return new CatalogServiceLocal(this.catalogRepositoryFactory.createRepository(), this.ingestMapperFactory.createMapper(), new File(this.pluginStorageDir), this.transactionIdFactory, this.restrictQueryPermissions, this.restrictIngestPermissions, this.oneCatalogFailsAllFail, this.simplifyQueries); + return new CatalogServiceLocal(this.catalogRepositoryFactory.createRepository(), this.ingestMapperFactory.createMapper(), new File(this.pluginStorageDir), this.transactionIdFactory, this.restrictQueryPermissions, this.restrictIngestPermissions, this.oneCatalogFailsAllFail, this.simplifyQueries, this.disableIntersectingCrossCatalogQueries, this.crossCatalogResultSortingThreshold); }catch (Exception e) { LOG.log(Level.SEVERE, "Failed to create CatalogServiceLocal : " + e.getMessage(), e); return null; @@ -104,4 +106,15 @@ public class CatalogServiceLocalFactory this.simplifyQueries = simplifyQueries; } + @Required + public void setDisableIntersectingCrossCatalogQueries(boolean disableIntersectingCrossCatalogQueries) { + this.disableIntersectingCrossCatalogQueries = disableIntersectingCrossCatalogQueries; + } + + @Required + public void setCrossCatalogResultSortingThreshold( + int crossCatalogResultSortingThreshold) { + this.crossCatalogResultSortingThreshold = crossCatalogResultSortingThreshold; + } + } Modified: oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml?rev=1051347&r1=1051346&r2=1051347&view=diff ============================================================================== --- oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml (original) +++ oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml Tue Dec 21 00:42:17 2010 @@ -32,6 +32,8 @@ <property name="restrictIngestPermissions" value="false"/> <property name="oneCatalogFailsAllFail" value="true"/> <property name="simplifyQueries" value="true"/> + <property name="disableIntersectingCrossCatalogQueries" value="false"/> + <property name="crossCatalogResultSortingThreshold" value="200"/> </bean> <bean id="CatalogServiceClientFactory" lazy-init="true" class="org.apache.oodt.cas.catalog.system.impl.CatalogServiceClientFactory"> <property name="communicationChannelClientFactory" ref="XmlRpcClientFactory"/>
