This fixes [OODT-970] and [OODT-969] - Tests for Avro File Manager and solve the NPE problem in NettyTransceiver.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/36f320e3 Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/36f320e3 Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/36f320e3 Branch: refs/heads/development Commit: 36f320e35155ff38eebdde60fb90a07074ad4148 Parents: 75d292c Author: Imesha Sudasingha <[email protected]> Authored: Wed Jan 10 16:29:41 2018 +0530 Committer: Imesha Sudasingha <[email protected]> Committed: Wed Jan 10 16:29:41 2018 +0530 ---------------------------------------------------------------------- core/pom.xml | 10 + filemgr/pom.xml | 4 - filemgr/src/main/assembly/assembly.xml | 6 +- .../filemgr/system/AvroFileManagerClient.java | 335 ++----------------- .../filemgr/system/AvroFileManagerServer.java | 2 +- .../filemgr/util/RpcCommunicationFactory.java | 2 + filemgr/src/main/resources/filemgr.properties | 8 +- .../system/AbstractFileManagerServerTest.java | 172 ++++++++++ .../system/TestAvroFileManagerClient.java | 220 ++++++++++++ .../system/TestAvroFileManagerServer.java | 155 +++++++++ .../cas/filemgr/system/TestRpcFileManager.java | 1 + .../system/TestRpcFileManagerClient.java | 1 + .../filemgr/system/TestXmlRpcFileManager.java | 303 ----------------- .../system/TestXmlRpcFileManagerClient.java | 268 ++++----------- .../system/TestXmlRpcFileManagerServer.java | 149 +++++++++ .../TestDistributedXmlRpcFileManager.java | 4 +- 16 files changed, 805 insertions(+), 835 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index a21c37c..a01df20 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -75,6 +75,16 @@ the License. <version>1.7.4</version> </dependency> <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-ipc</artifactId> + <version>1.8.2</version> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.8.2</version> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>19.0</version> http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/pom.xml ---------------------------------------------------------------------- diff --git a/filemgr/pom.xml b/filemgr/pom.xml index 9d3c424..9718319 100644 --- a/filemgr/pom.xml +++ b/filemgr/pom.xml @@ -54,13 +54,10 @@ <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> - <version>1.8.1</version> </dependency> - <!-- https://mvnrepository.com/artifact/org.apache.avro/avro-ipc --> <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro-ipc</artifactId> - <version>1.8.1</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -291,7 +288,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> - <!--forkMode>pertest</forkMode--> <forkMode>pertest</forkMode> <!-- so we can DEBUG --> <argLine>-enableassertions</argLine> http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/filemgr/src/main/assembly/assembly.xml b/filemgr/src/main/assembly/assembly.xml index 07a8a9c..2e90515 100644 --- a/filemgr/src/main/assembly/assembly.xml +++ b/filemgr/src/main/assembly/assembly.xml @@ -45,9 +45,9 @@ $Id$ <fileSet> <directory>${basedir}/src/main/resources</directory> <outputDirectory>logs</outputDirectory> - <includes> - <include>REMOVE.log</include> - </includes> + <excludes> + <exclude>*/**</exclude> + </excludes> </fileSet> <fileSet> <directory>${basedir}/src/main/resources</directory> http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java index 043330a..ca3839d 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java @@ -17,20 +17,11 @@ package org.apache.oodt.cas.filemgr.system; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import org.apache.avro.AvroRemoteException; import org.apache.avro.ipc.NettyTransceiver; import org.apache.avro.ipc.Transceiver; import org.apache.avro.ipc.specific.SpecificRequestor; import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer; -import org.apache.oodt.cas.filemgr.exceptions.FileManagerException; import org.apache.oodt.cas.filemgr.structs.Element; import org.apache.oodt.cas.filemgr.structs.FileTransferStatus; import org.apache.oodt.cas.filemgr.structs.Product; @@ -58,6 +49,15 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; import org.apache.oodt.cas.filemgr.versioning.Versioner; import org.apache.oodt.cas.metadata.Metadata; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * @author radu * @@ -68,11 +68,11 @@ public class AvroFileManagerClient implements FileManagerClient { private static Logger LOG = Logger.getLogger(AvroFileManagerClient.class .getName()); - /* Avro-Rpc client */ - Transceiver client; + /** Avro-Rpc client */ + private Transceiver client; - /* proxy for the server */ - AvroFileManager proxy; + /** proxy for the server */ + private AvroFileManager proxy; /* URL where the fileManager is */ private URL fileManagerUrl; @@ -89,9 +89,8 @@ public class AvroFileManagerClient implements FileManagerClient { try { this.fileManagerUrl = url; InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(),this.fileManagerUrl.getPort()); - this.client = new NettyTransceiver(inetSocketAddress); + this.client = new NettyTransceiver(inetSocketAddress, 40000L); proxy = (AvroFileManager) SpecificRequestor.getClient(AvroFileManager.class, client); - } catch (IOException e) { e.printStackTrace(); LOG.log(Level.WARNING, "IOException when connecting to filemgr: [" @@ -99,8 +98,7 @@ public class AvroFileManagerClient implements FileManagerClient { } if (testConnection && !isAlive()) { - throw new ConnectionException("Exception connecting to filemgr: [" - + this.fileManagerUrl + "]"); + throw new ConnectionException("Exception connecting to filemgr: [" + this.fileManagerUrl + "]"); } } @@ -124,8 +122,9 @@ public class AvroFileManagerClient implements FileManagerClient { boolean success; try { - if(proxy != null) + if (proxy != null) { success = proxy.isAlive(); + } else return false; } catch (AvroRemoteException e) { LOG.log(Level.WARNING, "AvroRemoteException when connecting to filemgr: [" @@ -144,12 +143,6 @@ public class AvroFileManagerClient implements FileManagerClient { success = proxy.transferringProduct(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } return success; @@ -162,13 +155,7 @@ public class AvroFileManagerClient implements FileManagerClient { success = proxy.removeProductTransferStatus(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } - } + } return success; } @@ -179,12 +166,6 @@ public class AvroFileManagerClient implements FileManagerClient { success = this.proxy.isTransferComplete(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } return success; } @@ -196,12 +177,6 @@ public class AvroFileManagerClient implements FileManagerClient { success = this.proxy.moveProduct(AvroTypeFactory.getAvroProduct(product), newPath); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } return success; } @@ -213,12 +188,6 @@ public class AvroFileManagerClient implements FileManagerClient { success = this.proxy.modifyProduct(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return success; } @@ -230,12 +199,6 @@ public class AvroFileManagerClient implements FileManagerClient { success = this.proxy.removeProduct(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return success; } @@ -246,12 +209,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getFileTransferStatus(this.proxy.getCurrentFileTransfer()); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } } @@ -264,12 +221,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } return fileTransferStatuses; } @@ -280,12 +231,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.getProductPctTransferred(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } } @@ -295,12 +240,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.getRefPctTransferred(AvroTypeFactory.getAvroReference(reference)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } } @@ -315,12 +254,6 @@ public class AvroFileManagerClient implements FileManagerClient { )); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -330,12 +263,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProductPage(this.proxy.getFirstPage(AvroTypeFactory.getAvroProductType(type))); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -345,12 +272,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProductPage(this.proxy.getLastPage(AvroTypeFactory.getAvroProductType(type))); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -363,12 +284,6 @@ public class AvroFileManagerClient implements FileManagerClient { )); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -381,12 +296,6 @@ public class AvroFileManagerClient implements FileManagerClient { )); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -396,12 +305,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.addProductType(AvroTypeFactory.getAvroProductType(type)); } catch (AvroRemoteException e) { throw new RepositoryManagerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new RepositoryManagerException(e.getMessage()); - } } } @@ -411,12 +314,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.hasProduct(productName); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -426,12 +323,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.getNumProducts(AvroTypeFactory.getAvroProductType(type)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -444,12 +335,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return products; } @@ -463,12 +348,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return products; } @@ -479,12 +358,6 @@ public class AvroFileManagerClient implements FileManagerClient { this.proxy.setProductTransferStatus(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -494,12 +367,6 @@ public class AvroFileManagerClient implements FileManagerClient { this.proxy.addProductReferences(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -510,12 +377,6 @@ public class AvroFileManagerClient implements FileManagerClient { AvroTypeFactory.getAvroMetadata(metadata)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -529,12 +390,6 @@ public class AvroFileManagerClient implements FileManagerClient { ); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -544,12 +399,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.catalogProduct(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -559,12 +408,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getMetadata(this.proxy.getMetadata(AvroTypeFactory.getAvroProduct(product))); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -575,12 +418,6 @@ public class AvroFileManagerClient implements FileManagerClient { this.proxy.getReducedMetadata(AvroTypeFactory.getAvroProduct(product), (List<String>) elements)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -590,12 +427,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.removeFile(filePath); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } } @@ -605,12 +436,6 @@ public class AvroFileManagerClient implements FileManagerClient { return this.proxy.retrieveFile(filePath, offset, numBytes).array(); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } } @@ -620,14 +445,7 @@ public class AvroFileManagerClient implements FileManagerClient { this.proxy.transferFile(filePath, ByteBuffer.wrap(fileData), offset, numBytes); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new DataTransferException(e.getMessage()); - } } - } @Override @@ -641,14 +459,7 @@ public class AvroFileManagerClient implements FileManagerClient { } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } - } @Override @@ -660,12 +471,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new ValidationLayerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new ValidationLayerException(e.getMessage()); - } } return products; } @@ -676,12 +481,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getElement(this.proxy.getElementById(elementId)); } catch (AvroRemoteException e) { throw new ValidationLayerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new ValidationLayerException(e.getMessage()); - } } } @@ -691,12 +490,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getElement(this.proxy.getElementByName(elementName)); } catch (AvroRemoteException e) { throw new ValidationLayerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new ValidationLayerException(e.getMessage()); - } } } @@ -710,12 +503,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return queryResults; } @@ -729,12 +516,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return products; } @@ -745,12 +526,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProductType(this.proxy.getProductTypeByName(productTypeName)); } catch (AvroRemoteException e) { throw new RepositoryManagerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new RepositoryManagerException(e.getMessage()); - } } } @@ -760,12 +535,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProductType(this.proxy.getProductTypeById(productTypeId)); } catch (AvroRemoteException e) { throw new RepositoryManagerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new RepositoryManagerException(e.getMessage()); - } } } @@ -778,12 +547,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new RepositoryManagerException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new RepositoryManagerException(e.getMessage()); - } } return productTypes; } @@ -797,12 +560,6 @@ public class AvroFileManagerClient implements FileManagerClient { } } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } return references; } @@ -813,12 +570,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProduct(this.proxy.getProductById(productId)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -828,12 +579,6 @@ public class AvroFileManagerClient implements FileManagerClient { return AvroTypeFactory.getProduct(this.proxy.getProductByName(productName)); } catch (AvroRemoteException e) { throw new CatalogException(e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new CatalogException(e.getMessage()); - } } } @@ -848,9 +593,7 @@ public class AvroFileManagerClient implements FileManagerClient { clientTransfer); if (clientTransfer) { - LOG.log(Level.FINEST, - "File Manager Client: clientTransfer enabled: " - + "transfering product [" + LOG.log(Level.FINEST, "File Manager Client: clientTransfer enabled: transfering product [" + product.getProductName() + "]"); // we need to transfer the product ourselves @@ -948,60 +691,28 @@ public class AvroFileManagerClient implements FileManagerClient { } throw new Exception("Failed to ingest product [" + product + "] : " + e.getMessage()); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new FileManagerException(e.getMessage()); - } } - } @Override public Metadata getCatalogValues(Metadata metadata, ProductType productType) throws Exception { - try { - return AvroTypeFactory.getMetadata(this.proxy.getCatalogValues( + return AvroTypeFactory.getMetadata(this.proxy.getCatalogValues( AvroTypeFactory.getAvroMetadata(metadata), AvroTypeFactory.getAvroProductType(productType))); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new FileManagerException(e.getMessage()); - } - } } @Override public Metadata getOrigValues(Metadata metadata, ProductType productType) throws Exception { - try { - return AvroTypeFactory.getMetadata(this.proxy.getOrigValues( + return AvroTypeFactory.getMetadata(this.proxy.getOrigValues( AvroTypeFactory.getAvroMetadata(metadata), AvroTypeFactory.getAvroProductType(productType))); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new FileManagerException(e.getMessage()); - } - } } @Override public Query getCatalogQuery(Query query, ProductType productType) throws Exception { - try { - return AvroTypeFactory.getQuery(this.proxy.getCatalogQuery( + return AvroTypeFactory.getQuery(this.proxy.getCatalogQuery( AvroTypeFactory.getAvroQuery(query), - AvroTypeFactory.getAvroProductType(productType) - )); - } finally { - try { - this.client.close(); - } catch (IOException e) { - throw new FileManagerException(e.getMessage()); - } - } + AvroTypeFactory.getAvroProductType(productType))); } @Override http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java index 386a1c4..91258d9 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java @@ -70,7 +70,7 @@ public class AvroFileManagerServer implements AvroFileManager, FileManagerServer private Server server; /* file manager tools */ - FileManager fileManager; + private FileManager fileManager; public AvroFileManagerServer(int port){ this.port = port; http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java index 861d374..0ea46b2 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java @@ -124,6 +124,8 @@ public class RpcCommunicationFactory { * @throws IOException */ public static FileManagerServer createServer(int port) throws IOException { + setPror(); + String serverFactory = System.getProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory"); http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/resources/filemgr.properties ---------------------------------------------------------------------- diff --git a/filemgr/src/main/resources/filemgr.properties b/filemgr/src/main/resources/filemgr.properties index ff12d63..2297555 100644 --- a/filemgr/src/main/resources/filemgr.properties +++ b/filemgr/src/main/resources/filemgr.properties @@ -16,10 +16,10 @@ # Configuration properties for the File Manager # rpc configuration, uncomment the avro implementations to use AvroRPC -filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory -filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory -#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory -#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory +#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory +#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory +filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory +filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory # repository factory filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java new file mode 100644 index 0000000..486532b --- /dev/null +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.system; + +import junit.framework.TestCase; +import org.apache.oodt.cas.filemgr.ingest.StdIngester; +import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.SerializableMetadata; + +import java.io.File; +import java.io.FileInputStream; +import java.net.URL; +import java.util.Properties; +import java.util.logging.Logger; + +public abstract class AbstractFileManagerServerTest extends TestCase { + + private static final Logger LOG = Logger.getLogger(AbstractFileManagerServerTest.class.getName()); + + protected static final int FM_PORT = 50001; + protected static final String transferServiceFacClass = + "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory"; + + protected FileManagerServer fileManagerServer; + private String luceneCatLoc; + private Properties initialProperties = new Properties(System.getProperties()); + + + @Override + public void setUp() throws Exception { + super.setUp(); + startFileManagerServer(); + ingestTestFile(); + } + + private void startFileManagerServer() { + Properties properties = new Properties(System.getProperties()); + + // first make sure to load properties for the file manager + // and make sure to load logging properties as well + + // set the log levels + URL loggingPropertiesUrl = this.getClass().getResource("/test.logging.properties"); + properties.setProperty("java.util.logging.config.file", new File(loggingPropertiesUrl.getFile()) + .getAbsolutePath()); + + // first load the example configuration + try { + URL filemgrPropertiesUrl = this.getClass().getResource("/filemgr.properties"); + properties.load(new FileInputStream(new File(filemgrPropertiesUrl.getFile()))); + } catch (Exception e) { + fail(e.getMessage()); + } + + // override the catalog to use: we'll use lucene + try { + URL ingestUrl = this.getClass().getResource("/ingest"); + luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath() + "/cat"; + } catch (Exception e) { + fail(e.getMessage()); + } + + properties.setProperty("filemgr.catalog.factory", "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory"); + properties.setProperty("org.apache.oodt.cas.filemgr.catalog.lucene.idxPath", luceneCatLoc); + + // now override the repo mgr policy + URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy"); + try { + properties.setProperty("org.apache.oodt.cas.filemgr.repositorymgr.dirs", + "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath()); + } catch (Exception e) { + fail(e.getMessage()); + } + + // now override the val layer ones + properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs", + "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath()); + + // set up mime repo path + URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml"); + properties.setProperty("org.apache.oodt.cas.filemgr.mime.type.repository", + new File(mimeTypesUrl.getFile()).getAbsolutePath()); + + // override expand product met + properties.setProperty("org.apache.oodt.cas.filemgr.metadata.expandProduct", + Boolean.toString(shouldExpandProduct())); + + System.setProperties(properties); + + setProperties(); + + try { + fileManagerServer = newFileManagerServer(FM_PORT); + fileManagerServer.startUp(); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Override + public void tearDown() throws Exception { + fileManagerServer.shutdown(); + fileManagerServer = null; + + // blow away lucene cat + deleteAllFiles(luceneCatLoc); + + // blow away test file + deleteAllFiles("/tmp/test.txt"); + + // Reset the System properties to initial values. + System.setProperties(initialProperties); + super.tearDown(); + } + + protected void deleteAllFiles(String startDir) { + File startDirFile = new File(startDir); + File[] delFiles = startDirFile.listFiles(); + + if (delFiles != null && delFiles.length > 0) { + for (File delFile : delFiles) { + delFile.delete(); + } + } + + startDirFile.delete(); + } + + @SuppressWarnings("Duplicates") + private void ingestTestFile() { + Metadata prodMet; + StdIngester ingester = new StdIngester(transferServiceFacClass); + + try { + URL ingestUrl = this.getClass().getResource("/ingest"); + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + URL metUrl = this.getClass().getResource("/ingest/test.txt.met"); + + prodMet = new SerializableMetadata(new FileInputStream(new File(metUrl.getFile()))); + + // now add the right file location + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); + ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File(refUrl.getFile()), prodMet); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + protected abstract void setProperties(); + + protected abstract FileManagerServer newFileManagerServer(int port) throws Exception; + + protected abstract boolean shouldExpandProduct(); +} http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java new file mode 100644 index 0000000..30796ff --- /dev/null +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.system; + +import org.apache.oodt.cas.filemgr.ingest.StdIngester; +import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.structs.ProductType; +import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery; +import org.apache.oodt.cas.filemgr.structs.query.QueryResult; +import org.apache.oodt.cas.filemgr.util.SqlParser; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.metadata.SerializableMetadata; + +import java.io.File; +import java.io.FileInputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Vector; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +public class TestAvroFileManagerClient extends AbstractFileManagerServerTest { + + /** + * @since OODT-161 + */ + public void testGetReducedMetadata() { + List<String> arrayListElems = new ArrayList<>(); + List<String> vectorElemList = new Vector<>(); + List<String> linkedListElemList = new LinkedList<>(); + + arrayListElems.add(CoreMetKeys.FILENAME); + vectorElemList.add(CoreMetKeys.FILENAME); + linkedListElemList.add(CoreMetKeys.FILENAME); + + try { + FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + + Metadata reducedMet; + List pTypes = fmc.getProductTypes(); + assertNotNull(pTypes); + assertTrue(pTypes.size() > 0); + ProductType genericFileType = fmc.getProductTypeByName("GenericFile"); + assertNotNull(genericFileType); + List products = fmc.getProductsByProductType(genericFileType); + assertNotNull(products); + assertTrue(products.size() > 0); + Product product = (Product) products.get(0); + assertNotNull(product); + + reducedMet = fmc.getReducedMetadata(product, arrayListElems); + assertNotNull(reducedMet); + assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME)); + assertEquals(reducedMet.getMap().keySet().size(), 1); + + reducedMet = fmc.getReducedMetadata(product, vectorElemList); + assertNotNull(reducedMet); + assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME)); + assertEquals(reducedMet.getMap().keySet().size(), 1); + + reducedMet = fmc.getReducedMetadata(product, linkedListElemList); + assertNotNull(reducedMet); + assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME)); + assertEquals(reducedMet.getMap().keySet().size(), 1); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + public void testIngest() throws Exception { + URL ingestUrl = this.getClass().getResource("/ingest"); + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + + Metadata prodMet = new Metadata(); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( + ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); + + StdIngester ingester = new StdIngester(transferServiceFacClass); + String productId = ingester.ingest( + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + Metadata m = fmc.getMetadata(fmc.getProductById(productId)); + assertEquals(m.getMetadata("Filename"), "test.txt"); + deleteAllFiles("/tmp/test-type"); + } + + /** + * todo This is an incomplete test. No verification done after the file is deleted + * + * @throws Exception + */ + public void testRemoveFile() throws Exception { + Path tmpFilePath = Paths.get("/tmp", "test-delete.txt"); + URL ingestUrl = this.getClass().getResource("/ingest"); + URL refUrl = this.getClass().getResource("/ingest/test-delete.txt"); + Files.copy(Paths.get(refUrl.toURI()), tmpFilePath, REPLACE_EXISTING); + + try { + Metadata prodMet = new Metadata(); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); + + StdIngester ingester = new StdIngester(transferServiceFacClass); + String productId = ingester.ingest( + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + Metadata m = fmc.getMetadata(fmc.getProductById(productId)); + assertEquals(m.getMetadata("Filename"), "test-delete.txt"); + String loc = m.getMetadata("FileLocation"); + fmc.removeFile(loc + "/" + m.getMetadata("Filename")); + fmc.getProductById(productId); + } finally { + Files.copy(tmpFilePath, Paths.get(refUrl.toURI())); + } + deleteAllFiles("/tmp/test-type"); + } + + /** + * @since OODT-404 + */ + public void testMetadataPersistence() throws Exception { + URL ingestUrl = this.getClass().getResource("/ingest"); + URL refUrl = this.getClass().getResource("/ingest/test-file-3.txt"); + URL metUrl = this.getClass().getResource("/ingest/test-file-3.txt.met"); + + Metadata prodMet; + StdIngester ingester = new StdIngester(transferServiceFacClass); + prodMet = new SerializableMetadata(new FileInputStream(metUrl.getFile())); + // now add the right file location + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); + String productId = ingester.ingest( + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + + Metadata m = fmc.getMetadata(fmc.getProductById(productId)); + assertEquals(m.getAllMetadata("TestElement").size(), 4); + assertEquals(m.getMetadata("TestElement"), "fe"); + } + + public void testComplexQuery() throws Exception { + URL ingestUrl = this.getClass().getResource("/ingest"); + URL refUrl1 = this.getClass().getResource("/ingest/test-file-1.txt"); + URL refUrl2 = this.getClass().getResource("/ingest/test-file-2.txt"); + + StdIngester ingester = new StdIngester(transferServiceFacClass); + + //ingest first file + Metadata prodMet = new Metadata(); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILENAME, "test-file-1.txt"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile1"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); + ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File(refUrl1.getFile()), prodMet); + + //ingest second file + prodMet.replaceMetadata(CoreMetKeys.FILENAME, "test-file-2.txt"); + prodMet.replaceMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile2"); + ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File(refUrl2.getFile()), prodMet); + + //perform complex query + ComplexQuery complexQuery = new ComplexQuery(); + List<String> reducedMetadata = new Vector<String>(); + reducedMetadata.add(CoreMetKeys.FILENAME); + complexQuery.setReducedMetadata(reducedMetadata); + List<String> productTypeNames = new Vector<String>(); + productTypeNames.add("GenericFile"); + complexQuery.setReducedProductTypeNames(productTypeNames); + complexQuery.setSortByMetKey(CoreMetKeys.FILENAME); + complexQuery.setToStringResultFormat("$" + CoreMetKeys.FILENAME); + complexQuery.addCriterion(SqlParser.parseSqlWhereClause("Filename != 'test.txt'")); + FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + List<QueryResult> queryResults = fmc.complexQuery(complexQuery); + assertEquals("[test-file-1.txt, test-file-2.txt]", queryResults.toString()); + } + + @Override + protected void setProperties() { + System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory"); + System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory"); + } + + @Override + protected FileManagerServer newFileManagerServer(int port) throws Exception { + return new AvroFileManagerServer(port); + } + + @Override + protected boolean shouldExpandProduct() { + return false; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java new file mode 100644 index 0000000..9cbf617 --- /dev/null +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.system; + +import org.apache.oodt.cas.filemgr.metadata.ProductMetKeys; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; +import org.apache.oodt.cas.metadata.Metadata; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class TestAvroFileManagerServer extends AbstractFileManagerServerTest { + + private static Logger LOG = Logger.getLogger(TestAvroFileManagerServer.class.getName()); + + public void testExpandProductMet() { + AvroFileManagerClient fmc = null; + try { + fmc = new AvroFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + } catch (Exception e) { + fail(e.getMessage()); + } + + Metadata met = null; + try { + met = fmc.getMetadata(fmc.getProductByName("test.txt")); + } catch (CatalogException e) { + LOG.log(Level.SEVERE, e.getMessage()); + fail(e.getMessage()); + } + + assertNotNull(met); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID)); + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME)); + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE)); + assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); + assertEquals(Product.STATUS_RECEIVED, met + .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); + + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES)); + + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS) + .size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES) + .size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES) + .size()); + + URL refUrl = this.getClass().getResource("/ingest/test.txt"); + + String origPath = null; + try { + origPath = new File(refUrl.getFile()).getCanonicalPath(); + } catch (IOException e) { + fail(e.getMessage()); + } + assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS)); + assertEquals("/tmp/test.txt/test.txt", met + .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)); + + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)); + assertEquals("text/plain", met + .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)); + + try { + met = fmc.getReducedMetadata(fmc.getProductByName("test.txt"), + Collections.EMPTY_LIST); + } catch (CatalogException e) { + LOG.log(Level.SEVERE, e.getMessage()); + fail(e.getMessage()); + } + + assertNotNull(met); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID)); + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME)); + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE)); + assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); + assertEquals(Product.STATUS_RECEIVED, met + .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); + + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES)); + assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES)); + + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS) + .size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES) + .size()); + assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES) + .size()); + + origPath = null; + try { + origPath = new File(refUrl.getFile()).getCanonicalPath(); + } catch (IOException e) { + fail(e.getMessage()); + } + assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS)); + assertEquals("/tmp/test.txt/test.txt", met + .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)); + + assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)); + assertEquals("text/plain", met + .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)); + + } + + @Override + protected void setProperties() { + System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory"); + System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory"); + } + + @Override + protected FileManagerServer newFileManagerServer(int port) throws Exception { + return new AvroFileManagerServer(port); + } + + @Override + protected boolean shouldExpandProduct() { + return true; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java index ee14ef7..256f019 100644 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java @@ -44,6 +44,7 @@ import junit.framework.TestCase; * @since OODT-72 * */ +@Deprecated public class TestRpcFileManager extends TestCase { private static final int FM_PORT = 50002; http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java index 18e3cc9..bf60141 100644 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java @@ -51,6 +51,7 @@ import junit.framework.TestCase; * Test suite for the {@link XmlRpcFileManagerClient} * </p>. */ +@Deprecated public class TestRpcFileManagerClient extends TestCase { private static int FM_PORT = 50001; http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java deleted file mode 100644 index 1e3db7e..0000000 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java +++ /dev/null @@ -1,303 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oodt.cas.filemgr.system; - -import org.apache.oodt.cas.filemgr.ingest.StdIngester; -import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; -import org.apache.oodt.cas.filemgr.metadata.ProductMetKeys; -import org.apache.oodt.cas.filemgr.structs.Product; -import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; -import org.apache.oodt.cas.metadata.Metadata; -import org.apache.oodt.cas.metadata.SerializableMetadata; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URL; -import java.util.Collections; -import java.util.Properties; -import java.util.logging.Level; -import java.util.logging.Logger; - -import junit.framework.TestCase; - - -/** - * - * Test harness for the XmlRpcFileManager. - * - * @since OODT-72 - * - */ -public class TestXmlRpcFileManager extends TestCase { - private static Logger LOG = Logger.getLogger(TestXmlRpcFileManager.class.getName()); - private static final int FM_PORT = 50001; - - private XmlRpcFileManager fm; - - private String luceneCatLoc; - - private static final String transferServiceFacClass = - "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory"; - - private Properties initialProperties = new Properties( - System.getProperties()); - - /** - * @since OODT-72 - */ - public void testExpandProductMet() { - XmlRpcFileManagerClient fmc = null; - try { - fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); - } catch (Exception e) { - fail(e.getMessage()); - } - - Metadata met = null; - try { - met = fmc.getMetadata(fmc.getProductByName("test.txt")); - } catch (CatalogException e) { - LOG.log(Level.SEVERE, e.getMessage()); - fail(e.getMessage()); - } - - assertNotNull(met); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID)); - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME)); - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE)); - assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); - assertEquals(Product.STATUS_RECEIVED, met - .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); - - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES)); - - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS) - .size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES) - .size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES) - .size()); - - URL refUrl = this.getClass().getResource("/ingest/test.txt"); - - String origPath = null; - try { - origPath = new File(refUrl.getFile()).getCanonicalPath(); - } catch (IOException e) { - fail(e.getMessage()); - } - assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS)); - assertEquals("/tmp/test.txt/test.txt", met - .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)); - - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)); - assertEquals("text/plain", met - .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)); - - try { - met = fmc.getReducedMetadata(fmc.getProductByName("test.txt"), - Collections.EMPTY_LIST); - } catch (CatalogException e) { - LOG.log(Level.SEVERE, e.getMessage()); - fail(e.getMessage()); - } - - assertNotNull(met); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID)); - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME)); - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE)); - assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); - assertEquals(Product.STATUS_RECEIVED, met - .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS)); - - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES)); - assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES)); - - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS) - .size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES) - .size()); - assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES) - .size()); - - origPath = null; - try { - origPath = new File(refUrl.getFile()).getCanonicalPath(); - } catch (IOException e) { - fail(e.getMessage()); - } - assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS)); - assertEquals("/tmp/test.txt/test.txt", met - .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)); - - assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)); - assertEquals("text/plain", met - .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)); - - } - - /* - * (non-Javadoc) - * - * @see junit.framework.TestCase#setUp() - */ - @Override - public void setUp() throws Exception { - startXmlRpcFileManager(); - ingestTestFile(); - } - - /* - * (non-Javadoc) - * - * @see junit.framework.TestCase#tearDown() - */ - @Override - public void tearDown() throws Exception { - fm.shutdown(); - fm = null; - - // blow away lucene cat - deleteAllFiles(luceneCatLoc); - - // blow away test file - deleteAllFiles("/tmp/test.txt"); - - // Reset the System properties to initial values. - System.setProperties(initialProperties); - } - - private void deleteAllFiles(String startDir) { - File startDirFile = new File(startDir); - File[] delFiles = startDirFile.listFiles(); - - if (delFiles != null && delFiles.length > 0) { - for (File delFile : delFiles) { - delFile.delete(); - } - } - - startDirFile.delete(); - - } - - private void ingestTestFile() { - Metadata prodMet; - StdIngester ingester = new StdIngester(transferServiceFacClass); - - try { - URL ingestUrl = this.getClass().getResource("/ingest"); - URL refUrl = this.getClass().getResource("/ingest/test.txt"); - URL metUrl = this.getClass().getResource("/ingest/test.txt.met"); - - prodMet = new SerializableMetadata(new FileInputStream( - new File(metUrl.getFile()))); - - // now add the right file location - prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); - prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt"); - prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); - ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File( - refUrl.getFile()), prodMet); - } catch (Exception e) { - fail(e.getMessage()); - } - } - - private void startXmlRpcFileManager() { - - Properties properties = new Properties(System.getProperties()); - - // first make sure to load properties for the file manager - // and make sure to load logging properties as well - - // set the log levels - URL loggingPropertiesUrl = this.getClass().getResource( - "/test.logging.properties"); - properties.setProperty("java.util.logging.config.file", new File( - loggingPropertiesUrl.getFile()).getAbsolutePath()); - - // first load the example configuration - try { - URL filemgrPropertiesUrl = this.getClass().getResource( - "/filemgr.properties"); - properties.load( - new FileInputStream(new File(filemgrPropertiesUrl.getFile()))); - } catch (Exception e) { - fail(e.getMessage()); - } - - // override the catalog to use: we'll use lucene - try { - URL ingestUrl = this.getClass().getResource("/ingest"); - luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath() + "/cat"; - } catch (Exception e) { - fail(e.getMessage()); - } - - properties.setProperty("filemgr.catalog.factory", - "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory"); - properties.setProperty("org.apache.oodt.cas.filemgr.catalog.lucene.idxPath", - luceneCatLoc); - - // now override the repo mgr policy - URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy"); - try { - properties.setProperty("org.apache.oodt.cas.filemgr.repositorymgr.dirs", - "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath()); - } catch (Exception e) { - fail(e.getMessage()); - } - - // now override the val layer ones - properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs", - "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath()); - - // set up mime repo path - URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml"); - properties.setProperty("org.apache.oodt.cas.filemgr.mime.type.repository", - new File(mimeTypesUrl.getFile()).getAbsolutePath()); - - // override expand product met - properties.setProperty("org.apache.oodt.cas.filemgr.metadata.expandProduct", - "true"); - - System.setProperties(properties); - - try { - fm = new XmlRpcFileManager(FM_PORT); - } catch (Exception e) { - fail(e.getMessage()); - } - } -} http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java index 8ad4c70..c368d48 100644 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java @@ -19,6 +19,7 @@ package org.apache.oodt.cas.filemgr.system; //OODT imports + import org.apache.oodt.cas.filemgr.ingest.StdIngester; import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; import org.apache.oodt.cas.filemgr.structs.Product; @@ -29,62 +30,47 @@ import org.apache.oodt.cas.filemgr.util.SqlParser; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.SerializableMetadata; -//JDK imports import java.io.File; import java.io.FileInputStream; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import java.util.Properties; import java.util.Vector; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +//JDK imports //Junit imports -import junit.framework.TestCase; /** * @author mattmann * @version $Revision$ - * + * <p> * <p> * Test suite for the {@link XmlRpcFileManagerClient} * </p>. */ -public class TestXmlRpcFileManagerClient extends TestCase { - - private static final int FM_PORT = 50001; - - private XmlRpcFileManager fm; - - private String luceneCatLoc; - - private static final String transferServiceFacClass = "org.apache.oodt.cas." - + "filemgr.datatransfer.LocalDataTransferFactory"; - - private Properties initialProperties = new Properties( - System.getProperties()); - - public TestXmlRpcFileManagerClient() { - } +public class TestXmlRpcFileManagerClient extends AbstractFileManagerServerTest { - /** * @since OODT-161 - * */ public void testGetReducedMetadata() { - List arrayListElems = new ArrayList(); - List vectorElemList = new Vector(); - List linkedListElemList = new LinkedList(); + List<String> arrayListElems = new ArrayList<>(); + List<String> vectorElemList = new Vector<>(); + List<String> linkedListElemList = new LinkedList<>(); arrayListElems.add(CoreMetKeys.FILENAME); vectorElemList.add(CoreMetKeys.FILENAME); linkedListElemList.add(CoreMetKeys.FILENAME); try { - XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL( - "http://localhost:"; + FM_PORT)); - + FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + Metadata reducedMet; List pTypes = fmc.getProductTypes(); assertNotNull(pTypes); @@ -111,66 +97,63 @@ public class TestXmlRpcFileManagerClient extends TestCase { assertNotNull(reducedMet); assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME)); assertEquals(reducedMet.getMap().keySet().size(), 1); - } catch (Exception e) { fail(e.getMessage()); - }finally { - fm.shutdown(); } - } - + public void testIngest() throws Exception { URL ingestUrl = this.getClass().getResource("/ingest"); URL refUrl = this.getClass().getResource("/ingest/test.txt"); Metadata prodMet = new Metadata(); prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); + ingestUrl.getFile()).getCanonicalPath()); prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt"); prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile"); prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); StdIngester ingester = new StdIngester(transferServiceFacClass); String productId = ingester.ingest( - new URL("http://localhost:"; + FM_PORT), - new File(refUrl.getFile()), prodMet); - XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL( - "http://localhost:"; + FM_PORT)); + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); Metadata m = fmc.getMetadata(fmc.getProductById(productId)); assertEquals(m.getMetadata("Filename"), "test.txt"); deleteAllFiles("/tmp/test-type"); } public void testRemoveFile() throws Exception { + Path tmpFilePath = Paths.get("/tmp", "test-delete.txt"); URL ingestUrl = this.getClass().getResource("/ingest"); URL refUrl = this.getClass().getResource("/ingest/test-delete.txt"); + Files.copy(Paths.get(refUrl.toURI()), tmpFilePath, REPLACE_EXISTING); - Metadata prodMet = new Metadata(); - prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); - prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt"); - prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile"); - prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); - - StdIngester ingester = new StdIngester(transferServiceFacClass); - String productId = ingester.ingest( - new URL("http://localhost:"; + FM_PORT), - new File(refUrl.getFile()), prodMet); - XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL( - "http://localhost:"; + FM_PORT)); - Metadata m = fmc.getMetadata(fmc.getProductById(productId)); - assertEquals(m.getMetadata("Filename"), "test-delete.txt"); - String loc = m.getMetadata("FileLocation"); - fmc.removeFile(loc+"/"+m.getMetadata("Filename")); + try { + Metadata prodMet = new Metadata(); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile"); + prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); - fmc.getProductById(productId); + StdIngester ingester = new StdIngester(transferServiceFacClass); + String productId = ingester.ingest( + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + Metadata m = fmc.getMetadata(fmc.getProductById(productId)); + assertEquals(m.getMetadata("Filename"), "test-delete.txt"); + String loc = m.getMetadata("FileLocation"); + fmc.removeFile(loc + "/" + m.getMetadata("Filename")); + fmc.getProductById(productId); + } finally { + Files.copy(tmpFilePath, Paths.get(refUrl.toURI())); + } deleteAllFiles("/tmp/test-type"); } /** * @since OODT-404 - * */ public void testMetadataPersistence() throws Exception { URL ingestUrl = this.getClass().getResource("/ingest"); @@ -179,23 +162,19 @@ public class TestXmlRpcFileManagerClient extends TestCase { Metadata prodMet; StdIngester ingester = new StdIngester(transferServiceFacClass); - prodMet = new SerializableMetadata(new FileInputStream( - metUrl.getFile())); + prodMet = new SerializableMetadata(new FileInputStream(metUrl.getFile())); // now add the right file location - prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); String productId = ingester.ingest( - new URL("http://localhost:"; + FM_PORT), - new File(refUrl.getFile()), prodMet); - XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL( - "http://localhost:"; + FM_PORT)); - + new URL("http://localhost:"; + FM_PORT), + new File(refUrl.getFile()), prodMet); + FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); + Metadata m = fmc.getMetadata(fmc.getProductById(productId)); assertEquals(m.getAllMetadata("TestElement").size(), 4); assertEquals(m.getMetadata("TestElement"), "fe"); } - public void testComplexQuery() throws Exception { URL ingestUrl = this.getClass().getResource("/ingest"); URL refUrl1 = this.getClass().getResource("/ingest/test-file-1.txt"); @@ -205,20 +184,17 @@ public class TestXmlRpcFileManagerClient extends TestCase { //ingest first file Metadata prodMet = new Metadata(); - prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); + prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath()); prodMet.addMetadata(CoreMetKeys.FILENAME, "test-file-1.txt"); prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile1"); prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); - ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File( - refUrl1.getFile()), prodMet); - + ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File(refUrl1.getFile()), prodMet); + //ingest second file prodMet.replaceMetadata(CoreMetKeys.FILENAME, "test-file-2.txt"); prodMet.replaceMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile2"); - ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File( - refUrl2.getFile()), prodMet); - + ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File(refUrl2.getFile()), prodMet); + //perform complex query ComplexQuery complexQuery = new ComplexQuery(); List<String> reducedMetadata = new Vector<String>(); @@ -230,144 +206,24 @@ public class TestXmlRpcFileManagerClient extends TestCase { complexQuery.setSortByMetKey(CoreMetKeys.FILENAME); complexQuery.setToStringResultFormat("$" + CoreMetKeys.FILENAME); complexQuery.addCriterion(SqlParser.parseSqlWhereClause("Filename != 'test.txt'")); - XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL( - "http://localhost:"; + FM_PORT)); + FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:"; + FM_PORT)); List<QueryResult> queryResults = fmc.complexQuery(complexQuery); assertEquals("[test-file-1.txt, test-file-2.txt]", queryResults.toString()); } - /* - * (non-Javadoc) - * - * @see junit.framework.TestCase#setUp() - */ - protected void setUp() throws Exception { - startXmlRpcFileManager(); - ingestTestFile(); - } - - /* - * (non-Javadoc) - * - * @see junit.framework.TestCase#tearDown() - */ - protected void tearDown() throws Exception { - fm.shutdown(); - fm = null; - - // blow away lucene cat - deleteAllFiles(luceneCatLoc); - - // blow away test file - deleteAllFiles("/tmp/test.txt"); - - // Reset the System properties to initial values. - System.setProperties(initialProperties); - } - - private void deleteAllFiles(String startDir) { - File startDirFile = new File(startDir); - File[] delFiles = startDirFile.listFiles(); - - if (delFiles != null && delFiles.length > 0) { - for (File delFile : delFiles) { - delFile.delete(); - } - } - - startDirFile.delete(); - + @Override + protected void setProperties() { + System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory"); + System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory"); } - private void ingestTestFile() { - Metadata prodMet; - StdIngester ingester = new StdIngester(transferServiceFacClass); - - try { - URL ingestUrl = this.getClass().getResource("/ingest"); - URL refUrl = this.getClass().getResource("/ingest/test.txt"); - URL metUrl = this.getClass().getResource("/ingest/test.txt.met"); - - prodMet = new SerializableMetadata(new FileInputStream( - new File(metUrl.getFile()))); - - // now add the right file location - prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File( - ingestUrl.getFile()).getCanonicalPath()); - prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt"); - prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile"); - ingester.ingest(new URL("http://localhost:"; + FM_PORT), new File( - refUrl.getFile()), prodMet); - } catch (Exception e) { - fail(e.getMessage()); - } + @Override + protected FileManagerServer newFileManagerServer(int port) throws Exception { + return new XmlRpcFileManagerServer(port); } - private void startXmlRpcFileManager() { - - Properties properties = new Properties(System.getProperties()); - - // first make sure to load properties for the file manager - // and make sure to load logging properties as well - - // set the log levels - URL loggingPropertiesUrl = this.getClass().getResource( - "/test.logging.properties"); - properties.setProperty("java.util.logging.config.file", new File( - loggingPropertiesUrl.getFile()).getAbsolutePath()); - - // first load the example configuration - try { - URL filemgrPropertiesUrl = this.getClass().getResource( - "/filemgr.properties"); - properties.load( - new FileInputStream(new File(filemgrPropertiesUrl.getFile()))); - } catch (Exception e) { - fail(e.getMessage()); - } - - // override the catalog to use: we'll use lucene - try { - URL ingestUrl = this.getClass().getResource("/ingest"); - luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath() - + "/cat"; - } catch (Exception e) { - fail(e.getMessage()); - } - - properties.setProperty("filemgr.catalog.factory", - "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory"); - properties.setProperty( - "org.apache.oodt.cas.filemgr.catalog.lucene.idxPath", - luceneCatLoc); - - // now override the repo mgr policy - URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy"); - try { - properties.setProperty( - "org.apache.oodt.cas.filemgr.repositorymgr.dirs", - "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath()); - } catch (Exception e) { - fail(e.getMessage()); - } - - // now override the val layer ones - properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs", - "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath()); - - // set up mime repo path - URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml"); - properties.setProperty( - "org.apache.oodt.cas.filemgr.mime.type.repository", - new File(mimeTypesUrl.getFile()).getAbsolutePath()); - - System.setProperties(properties); - - try { - fm = new XmlRpcFileManager(FM_PORT); - } catch (Exception e) { - fail(e.getMessage()); - } + @Override + protected boolean shouldExpandProduct() { + return false; } - }
