http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/ModelConversionHelper.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/ModelConversionHelper.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/ModelConversionHelper.java new file mode 100644 index 0000000..f307d2d --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/ModelConversionHelper.java @@ -0,0 +1,82 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.conversion; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.apache.airavata.model.file.FileTransferRequest; +import org.apache.thrift.TBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + + +/** + * This is utility class for model conversion of thrift to/from json + */ +public class ModelConversionHelper { + private final static Logger logger = LoggerFactory.getLogger(ModelConversionHelper.class); + + private ObjectMapper objectMapper; + + public ModelConversionHelper(){ + init(); + } + + /** + * Private method to register the custom serializers and deserializers + */ + private void init(){ + this.objectMapper = new ObjectMapper(); + SimpleModule module = new SimpleModule("FileManager", + new Version(1,0,0,null,null,null)); + + module.addSerializer(FileTransferRequest.class, new FileTransferRequestSerializer()); + module.addDeserializer(FileTransferRequest.class, new FileTransferRequestDeserializer()); + + objectMapper.registerModule(module); + } + + /** + * Method to serialize a thrift object to json + * @param object + * @return + * @throws JsonProcessingException + */ + public String serializeObject(TBase object) throws JsonProcessingException { + String json = this.objectMapper.writeValueAsString(object); + return json; + } + + /** + * Method to deserialize a json to the thrift object + * @param clz + * @param json + * @return + * @throws IOException + */ + public TBase deserializeObject(Class<?> clz, String json) throws IOException { + return (TBase)this.objectMapper.readValue(json, clz); + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDao.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDao.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDao.java new file mode 100644 index 0000000..142fa13 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDao.java @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.dao; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.mongodb.*; +import com.mongodb.util.JSON; +import org.apache.airavata.file.manager.core.db.conversion.ModelConversionHelper; +import org.apache.airavata.file.manager.core.db.utils.MongoUtils; +import org.apache.airavata.model.file.FileTransferRequest; +import org.apache.airavata.registry.core.experiment.catalog.model.Gateway; +import org.apache.airavata.registry.cpi.RegistryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class FileTransferRequestDao { + private final static Logger logger = LoggerFactory.getLogger(FileTransferRequestDao.class); + + private static final String FILE_TRANSFER_REQUESTS_COLLECTION_NAME = "file-transfer-requests"; + private DBCollection collection; + private ModelConversionHelper modelConversionHelper; + + private static final String TRANSFER_ID = "transfer_id"; + + public FileTransferRequestDao() throws IOException { + collection = MongoUtils.getFileManagerRegistry().getCollection(FILE_TRANSFER_REQUESTS_COLLECTION_NAME); + modelConversionHelper = new ModelConversionHelper(); + collection.dropIndexes(); + initIndexes(); + } + + /** + * If indexes are already defined this will simply ignore them + */ + private void initIndexes() { + collection.createIndex(new BasicDBObject(TRANSFER_ID, 1), new BasicDBObject("unique", true)); + } + + public List<FileTransferRequest> getAllFileTransferRequests() throws IOException { + List<FileTransferRequest> fileTransferRequestList = new ArrayList(); + DBCursor cursor = collection.find(); + for (DBObject document : cursor) { + fileTransferRequestList.add((FileTransferRequest) modelConversionHelper.deserializeObject( + FileTransferRequest.class, document.toString())); + } + return fileTransferRequestList; + } + + public String createFileTransferRequest(FileTransferRequest fileTransferRequest) throws JsonProcessingException { + fileTransferRequest.setTransferId(UUID.randomUUID().toString()); + WriteResult result = collection.insert((DBObject) JSON.parse( + modelConversionHelper.serializeObject(fileTransferRequest))); + logger.debug("No of inserted results " + result.getN()); + return fileTransferRequest.getTransferId(); + } + + public void updateFileTransferRequest(FileTransferRequest fileTransferRequest) throws JsonProcessingException { + DBObject query = BasicDBObjectBuilder.start().add( + TRANSFER_ID, fileTransferRequest.getTransferId()).get(); + WriteResult result = collection.update(query, (DBObject) JSON.parse( + modelConversionHelper.serializeObject(fileTransferRequest))); + logger.debug("No of updated results " + result.getN()); + } + + public void deleteFileTransferRequest(FileTransferRequest fileTransferRequest){ + DBObject query = BasicDBObjectBuilder.start().add( + TRANSFER_ID, fileTransferRequest.getTransferId()).get(); + WriteResult result = collection.remove(query); + logger.debug("No of removed file transfer requests " + result.getN()); + } + + public FileTransferRequest getFileTransferRequest(String transferId) throws IOException { + + DBObject criteria = new BasicDBObject(TRANSFER_ID, transferId); + DBObject doc = collection.findOne(criteria); + if (doc != null) { + String json = doc.toString(); + return (FileTransferRequest) modelConversionHelper.deserializeObject( + FileTransferRequest.class, json); + } + return null; + } + + public static void main(String[] args) throws IOException { + FileTransferRequest fileTransferRequest = new FileTransferRequest(); + fileTransferRequest.setSrcHostCredToken("djkalbsbdaslfbalsfbslf"); + fileTransferRequest.setSrcFilePath("test-file-path"); + FileTransferRequestDao fileTransferRequestDao = new FileTransferRequestDao(); + String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequest); + fileTransferRequest = fileTransferRequestDao.getFileTransferRequest(transferId); + System.out.println("Transfer Id:" + fileTransferRequest.getTransferId()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/utils/MongoUtils.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/utils/MongoUtils.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/utils/MongoUtils.java new file mode 100644 index 0000000..5148dd3 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/utils/MongoUtils.java @@ -0,0 +1,69 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.utils; + +import org.apache.airavata.file.manager.core.utils.FileManagerConstants; +import org.apache.airavata.file.manager.core.utils.FileManagerProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.mongodb.DB; +import com.mongodb.MongoClient; + +import java.io.IOException; + +public class MongoUtils { + private final static Logger logger = LoggerFactory.getLogger(MongoUtils.class); + + private static int port; + private static String host; + private static MongoClient mongoClient = null; + private static DB fileManagerRegistry; + private static String FILE_MANAGER_REGISTRY_NAME; + + public static MongoClient getMongoClient() throws IOException { + if (mongoClient == null) { + FileManagerProperties fileManagerProperties = FileManagerProperties.getInstance(); + host = fileManagerProperties.getProperty(FileManagerConstants.MONGODB_HOST, "localhost"); + port = Integer.parseInt(fileManagerProperties.getProperty(FileManagerConstants.MONGODB_PORT, "27017")); + FILE_MANAGER_REGISTRY_NAME = fileManagerProperties.getProperty(FileManagerConstants.MONGODB_DB_NAME, + "file-manager-db"); + mongoClient = new MongoClient(host, port); + logger.debug("New Mongo Client created with [" + host + "] and [" + + port + "]"); + + } + return mongoClient; + } + + public static DB getFileManagerRegistry() throws IOException { + if (fileManagerRegistry == null) { + fileManagerRegistry = getMongoClient().getDB(FILE_MANAGER_REGISTRY_NAME); + } + return fileManagerRegistry; + } + + public static void dropFileManagerRegistry() throws IOException { + getMongoClient().dropDatabase(FILE_MANAGER_REGISTRY_NAME); + logger.debug("Dropped File Manager Registry"); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/RemoteStorageClient.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/RemoteStorageClient.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/RemoteStorageClient.java new file mode 100644 index 0000000..4293e2e --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/RemoteStorageClient.java @@ -0,0 +1,108 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client; + + +import org.apache.airavata.model.file.FileNode; + +import java.io.File; +import java.util.List; + +public interface RemoteStorageClient { + + /** + * Reads a remote file, write it to local temporary directory and returns a file pointer to it + * @param filePath + * @return + * @throws Exception + */ + File readFile(String filePath) throws Exception; + + /** + * Writes the source file in the local storage to specified path in the remote storage + * @param sourceFile + * @return + * @throws Exception + */ + void writeFile(File sourceFile, String filePath) throws Exception; + + /** + * Returns a directory listing of the specified directory + * @param directoryPath + * @return + * @throws Exception + */ + List<FileNode> getDirectoryListing(String directoryPath) throws Exception; + + /** + * Move the specified file from source to destination within the same storage resource + * @param currentPath + * @param newPath + * @throws Exception + */ + void moveFile(String currentPath, String newPath) throws Exception; + + /** + * + * @param sourcePath + * @param destinationPath + * @throws Exception + */ + void copyFile(String sourcePath, String destinationPath) throws Exception; + + /** + * Rename file with the given name + * @param filePath + * @param newFileName + * @throws Exception + */ + void renameFile(String filePath, String newFileName) throws Exception; + + /** + * Delete the specified file + * @param filePath + * @throws Exception + */ + void deleteFile(String filePath) throws Exception; + + /** + * Create new directory in the specified file + * @param newDirPath + * @throws Exception + */ + void mkdir(String newDirPath) throws Exception; + + /** + * Checks whether specified file exists in the remote storage system + * @param filePath + * @return + * @throws Exception + */ + boolean checkFileExists(String filePath) throws Exception; + + /** + * Checks whether the given path is a directory + * @param filePath + * @return + * @throws Exception + */ + boolean checkIsDirectory(String filePath) throws Exception; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClient.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClient.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClient.java new file mode 100644 index 0000000..85d2056 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClient.java @@ -0,0 +1,217 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.http; + +import org.apache.airavata.file.manager.core.remote.client.RemoteStorageClient; +import org.apache.airavata.model.file.FileNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.Security; +import java.util.List; +import java.util.UUID; + +public class HTTPStorageClient implements RemoteStorageClient { + private final static Logger logger = LoggerFactory.getLogger(HTTPStorageClient.class); + + public static enum Protocol { + HTTP, HTTPS + } + + private String host; + private int port; + private Protocol protocol; + + public HTTPStorageClient(Protocol protocol, String host, int port) throws KeyManagementException, NoSuchAlgorithmException { + this.protocol = protocol; + this.host = host; + this.port = port; + + // Create a new trust manager that trust all certificates + TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + // Activate the new trust manager + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } + + /** + * Reads a remote file, write it to local temporary directory and returns a file pointer to it + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public File readFile(String filePath) throws Exception { + String url = ""; + if (protocol == Protocol.HTTP) + url += "http://";; + else + url += "https://";; + url += host + ":" + port; + if (!filePath.startsWith("/")) + filePath += "/" + filePath; + url += filePath; + + URL fileUrl = new URL(url); + URLConnection urlConnection = fileUrl.openConnection(); + ReadableByteChannel rbc = Channels.newChannel(urlConnection.getInputStream()); + String localFilePath = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString(); + FileOutputStream fos = new FileOutputStream(localFilePath); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + return new File(localFilePath); + } + + /** + * Writes the source file in the local storage to specified path in the remote storage + * + * @param sourceFile + * @param filePath + * @return + * @throws Exception + */ + @Override + public void writeFile(File sourceFile, String filePath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Returns a directory listing of the specified directory + * + * @param directoryPath + * @return + * @throws Exception + */ + @Override + public List<FileNode> getDirectoryListing(String directoryPath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Move the specified file from source to destination within the same storage resource + * + * @param currentPath + * @param newPath + * @throws Exception + */ + @Override + public void moveFile(String currentPath, String newPath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * @param sourcePath + * @param destinationPath + * @throws Exception + */ + @Override + public void copyFile(String sourcePath, String destinationPath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Rename file with the given name + * + * @param filePath + * @param newFileName + * @throws Exception + */ + @Override + public void renameFile(String filePath, String newFileName) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Delete the specified file + * + * @param filePath + * @throws Exception + */ + @Override + public void deleteFile(String filePath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Create new directory in the specified file + * + * @param newDirPath + * @throws Exception + */ + @Override + public void mkdir(String newDirPath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Checks whether specified file exists in the remote storage system + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkFileExists(String filePath) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Checks whether the given path is a directory + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkIsDirectory(String filePath) throws Exception { + return false; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/CommandOutput.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/CommandOutput.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/CommandOutput.java new file mode 100644 index 0000000..5fd2b52 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/CommandOutput.java @@ -0,0 +1,34 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.scp; + +import com.jcraft.jsch.Channel; +import java.io.OutputStream; + +public interface CommandOutput { + void onOutput(Channel var1); + + OutputStream getStandardError(); + + void exitCode(int var1); + + int getExitCode(); +} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPApiException.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPApiException.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPApiException.java new file mode 100644 index 0000000..0f07707 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPApiException.java @@ -0,0 +1,33 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.scp; + +public class SCPApiException extends Exception { + + public SCPApiException(String message) { + super(message); + } + + public SCPApiException(String message, Exception e) { + super(message, e); + } + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClient.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClient.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClient.java new file mode 100644 index 0000000..2c2c0a6 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClient.java @@ -0,0 +1,425 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.scp; + +import com.jcraft.jsch.*; +import org.apache.airavata.file.manager.core.remote.client.RemoteStorageClient; +import org.apache.airavata.model.file.FileNode; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.List; +import java.util.Properties; +import java.util.UUID; + +public class SCPStorageClient implements RemoteStorageClient { + private final static Logger logger = LoggerFactory.getLogger(SCPStorageClient.class); + + private JSch jSch; + private Session session; + + /** + * Constructor + * @param hostName + * @param port + * @param loginUsername + * @param password + * @throws JSchException + */ + public SCPStorageClient(String hostName, int port, String loginUsername, String password) throws JSchException { + Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + jSch = new JSch(); + jSch.addIdentity(loginUsername, password); + session = jSch.getSession(loginUsername, hostName, port); + session.setConfig(config); + session.connect(); + } + + /** + * Constructor + * @param hostName + * @param port + * @param loginUsername + * @param privateKey + * @param publicKey + * @param passPhrase + * @throws JSchException + */ + public SCPStorageClient(String hostName, int port, String loginUsername, byte[] privateKey, byte[] publicKey, + byte[] passPhrase) throws JSchException { + Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + jSch = new JSch(); + jSch.addIdentity(UUID.randomUUID().toString(), privateKey, publicKey, passPhrase); + session = jSch.getSession(loginUsername, hostName, port); + session.setConfig(config); + session.connect(); + } + + + /** + * Reads a remote file, write it to local temporary directory and returns a File + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public File readFile(String filePath) throws Exception { + if (!session.isConnected()) + session.connect(); + + FileOutputStream fos; + String localFile = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString(); + String prefix = null; + if (new File(localFile).isDirectory()) { + prefix = localFile + File.separator; + } + + // exec 'scp -f remotefile' remotely + String command = "scp -f " + filePath; + Channel channel = session.openChannel("exec"); + ((ChannelExec) channel).setCommand(command); + + StandardOutReader stdOutReader = new StandardOutReader(); + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + if (!channel.isClosed()) { + channel.connect(); + } + + byte[] buf = new byte[1024]; + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + while (true) { + int c = checkAck(in); + if (c != 'C') { + break; + } + + // read '0644 ' + in.read(buf, 0, 5); + + long filesize = 0L; + while (true) { + if (in.read(buf, 0, 1) < 0) { + // error + break; + } + if (buf[0] == ' ') break; + filesize = filesize * 10L + (long) (buf[0] - '0'); + } + + String file = null; + for (int i = 0; ; i++) { + in.read(buf, i, 1); + if (buf[i] == (byte) 0x0a) { + file = new String(buf, 0, i); + break; + } + } + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + // read a content of lfile + localFile = prefix == null ? localFile : prefix + file; + fos = new FileOutputStream(localFile); + int foo; + while (true) { + if (buf.length < filesize) foo = buf.length; + else foo = (int) filesize; + foo = in.read(buf, 0, foo); + if (foo < 0) { + // error + break; + } + fos.write(buf, 0, foo); + filesize -= foo; + if (filesize == 0L) break; + } + fos.close(); + fos = null; + + if (checkAck(in) != 0) { + String error = "Error transferring the file content"; + logger.error(error); + throw new SCPApiException(error); + } + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + } + stdOutReader.onOutput(channel); + if (stdOutReader.getStdErrorString().contains("scp:")) { + throw new SCPApiException(stdOutReader.getStdErrorString()); + } + + return new File(localFile); + } + + /** + * Writes the source file in the local storage to specified path in the remote storage + * + * @param sourceFile + * @param filePath + * @return + * @throws Exception + */ + @Override + public void writeFile(File sourceFile, String filePath) throws Exception { + if (!session.isConnected()) + session.connect(); + + FileInputStream fis; + String localFile = sourceFile.getAbsolutePath(); + boolean ptimestamp = true; + + // exec 'scp -t rfile' remotely + String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + filePath; + Channel channel = session.openChannel("exec"); + + StandardOutReader stdOutReader = new StandardOutReader(); + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + ((ChannelExec) channel).setCommand(command); + + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + channel.connect(); + + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + logger.error(error); + throw new SCPApiException(error); + } + + File _lfile = new File(localFile); + + if (ptimestamp) { + command = "T" + (_lfile.lastModified() / 1000) + " 0"; + // The access time should be sent here, + // but it is not accessible with JavaAPI ;-< + command += (" " + (_lfile.lastModified() / 1000) + " 0\n"); + out.write(command.getBytes()); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + logger.error(error); + throw new SCPApiException(error); + } + } + + // send "C0644 filesize filename", where filename should not include '/' + long filesize = _lfile.length(); + command = "C0644 " + filesize + " "; + if (localFile.lastIndexOf('/') > 0) { + command += localFile.substring(localFile.lastIndexOf('/') + 1); + } else { + command += localFile; + } + command += "\n"; + out.write(command.getBytes()); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + logger.error(error); + throw new SCPApiException(error); + } + + // send a content of localFile + fis = new FileInputStream(localFile); + byte[] buf = new byte[1024]; + while (true) { + int len = fis.read(buf, 0, buf.length); + if (len <= 0) break; + out.write(buf, 0, len); //out.flush(); + } + fis.close(); + fis = null; + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + logger.error(error); + throw new SCPApiException(error); + } + out.close(); + stdOutReader.onOutput(channel); + + + channel.disconnect(); + if (stdOutReader.getStdErrorString().contains("scp:")) { + throw new SCPApiException(stdOutReader.getStdErrorString()); + } + } + + /** + * Returns a directory listing of the specified directory + * + * @param directoryPath + * @return + * @throws Exception + */ + @Override + public List<FileNode> getDirectoryListing(String directoryPath) throws Exception { + return null; + } + + /** + * Move the specified file from source to destination within the same storage resource + * + * @param currentPath + * @param newPath + * @throws Exception + */ + @Override + public void moveFile(String currentPath, String newPath) throws Exception { + + } + + /** + * @param sourcePath + * @param destinationPath + * @throws Exception + */ + @Override + public void copyFile(String sourcePath, String destinationPath) throws Exception { + + } + + /** + * Rename file with the given name + * + * @param filePath + * @param newFileName + * @throws Exception + */ + @Override + public void renameFile(String filePath, String newFileName) throws Exception { + + } + + /** + * Delete the specified file + * + * @param filePath + * @throws Exception + */ + @Override + public void deleteFile(String filePath) throws Exception { + + } + + /** + * Create new directory in the specified file + * + * @param newDirPath + * @throws Exception + */ + @Override + public void mkdir(String newDirPath) throws Exception { + + } + + /** + * Checks whether specified file exists in the remote storage system + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkFileExists(String filePath) throws Exception { + return false; + } + + /** + * Checks whether the given path is a directory + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkIsDirectory(String filePath) throws Exception { + return false; + } + + private int checkAck(InputStream in) throws IOException { + int b = in.read(); + if (b == 0) return b; + if (b == -1) return b; + + if (b == 1 || b == 2) { + StringBuffer sb = new StringBuffer(); + int c; + do { + c = in.read(); + sb.append((char) c); + } + while (c != '\n'); + if (b == 1) { // error + System.out.print(sb.toString()); + } + if (b == 2) { // fatal error + System.out.print(sb.toString()); + } + } + return b; + } + + public static void main(String[] args) throws Exception { + File privateKey = new File("/Users/supun/.ssh/id_rsa"); + byte[] privateKeyBytes = IOUtils.toByteArray(new FileInputStream(privateKey)); + + File publicKey = new File("/Users/supun/.ssh/id_rsa.pub"); + byte[] publicKeyBytes = IOUtils.toByteArray(new FileInputStream(publicKey)); + + String passPhrase = ""; + byte[] passPhraseBytes = passPhrase.getBytes(); + + SCPStorageClient scpStorageClient = new SCPStorageClient("gw75.iu.xsede.org", 22, "pga", privateKeyBytes, + publicKeyBytes, passPhraseBytes); + File file = scpStorageClient.readFile("/var/www/portals/gateway-user-data/testdrive/test.txt"); + System.out.println("File exists ? " + file.exists()); + scpStorageClient.writeFile(file, "/var/www/portals/gateway-user-data/testdrive/test2.txt"); + file.delete(); + System.out.println("File exists ? " + file.exists()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/StandardOutReader.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/StandardOutReader.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/StandardOutReader.java new file mode 100644 index 0000000..d3de445 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/scp/StandardOutReader.java @@ -0,0 +1,86 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.scp; + +import com.jcraft.jsch.Channel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class StandardOutReader implements CommandOutput { + + private static final Logger logger = LoggerFactory.getLogger(StandardOutReader.class); + String stdOutputString = null; + ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); + private int exitCode; + + public void onOutput(Channel channel) { + try { + StringBuffer pbsOutput = new StringBuffer(""); + InputStream inputStream = channel.getInputStream(); + byte[] tmp = new byte[1024]; + do { + while (inputStream.available() > 0) { + int i = inputStream.read(tmp, 0, 1024); + if (i < 0) break; + pbsOutput.append(new String(tmp, 0, i)); + } + } while (!channel.isClosed()) ; + String output = pbsOutput.toString(); + this.setStdOutputString(output); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + + } + + + public void exitCode(int code) { + System.out.println("Program exit code - " + code); + this.exitCode = code; + } + + @Override + public int getExitCode() { + return exitCode; + } + + public String getStdOutputString() { + return stdOutputString; + } + + public void setStdOutputString(String stdOutputString) { + this.stdOutputString = stdOutputString; + } + + public String getStdErrorString() { + return errorStream.toString(); + } + + public OutputStream getStandardError() { + return errorStream; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/sftp/SFTPStorageClient.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/sftp/SFTPStorageClient.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/sftp/SFTPStorageClient.java new file mode 100644 index 0000000..38275a9 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/remote/client/sftp/SFTPStorageClient.java @@ -0,0 +1,209 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.sftp; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import org.apache.airavata.file.manager.core.remote.client.RemoteStorageClient; +import org.apache.airavata.model.file.FileNode; +import org.apache.airavata.model.file.FileNodeTypes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; + +public class SFTPStorageClient implements RemoteStorageClient { + private final static Logger logger = LoggerFactory.getLogger(SFTPStorageClient.class); + + private JSch jSch; + private Session session; + private ChannelSftp sftpChannel; + private final String hostName; + + public SFTPStorageClient(String hostName, int port, String loginUsername, String password) throws JSchException { + this.hostName = hostName; + Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + jSch = new JSch(); + jSch.addIdentity(loginUsername, password); + session = jSch.getSession(loginUsername, hostName, port); + session.setConfig(config); + session.connect(); + sftpChannel = (ChannelSftp) session.openChannel("sftp"); + } + + public SFTPStorageClient(String hostName, int port, String loginUsername, byte[] privateKey, byte[] publicKey, + byte[] passPhrase) throws JSchException { + this.hostName = hostName; + Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + jSch = new JSch(); + jSch.addIdentity(UUID.randomUUID().toString(), privateKey, publicKey, passPhrase); + session = jSch.getSession(loginUsername, hostName, port); + session.setConfig(config); + session.connect(); + sftpChannel = (ChannelSftp) session.openChannel("sftp"); + } + + /** + * Reads a remote file, write it to local temporary directory and returns a file pointer to it + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public File readFile(String filePath) throws Exception { + String localFile = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString(); + return null; + } + + /** + * Writes the source file in the local storage to specified path in the remote storage + * + * @param sourceFile + * @param filePath + * @return + * @throws Exception + */ + @Override + public void writeFile(File sourceFile, String filePath) throws Exception { + + } + + /** + * Returns a directory listing of the specified directory + * + * @param directoryPath + * @return + * @throws Exception + */ + @Override + public List<FileNode> getDirectoryListing(String directoryPath) throws Exception { + if(directoryPath.endsWith(File.separator)){ + directoryPath = directoryPath.substring(0, directoryPath.length() -1); + } + final String finalDirPath = directoryPath; + //channel may get timeout + if(sftpChannel.isClosed()){ + sftpChannel.connect(); + } + sftpChannel.cd(directoryPath); + Vector<ChannelSftp.LsEntry> lsEntryVector = sftpChannel.ls(directoryPath); + ArrayList<FileNode> fileNodeList = new ArrayList<>(); + lsEntryVector.stream().forEach(lsEntry -> { + FileNode fileNode = new FileNode(); + fileNode.setName(lsEntry.getFilename()); + fileNode.setPath(finalDirPath + File.separator + lsEntry.getFilename()); + fileNode.setStorageHostName(hostName); + fileNode.setSize(lsEntry.getAttrs().getSize()); + if(lsEntry.getAttrs().isDir()) + fileNode.setType(FileNodeTypes.DIRECTORY); + else + fileNode.setType(FileNodeTypes.FILE); + fileNodeList.add(fileNode); + }); + return fileNodeList; + } + + /** + * Move the specified file from source to destination within the same storage resource + * + * @param currentPath + * @param newPath + * @throws Exception + */ + @Override + public void moveFile(String currentPath, String newPath) throws Exception { + + } + + /** + * @param sourcePath + * @param destinationPath + * @throws Exception + */ + @Override + public void copyFile(String sourcePath, String destinationPath) throws Exception { + + } + + /** + * Rename file with the given name + * + * @param filePath + * @param newFileName + * @throws Exception + */ + @Override + public void renameFile(String filePath, String newFileName) throws Exception { + + } + + /** + * Delete the specified file + * + * @param filePath + * @throws Exception + */ + @Override + public void deleteFile(String filePath) throws Exception { + + } + + /** + * Create new directory in the specified file + * + * @param newDirPath + * @throws Exception + */ + @Override + public void mkdir(String newDirPath) throws Exception { + + } + + /** + * Checks whether specified file exists in the remote storage system + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkFileExists(String filePath) throws Exception { + return false; + } + + /** + * Checks whether the given path is a directory + * + * @param filePath + * @return + * @throws Exception + */ + @Override + public boolean checkIsDirectory(String filePath) throws Exception { + return false; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerConstants.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerConstants.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerConstants.java new file mode 100644 index 0000000..daf4685 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerConstants.java @@ -0,0 +1,33 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileManagerConstants { + private final static Logger logger = LoggerFactory.getLogger(FileManagerConstants.class); + + public static String MONGODB_HOST = "mongodb.host"; + public static String MONGODB_PORT = "mongodb.port"; + public static String MONGODB_DB_NAME = "mongodb.db.name"; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerProperties.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerProperties.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerProperties.java new file mode 100644 index 0000000..2cd5e52 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/utils/FileManagerProperties.java @@ -0,0 +1,58 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class FileManagerProperties { + private final static Logger logger = LoggerFactory.getLogger(FileManagerProperties.class); + + private static FileManagerProperties instance; + + private Properties properties; + + private FileManagerProperties() throws IOException { + properties = new Properties(); + properties.load(FileManagerProperties.class.getClassLoader().getResourceAsStream("file-manager.properties")); + } + + public static FileManagerProperties getInstance() throws IOException { + if(instance == null){ + instance = new FileManagerProperties(); + } + return instance; + } + + public String getProperty(String field, String defaultVal){ + String returnVal = properties.getProperty(field); + if(returnVal == null) + return defaultVal; + else + return returnVal; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/resources/file-manager.properties ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/resources/file-manager.properties b/modules/file-manager/file-manager-core/src/main/resources/file-manager.properties new file mode 100644 index 0000000..ad4157e --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/resources/file-manager.properties @@ -0,0 +1,24 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +mongodb.host=localhost +mongodb.port=27017 +mongodb.db.name=file-manager-db \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClientTest.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClientTest.java b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClientTest.java new file mode 100644 index 0000000..22d36d6 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/http/HTTPStorageClientTest.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.http; + +import junit.framework.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class HTTPStorageClientTest { + private final static Logger logger = LoggerFactory.getLogger(HTTPStorageClientTest.class); + + @Test + public void testHTTPStorageClient(){ + try { + HTTPStorageClient httpStorageClient = new HTTPStorageClient(HTTPStorageClient.Protocol.HTTPS, + "www.google.lk", 443); + File file = httpStorageClient.readFile("/"); + Assert.assertTrue(file.exists()); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/pom.xml ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/pom.xml b/modules/file-manager/file-manager-cpi/pom.xml new file mode 100644 index 0000000..9338b7e --- /dev/null +++ b/modules/file-manager/file-manager-cpi/pom.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>file-manager</artifactId> + <groupId>org.apache.airavata</groupId> + <version>0.16-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <artifactId>file-manager-cpi</artifactId> + <packaging>jar</packaging> + <name>Airavata File Manager CPI</name> + <url>http://airavata.apache.org/</url> + + <dependencies> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-data-models</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-commons</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManager.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManager.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManager.java new file mode 100644 index 0000000..8b9f9f4 --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManager.java @@ -0,0 +1,24 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.file.manager.cpi; + +public interface FileManager { +} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerConstants.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerConstants.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerConstants.java new file mode 100644 index 0000000..98ba692 --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerConstants.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.cpi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileManagerConstants { + private final static Logger logger = LoggerFactory.getLogger(FileManagerConstants.class); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerException.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerException.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerException.java new file mode 100644 index 0000000..153a2a9 --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileManagerException.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.airavata.file.manager.cpi; + +public class FileManagerException extends Exception{ + + public FileManagerException(Throwable e) { + super(e); + } + + public FileManagerException(String message) { + super(message, null); + } + + public FileManagerException(String message, Throwable e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java new file mode 100644 index 0000000..0b971b7 --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java @@ -0,0 +1,206 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.cpi; + +import org.apache.airavata.model.file.FileNode; +import org.apache.airavata.model.file.FileTransferRequest; +import org.apache.airavata.model.file.StorageResourceProtocol; + +import java.util.List; + +public interface FileTransferService { + + /** + * Method to upload the give bytes to the destination storage system + * @param fileData + * @param destHostname + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @return + * @throws FileManagerException + */ + String uploadFile(byte[] fileData, String destHostname, String destLoginName, int destPort, + StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken) + throws FileManagerException; + + /** + * Transfer file between two storage resources synchronously. Returns the file transfer request id + * @param srcHostname + * @param srcPort + * @param srcLoginName + * @param srcProtocol + * @param srcPath + * @param srcHostCredToken + * @param destHostname + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @return + * @throws FileManagerException + */ + String transferFile(String srcHostname, String srcLoginName, int srcPort, + StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken, + String destHostname, String destLoginName, int destPort, + StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken) + throws FileManagerException; + + /** + * Transfer file between two storage resources asynchronously. Returns the file transfer request id + * @param srcHostname + * @param srcLoginName + * @param srcPort + * @param srcProtocol + * @param srcPath + * @param srcHostCredToken + * @param destHostname + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @param callbackEmails + * @return + * @throws FileManagerException + */ + String transferFileAsync(String srcHostname, String srcLoginName, int srcPort, + StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken, + String destHostname, String destLoginName, int destPort, + StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken, + String[] callbackEmails) + throws FileManagerException; + + /** + * Get a directory listing of the specified source directory + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param path + * @param hostCredential + * @return + * @throws FileManagerException + */ + List<FileNode> getDirectoryListing(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String path, String hostCredential) + throws FileManagerException; + + /** + * Move file from one place to another inside the same storage resource + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param sourcePath + * @param destinationPath + * @throws FileManagerException + */ + void moveFile(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String sourcePath, String destinationPath) + throws FileManagerException; + + /** + * Rename a file + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param sourcePath + * @param newName + * @throws FileManagerException + */ + void renameFile(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String sourcePath, String newName) + throws FileManagerException; + + /** + * Create new directory + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param dirPath + * @throws FileManagerException + */ + void mkdir(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String dirPath) + throws FileManagerException; + + /** + * Delete File in storage resource + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @throws FileManagerException + */ + void deleteFile(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String filePath) + throws FileManagerException; + + /** + * Check whether the specified file exists + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @return + * @throws FileManagerException + */ + boolean isExists(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String filePath) + throws FileManagerException; + + /** + * Check whether the path points to a directory + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @return + * @throws FileManagerException + */ + boolean isDirectory(String hostname, String loginName, int port, + StorageResourceProtocol protocol, String hostCredential, String filePath) + throws FileManagerException; + + /** + * Method to retrieve file transfer status giving transfer id + * @param transferId + * @return + * @throws FileManagerException + */ + FileTransferRequest getFileTransferRequestStatus(String transferId) + throws FileManagerException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java new file mode 100644 index 0000000..e0cce5b --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java @@ -0,0 +1,25 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.cpi; + +public interface ReplicaManagementService { + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/pom.xml ---------------------------------------------------------------------- diff --git a/modules/file-manager/pom.xml b/modules/file-manager/pom.xml index eb81d40..01fc4da 100644 --- a/modules/file-manager/pom.xml +++ b/modules/file-manager/pom.xml @@ -10,14 +10,14 @@ </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>data-manager</artifactId> + <artifactId>file-manager</artifactId> <packaging>pom</packaging> - <name>Airavata Data Manager</name> + <name>Airavata File Manager</name> <url>http://airavata.apache.org/</url> <modules> - <module>data-manager-cpi</module> - <module>data-manager-core</module> + <module>file-manager-cpi</module> + <module>file-manager-core</module> </modules> <dependencies> <dependency>
