http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/AppCatInit.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/AppCatInit.java b/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/AppCatInit.java deleted file mode 100644 index cdeb0ce..0000000 --- a/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/AppCatInit.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.aoache.airavata.data.manager.core.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class AppCatInit { - private static final Logger logger = LoggerFactory.getLogger(AppCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "appcatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - public static final String COMPUTE_RESOURCE_TABLE = "COMPUTE_RESOURCE"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public AppCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - - try{ - jdbcDriver = ServerSettings.getSetting("appcatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("appcatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("appcatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("appcatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server cound not started within five seconds..."); - } -// startDerbyInEmbeddedMode(); - - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(COMPUTE_RESOURCE_TABLE, conn)) { - executeSQLScript(conn); - logger.info("New Database created for App Catalog !!!"); - } else { - logger.debug("Database already created for App Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata database", e); - }finally { - if (reader != null) { - reader.close(); - } - - } - - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/DataCatInit.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/DataCatInit.java b/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/DataCatInit.java deleted file mode 100644 index d1c8491..0000000 --- a/modules/file-manager/data-manager-core/src/test/java/org/aoache/airavata/data/manager/core/utils/DataCatInit.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.aoache.airavata.data.manager.core.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class DataCatInit { - private static final Logger logger = LoggerFactory.getLogger(DataCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "datacatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public DataCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - try{ - jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server could not started within five seconds..."); - } - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) { - executeSQLScript(conn); - logger.info("New Database created for Data Catalog !!!"); - } else { - logger.debug("Database already created for Data Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - }finally { - if (reader != null) { - reader.close(); - } - } - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-cpi/pom.xml ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/pom.xml b/modules/file-manager/data-manager-cpi/pom.xml deleted file mode 100644 index 93137ef..0000000 --- a/modules/file-manager/data-manager-cpi/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>data-manager</artifactId> - <groupId>org.apache.airavata</groupId> - <version>0.16-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <modelVersion>4.0.0</modelVersion> - <artifactId>data-manager-cpi</artifactId> - <packaging>jar</packaging> - <name>Airavata Data Manager CPI</name> - <url>http://airavata.apache.org/</url> - - <dependencies> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>airavata-data-models</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>airavata-commons</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java deleted file mode 100644 index fd06f8c..0000000 --- a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.airavata.data.manager.cpi; - -import org.apache.airavata.model.data.resource.DataReplicaLocationModel; -import org.apache.airavata.model.data.resource.DataResourceModel; - -import java.util.List; - -public interface DataManager { - - /** - * To create a new dataResourceModel. This is how the system comes to know about already - * existing resources - * @param dataResourceModel - * @return - */ - String registerResource(DataResourceModel dataResourceModel) throws DataManagerException; - - /** - * To remove a resource entry from the replica catalog - * @param resourceId - * @return - */ - boolean removeResource(String resourceId) throws DataManagerException; - - - /** - * To update an existing data resource model - * @param dataResourceModel - * @return - * @throws DataManagerException - */ - boolean updateResource(DataResourceModel dataResourceModel) throws DataManagerException; - - /** - * To retrieve a resource object providing the resourceId - * @param resourceId - * @return - */ - DataResourceModel getResource(String resourceId) throws DataManagerException; - - /** - * To create a new data replica location. This is how the system comes to know about already - * existing resources - * @param dataReplicaLocationModel - * @return - */ - String registerReplicaLocation(DataReplicaLocationModel dataReplicaLocationModel) throws DataManagerException; - - /** - * To remove a replica entry from the replica catalog - * @param replicaId - * @return - */ - boolean removeReplicaLocation(String replicaId) throws DataManagerException; - - /** - * To update an existing data replica model - * @param dataReplicaLocationModel - * @return - * @throws DataManagerException - */ - boolean updateReplicaLocation(DataReplicaLocationModel dataReplicaLocationModel) throws DataManagerException; - - /** - * To retrieve a replica object providing the replicaId - * @param replicaId - * @return - */ - DataReplicaLocationModel getReplicaLocation(String replicaId) throws DataManagerException; - - /** - * To retrieve all the replica entries for a given resource id - * @param resourceId - * @return - * @throws DataCatalogException - */ - List<DataReplicaLocationModel> getAllReplicaLocations(String resourceId) throws DataManagerException; - - - /** - * API method to copy a resource to the provided destination storage resource. Only resources of type FILE can be - * copied using this API method. Method returns the new replicaId. - * @param dataResourceId - * @param destStorageResourceId - * @param destinationParentPath - * @return - */ - String copyResource(String dataResourceId, String destStorageResourceId, String destinationParentPath) throws DataManagerException; - - /** - * API method to copy the specified replica to the provided destination storage resource. Only resources of type FILE - * can be copied using this API method. Method returns the new replicaId - * @param dataResourceId - * @param replicaId - * @param destStorageResourceId - * @param destinationParentPath - * @return - * @throws DataManagerException - */ - String copyReplica(String dataResourceId, String replicaId, String destStorageResourceId, String destinationParentPath) throws DataManagerException; -} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java deleted file mode 100644 index d46604a..0000000 --- a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager.cpi; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataManagerConstants { - private final static Logger logger = LoggerFactory.getLogger(DataManagerConstants.class); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java deleted file mode 100644 index b1c1cb8..0000000 --- a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.airavata.data.manager.cpi; - -public class DataManagerException extends Exception{ - - public DataManagerException(Throwable e) { - super(e); - } - - public DataManagerException(String message) { - super(message, null); - } - - public DataManagerException(String message, Throwable e) { - super(message, e); - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java deleted file mode 100644 index f701e49..0000000 --- a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager.cpi; - -import org.apache.airavata.model.file.FileNode; - -public interface FileTransferService { - - void uploadFile(String fileName, byte[] fileData, String destinationResourceId, String destinationPath) throws DataManagerException; - - void importFile(String sourceUrl, String destinationResourceId, String destinationPath) throws DataManagerException; - - void transferFile(String sourceResourceId, String sourcePath, String destinationResourceId, String destinationPath) throws DataManagerException; - - FileNode getDirectoryListing(String storageResourceId, String directoryPath) throws DataManagerException; - - void moveFile(String storageResourceId, String sourcePath, String destinationPath) throws DataManagerException; - - void renameFile(String storageResourceId, String sourcePath, String newName) throws DataManagerException; - - void mkdir(String storageResourceId, String directoryPath) throws DataManagerException; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/pom.xml ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/pom.xml b/modules/file-manager/file-manager-core/pom.xml new file mode 100644 index 0000000..affeea9 --- /dev/null +++ b/modules/file-manager/file-manager-core/pom.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>file-manager</artifactId> + <groupId>org.apache.airavata</groupId> + <version>0.16-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <artifactId>file-manager-core</artifactId> + <packaging>jar</packaging> + <name>Airavata File Manager Core</name> + <url>http://airavata.apache.org/</url> + + <dependencies> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-data-models</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-cpi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-commons</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-server-configuration</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>file-manager-cpi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-cpi</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.5.3</version> + </dependency> + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + <version>3.0.0</version> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java new file mode 100644 index 0000000..20a0857 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java @@ -0,0 +1,34 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core; + +import org.apache.airavata.file.manager.cpi.FileManager; +import org.apache.airavata.file.manager.cpi.FileManagerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileManagerFactory { + private final static Logger logger = LoggerFactory.getLogger(FileManagerFactory.class); + + public static FileManager getDataManager() throws FileManagerException { + return new FileManagerImpl(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java new file mode 100644 index 0000000..1595819 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java @@ -0,0 +1,29 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core; + +import org.apache.airavata.file.manager.cpi.FileManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileManagerImpl implements FileManager { + private final static Logger logger = LoggerFactory.getLogger(FileManagerImpl.class); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileTransferServiceImpl.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileTransferServiceImpl.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileTransferServiceImpl.java new file mode 100644 index 0000000..7558d4b --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileTransferServiceImpl.java @@ -0,0 +1,474 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential; +import org.apache.airavata.file.manager.core.db.dao.FileTransferRequestDao; +import org.apache.airavata.file.manager.core.remote.client.RemoteStorageClient; +import org.apache.airavata.file.manager.core.remote.client.http.HTTPStorageClient; +import org.apache.airavata.file.manager.core.remote.client.scp.SCPStorageClient; +import org.apache.airavata.file.manager.core.remote.client.sftp.SFTPStorageClient; +import org.apache.airavata.file.manager.cpi.FileManagerException; +import org.apache.airavata.file.manager.cpi.FileTransferService; +import org.apache.airavata.model.file.*; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; +import java.util.UUID; + +public class FileTransferServiceImpl implements FileTransferService { + + private final static Logger logger = LoggerFactory.getLogger(FileTransferServiceImpl.class); + + private FileTransferRequestDao fileTransferRequestDao; + + public FileTransferServiceImpl() throws IOException { + this.fileTransferRequestDao = new FileTransferRequestDao(); + } + + /** + * Method to upload the give bytes to the destination storage system + * + * @param fileData + * @param destHostName + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @return + * @throws FileManagerException + */ + @Override + public String uploadFile(byte[] fileData, String destHostName, String destLoginName, int destPort, + StorageResourceProtocol destProtocol, + String destinationPath, String destHostCredToken) throws FileManagerException { + long transferTime = System.currentTimeMillis(); + if(destProtocol == StorageResourceProtocol.SCP || destProtocol == StorageResourceProtocol.SFTP) { + Object credential = getCredential(destHostCredToken); + SSHCredential sshCredential; + if (credential instanceof SSHCredential) { + sshCredential = (SSHCredential) credential; + File srcFile = null; + FileWriter fileWriter = null; + FileTransferRequest fileTransferRequest = null; + try { + String srcFilePath = System.getProperty("java.io.tmpdir")+File.separator+ UUID.randomUUID().toString(); + srcFile = new File(srcFilePath); + fileWriter = new FileWriter(srcFile); + fileWriter.write(new String(fileData)); + fileWriter.close(); + RemoteStorageClient remoteStorageClient; + if(destProtocol == StorageResourceProtocol.SCP) + remoteStorageClient = new SCPStorageClient(destHostName, destPort, destLoginName, + sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + else + remoteStorageClient = new SFTPStorageClient(destHostName, destPort, destLoginName, + sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + + fileTransferRequest = new FileTransferRequest(); + fileTransferRequest.setSrcHostname(InetAddress.getLocalHost().getHostName()); + fileTransferRequest.setSrcProtocol(StorageResourceProtocol.LOCAL); + fileTransferRequest.setDestHostname(destHostName); + fileTransferRequest.setDestLoginName(destLoginName); + fileTransferRequest.setDestPort(destPort); + fileTransferRequest.setDestProtocol(destProtocol); + fileTransferRequest.setDestFilePath(destinationPath); + fileTransferRequest.setDestHostCredToken(destHostCredToken); + fileTransferRequest.setFileTransferMode(FileTransferMode.SYNC); + remoteStorageClient.writeFile(srcFile, destinationPath); + transferTime = System.currentTimeMillis() - transferTime; + fileTransferRequest.setTransferTime(transferTime); + fileTransferRequest.setTransferStatus(FileTransferStatus.COMPLETED); + fileTransferRequest.setCreatedTime(System.currentTimeMillis()); + fileTransferRequest.setLastModifiedType(fileTransferRequest.getCreatedTime()); + fileTransferRequest.setFileSize(srcFile.length()); + String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequest); + return transferId; + } catch (Exception e) { + logger.error(e.getMessage(), e); + if(fileTransferRequest != null) { + fileTransferRequest.setTransferStatus(FileTransferStatus.FAILED); + try { + fileTransferRequestDao.createFileTransferRequest(fileTransferRequest); + } catch (JsonProcessingException e1) { + logger.error(e.getMessage(), e); + throw new FileManagerException(e); + } + } + throw new FileManagerException(e.getMessage()); + } finally { + if(srcFile != null) + srcFile.delete(); + if(fileWriter != null) + try { + fileWriter.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + throw new FileManagerException(e); + } + } + } else { + throw new FileManagerException("Only SSHCredential type is supported"); + } + }else{ + throw new FileManagerException(destProtocol.toString() + " protocol is not supported for this method"); + } + } + + /** + * Transfer file between two storage resources synchronously. Returns the file transfer request id + * + * @param srcHostname + * @param srcLoginName + * @param srcPort + * @param srcProtocol + * @param srcPath + * @param srcHostCredToken + * @param destHostname + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @return + * @throws FileManagerException + */ + @Override + public String transferFile(String srcHostname, String srcLoginName, int srcPort, StorageResourceProtocol srcProtocol, + String srcPath, String srcHostCredToken, String destHostname, String destLoginName, int destPort, + StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken) + throws FileManagerException { + long transferTime = System.currentTimeMillis(); + File srcFile = null; + FileTransferRequest fileTransferRequest = null; + try{ + fileTransferRequest = new FileTransferRequest(); + fileTransferRequest.setSrcHostname(srcHostname); + fileTransferRequest.setSrcPort(srcPort); + fileTransferRequest.setSrcLoginName(srcLoginName); + fileTransferRequest.setSrcFilePath(srcPath); + fileTransferRequest.setSrcProtocol(srcProtocol); + fileTransferRequest.setSrcHostCredToken(srcHostCredToken); + fileTransferRequest.setDestHostname(destHostname); + fileTransferRequest.setDestPort(destPort); + fileTransferRequest.setDestLoginName(destLoginName); + fileTransferRequest.setDestFilePath(destinationPath); + fileTransferRequest.setDestProtocol(destProtocol); + fileTransferRequest.setDestHostCredToken(destHostCredToken); + fileTransferRequest.setCreatedTime(System.currentTimeMillis()); + fileTransferRequest.setLastModifiedType(fileTransferRequest.getCreatedTime()); + fileTransferRequest.setFileTransferMode(FileTransferMode.SYNC); + + if(srcProtocol == StorageResourceProtocol.HTTP || srcProtocol == StorageResourceProtocol.HTTPS || + srcProtocol == StorageResourceProtocol.SCP || srcProtocol == StorageResourceProtocol.SFTP){ + RemoteStorageClient srcClient = null; + if(srcProtocol == StorageResourceProtocol.HTTP){ + srcClient = new HTTPStorageClient(HTTPStorageClient.Protocol.HTTP, srcHostname, srcPort); + }else if(srcProtocol == StorageResourceProtocol.HTTPS){ + srcClient = new HTTPStorageClient(HTTPStorageClient.Protocol.HTTPS, srcHostname, srcPort); + }else if(srcProtocol == StorageResourceProtocol.SCP){ + Object credential = getCredential(srcHostCredToken); + if(credential instanceof SSHCredential){ + SSHCredential sshCredential = (SSHCredential) credential; + srcClient = new SCPStorageClient(srcHostname, srcPort, srcLoginName, sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + }else{ + throw new FileManagerException("Only support SSHCredentials for SCP host"); + } + }else{ + Object credential = getCredential(srcHostCredToken); + if(credential instanceof SSHCredential){ + SSHCredential sshCredential = (SSHCredential) credential; + srcClient = new SFTPStorageClient(srcHostname, srcPort, srcLoginName, sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + }else{ + throw new FileManagerException("Only support SSHCredentials for SFTP host"); + } + } + fileTransferRequest.setTransferStatus(FileTransferStatus.RUNNING); + srcFile = srcClient.readFile(srcPath); + }else{ + throw new FileManagerException("Unsupported src protocol " + srcProtocol); + } + + if(destProtocol == StorageResourceProtocol.SCP || destProtocol == StorageResourceProtocol.SFTP){ + RemoteStorageClient destClient = null; + if(destProtocol == StorageResourceProtocol.SCP){ + Object credential = getCredential(srcHostCredToken); + if(credential instanceof SSHCredential){ + SSHCredential sshCredential = (SSHCredential) credential; + destClient = new SCPStorageClient(srcHostname, srcPort, srcLoginName, sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + }else{ + throw new FileManagerException("Only support SSHCredentials for SCP host"); + } + }else{ + Object credential = getCredential(srcHostCredToken); + if(credential instanceof SSHCredential){ + SSHCredential sshCredential = (SSHCredential) credential; + destClient = new SFTPStorageClient(srcHostname, srcPort, srcLoginName, sshCredential.getPrivateKey(), + sshCredential.getPublicKey(), sshCredential.getPassphrase().getBytes()); + }else{ + throw new FileManagerException("Only support SSHCredentials for SFTP host"); + } + } + destClient.writeFile(srcFile, destinationPath); + transferTime = System.currentTimeMillis() - transferTime; + fileTransferRequest.setTransferTime(transferTime); + fileTransferRequest.setFileSize(srcFile.length()); + fileTransferRequest.setTransferStatus(FileTransferStatus.COMPLETED); + String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequest); + return transferId; + }else{ + throw new FileManagerException("Unsupported src protocol " + srcProtocol); + } + }catch (Exception e){ + logger.error(e.getMessage(), e); + if(fileTransferRequest != null) { + fileTransferRequest.setTransferStatus(FileTransferStatus.FAILED); + try { + fileTransferRequestDao.createFileTransferRequest(fileTransferRequest); + } catch (JsonProcessingException ex) { + logger.error(ex.getMessage(), ex); + throw new FileManagerException(ex); + } + } + throw new FileManagerException(e); + }finally { + if(srcFile != null) + srcFile.delete(); + } + } + + /** + * Transfer file between two storage resources asynchronously. Returns the file transfer request id + * + * @param srcHostname + * @param srcLoginName + * @param srcPort + * @param srcProtocol + * @param srcPath + * @param srcHostCredToken + * @param destHostname + * @param destLoginName + * @param destPort + * @param destProtocol + * @param destinationPath + * @param destHostCredToken + * @param callbackEmails + * @return + * @throws FileManagerException + */ + @Override + public String transferFileAsync(String srcHostname, String srcLoginName, int srcPort, StorageResourceProtocol srcProtocol, + String srcPath, String srcHostCredToken, String destHostname, String destLoginName, + int destPort, StorageResourceProtocol destProtocol, String destinationPath, + String destHostCredToken, String[] callbackEmails) throws FileManagerException { + return null; + } + + /** + * Get a directory listing of the specified source directory + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param path + * @param hostCredential + * @return + * @throws FileManagerException + */ + @Override + public List<FileNode> getDirectoryListing(String hostname, String loginName, int port, StorageResourceProtocol protocol, + String path, String hostCredential) throws FileManagerException { + return null; + } + + /** + * Move file from one place to another inside the same storage resource + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param sourcePath + * @param destinationPath + * @throws FileManagerException + */ + @Override + public void moveFile(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String sourcePath, String destinationPath) throws FileManagerException { + + } + + /** + * Rename a file + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param sourcePath + * @param newName + * @throws FileManagerException + */ + @Override + public void renameFile(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String sourcePath, String newName) throws FileManagerException { + + } + + /** + * Create new directory + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param dirPath + * @throws FileManagerException + */ + @Override + public void mkdir(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String dirPath) throws FileManagerException { + + } + + /** + * Delete File in storage resource + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @throws FileManagerException + */ + @Override + public void deleteFile(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String filePath) throws FileManagerException { + + } + + /** + * Check whether the specified file exists + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @return + * @throws FileManagerException + */ + @Override + public boolean isExists(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String filePath) throws FileManagerException { + return false; + } + + /** + * Check whether the path points to a directory + * + * @param hostname + * @param loginName + * @param port + * @param protocol + * @param hostCredential + * @param filePath + * @return + * @throws FileManagerException + */ + @Override + public boolean isDirectory(String hostname, String loginName, int port, StorageResourceProtocol protocol, String hostCredential, + String filePath) throws FileManagerException { + return false; + } + + /** + * Method to retrieve file transfer status giving transfer id + * + * @param transferId + * @return + * @throws FileManagerException + */ + @Override + public FileTransferRequest getFileTransferRequestStatus(String transferId) throws FileManagerException { + try{ + return fileTransferRequestDao.getFileTransferRequest(transferId); + }catch (Exception ex){ + logger.error(ex.getMessage(), ex); + throw new FileManagerException(ex); + } + } + + + //TODO API Call to Credential Store + private SSHCredential getCredential(String credentialStoreToken) throws FileManagerException{ + try{ + SSHCredential sshCredential = new SSHCredential(); + File privateKey = new File("/Users/supun/.ssh/id_rsa"); + byte[] privateKeyBytes = IOUtils.toByteArray(new FileInputStream(privateKey)); + File publicKey = new File("/Users/supun/.ssh/id_rsa.pub"); + byte[] publicKeyBytes = IOUtils.toByteArray(new FileInputStream(publicKey)); + String passPhrase = "cecilia@1990"; + sshCredential.setPrivateKey(privateKeyBytes); + sshCredential.setPublicKey(publicKeyBytes); + sshCredential.setPassphrase(passPhrase); + return sshCredential; + }catch (Exception ex){ + logger.error(ex.getMessage(), ex); + throw new FileManagerException(ex); + } + } + + public static void main(String[] args) throws IOException, FileManagerException { + FileTransferServiceImpl fileTransferService = new FileTransferServiceImpl(); + String sourceFile = "fsgsdgsdgsdgsdg"; + String transferId = fileTransferService.uploadFile(sourceFile.getBytes(), "gw75.iu.xsede.org", "pga", 22, + StorageResourceProtocol.SCP, "/var/www/portals/test.file", null); + FileTransferRequest fileTransferRequest = fileTransferService.fileTransferRequestDao.getFileTransferRequest(transferId); + System.out.println("file transfer id:" + fileTransferRequest.getTransferId()); + + transferId = fileTransferService.transferFile("gw75.iu.xsede.org", "pga", 22, StorageResourceProtocol.SCP, + "/var/www/portals/test.file", null, "gw75.iu.xsede.org", "pga", 22, StorageResourceProtocol.SCP, + "/var/www/portals/test2.file", null); + fileTransferRequest = fileTransferService.fileTransferRequestDao.getFileTransferRequest(transferId); + System.out.println("file transfer id:" + fileTransferRequest.getTransferId()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftDeserializer.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftDeserializer.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftDeserializer.java new file mode 100644 index 0000000..a50e0ea --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftDeserializer.java @@ -0,0 +1,154 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.conversion; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.google.common.base.CaseFormat; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TFieldIdEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.util.Iterator; +import java.util.Map; + +/** + * This abstract class represents a generic de-serializer for converting JSON to Thrift-based entities. + * + * @param <E> An implementation of the {@link org.apache.thrift.TFieldIdEnum} interface. + * @param <T> An implementation of the {@link org.apache.thrift.TBase} interface. + */ +public abstract class AbstractThriftDeserializer<E extends TFieldIdEnum, T extends TBase<T, E>> extends JsonDeserializer<T> { + + private static Logger log = LoggerFactory.getLogger(AbstractThriftDeserializer.class); + + @Override + public T deserialize(final JsonParser jp, final DeserializationContext ctxt) throws IOException, JsonProcessingException { + final T instance = newInstance(); + final ObjectMapper mapper = (ObjectMapper)jp.getCodec(); + final ObjectNode rootNode = (ObjectNode)mapper.readTree(jp); + final Iterator<Map.Entry<String, JsonNode>> iterator = rootNode.fields(); + + while(iterator.hasNext()) { + final Map.Entry<String, JsonNode> currentField = iterator.next(); + try { + /* + * If the current node is not a null value, process it. Otherwise, + * skip it. Jackson will treat the null as a 0 for primitive + * number types, which in turn will make Thrift think the field + * has been set. Also we ignore the MongoDB specific _id field + */ + if(currentField.getValue().getNodeType() != JsonNodeType.NULL) { + final E field = getField(CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, currentField.getKey())); + final JsonParser parser = currentField.getValue().traverse(); + parser.setCodec(mapper); + final Object value = mapper.readValue(parser, generateValueType(instance, field)); + if(value != null) { + log.debug(String.format("Field %s produced value %s of type %s.", + currentField.getKey(), value, value.getClass().getName())); + instance.setFieldValue(field, value); + } else { + log.debug("Field {} contains a null value. Skipping...", currentField.getKey()); + } + } else { + log.debug("Field {} contains a null value. Skipping...", currentField.getKey()); + } + } catch (final NoSuchFieldException | IllegalArgumentException e) { + log.error("Unable to de-serialize field '{}'.", currentField.getKey(), e); + ctxt.mappingException(e.getMessage()); + } + } + + try { + // Validate that the instance contains all required fields. + validate(instance); + } catch (final TException e) { + log.error(String.format("Unable to deserialize JSON '%s' to type '%s'.", + jp.getValueAsString(), instance.getClass().getName(), e)); + ctxt.mappingException(e.getMessage()); + } + + return instance; + } + + /** + * Returns the {@code <E>} enumerated value that represents the target + * field in the Thrift entity referenced in the JSON document. + * @param fieldName The name of the Thrift entity target field. + * @return The {@code <E>} enumerated value that represents the target + * field in the Thrift entity referenced in the JSON document. + */ + protected abstract E getField(String fieldName); + + /** + * Creates a new instance of the Thrift entity class represented by this deserializer. + * @return A new instance of the Thrift entity class represented by this deserializer. + */ + protected abstract T newInstance(); + + /** + * Validates that the Thrift entity instance contains all required fields after deserialization. + * @param instance A Thrift entity instance. + * @throws org.apache.thrift.TException if unable to validate the instance. + */ + protected abstract void validate(T instance) throws TException; + + /** + * Generates a {@link JavaType} that matches the target Thrift field represented by the provided + * {@code <E>} enumerated value. If the field's type includes generics, the generics will + * be added to the generated {@link JavaType} to support proper conversion. + * @param thriftInstance The Thrift-generated class instance that will be converted to/from JSON. + * @param field A {@code <E>} enumerated value that represents a field in a Thrift-based entity. + * @return The {@link JavaType} representation of the type associated with the field. + * @throws NoSuchFieldException if unable to determine the field's type. + * @throws SecurityException if unable to determine the field's type. + */ + protected JavaType generateValueType(final T thriftInstance, final E field) throws NoSuchFieldException, SecurityException { + final TypeFactory typeFactory = TypeFactory.defaultInstance(); + + final Field declaredField = thriftInstance.getClass().getDeclaredField(field.getFieldName()); + if(declaredField.getType().equals(declaredField.getGenericType())) { + log.debug("Generating JavaType for type '{}'.", declaredField.getType()); + return typeFactory.constructType(declaredField.getType()); + } else { + final ParameterizedType type = (ParameterizedType)declaredField.getGenericType(); + final Class<?>[] parameterizedTypes = new Class<?>[type.getActualTypeArguments().length]; + for(int i=0; i<type.getActualTypeArguments().length; i++) { + parameterizedTypes[i] = (Class<?>)type.getActualTypeArguments()[i]; + } + log.debug("Generating JavaType for type '{}' with generics '{}'", declaredField.getType(), parameterizedTypes); + return typeFactory.constructParametricType(declaredField.getType(), parameterizedTypes); + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftSerializer.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftSerializer.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftSerializer.java new file mode 100644 index 0000000..f8be22a --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/AbstractThriftSerializer.java @@ -0,0 +1,122 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.conversion; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.google.common.base.CaseFormat; +import org.apache.thrift.TBase; +import org.apache.thrift.TFieldIdEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; + +/** + * This abstract class represents a generic serializer for converting Thrift-based entities + * to JSON. + * + * @param <E> An implementation of the {@link org.apache.thrift.TFieldIdEnum} interface. + * @param <T> An implementation of the {@link org.apache.thrift.TBase} interface. + */ +public abstract class AbstractThriftSerializer<E extends TFieldIdEnum, T extends TBase<T, E>> + extends JsonSerializer<T> { + + private static final Logger log = LoggerFactory.getLogger(AbstractThriftSerializer.class); + + @Override + public Class<T> handledType() { + return getThriftClass(); + } + + @Override + public void serialize(final T value, final JsonGenerator jgen, final SerializerProvider provider) + throws IOException, JsonProcessingException { + jgen.writeStartObject(); + for(final E field : getFieldValues()) { + if(value.isSet(field)) { + final Object fieldValue = value.getFieldValue(field); + if(fieldValue != null) { + log.debug("Adding field {} to the JSON string...", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName()) + ); + + jgen.writeFieldName(CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName())); + if(fieldValue instanceof Short) { + jgen.writeNumber((Short)fieldValue); + } else if(fieldValue instanceof Integer) { + jgen.writeNumber((Integer)fieldValue); + } else if(fieldValue instanceof Long) { + jgen.writeNumber((Long)fieldValue); + } else if(fieldValue instanceof Double) { + jgen.writeNumber((Double)fieldValue); + } else if(fieldValue instanceof Float) { + jgen.writeNumber((Float)fieldValue); + } else if(fieldValue instanceof Boolean) { + jgen.writeBoolean((Boolean)fieldValue); + } else if(fieldValue instanceof String) { + jgen.writeString(fieldValue.toString()); + } else if(fieldValue instanceof Collection) { + log.debug("Array opened for field {}.", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName()) + ); + jgen.writeStartArray(); + for(final Object arrayObject : (Collection<?>)fieldValue) { + jgen.writeObject(arrayObject); + } + jgen.writeEndArray(); + log.debug("Array closed for field {}.", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName()) + ); + } else { + jgen.writeObject(fieldValue); + } + } else { + log.debug("Skipping converting field {} to JSON: value is null!", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName()) + ); + } + } else { + log.debug("Skipping converting field {} to JSON: field has not been set!", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE,field.getFieldName()) + ); + } + } + jgen.writeEndObject(); + } + + /** + * Returns an array of {@code <E>} enumerated values that represent the fields present in the + * Thrift class associated with this serializer. + * @return The array of {@code <E>} enumerated values that represent the fields present in the + * Thrift class. + */ + protected abstract E[] getFieldValues(); + + /** + * Returns the {@code <T>} implementation class associated with this serializer. + * @return The {@code <T>} implementation class + */ + protected abstract Class<T> getThriftClass(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestDeserializer.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestDeserializer.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestDeserializer.java new file mode 100644 index 0000000..8b7e481 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestDeserializer.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.conversion; + +import org.apache.airavata.model.file.FileTransferRequest; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileTransferRequestDeserializer extends + AbstractThriftDeserializer<FileTransferRequest._Fields, FileTransferRequest> { + + @Override + protected FileTransferRequest._Fields getField(final String fieldName) { + return FileTransferRequest._Fields.valueOf(fieldName); + } + + @Override + protected FileTransferRequest newInstance() { + return new FileTransferRequest(); + } + + @Override + protected void validate(final FileTransferRequest instance) throws TException { + instance.validate(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestSerializer.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestSerializer.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestSerializer.java new file mode 100644 index 0000000..64114e0 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/db/conversion/FileTransferRequestSerializer.java @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.conversion; + +import org.apache.airavata.model.file.FileTransferRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileTransferRequestSerializer extends + AbstractThriftSerializer<FileTransferRequest._Fields, FileTransferRequest> { + private final static Logger logger = LoggerFactory.getLogger(FileTransferRequestSerializer.class); + + @Override + protected FileTransferRequest._Fields[] getFieldValues() { + return FileTransferRequest._Fields.values(); + } + + @Override + protected Class<FileTransferRequest> getThriftClass() { + return null; + } +} \ No newline at end of file
