This is an automated email from the ASF dual-hosted git repository. lmccay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push: new b93cc1c KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use (#296) b93cc1c is described below commit b93cc1c68afa0e773027bfa2745f1d37f376bca1 Author: lmccay <lmc...@apache.org> AuthorDate: Wed Mar 25 01:08:37 2020 -0400 KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use (#296) * KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use Change-Id: I817969b5131dd08e5a7a174c73aacb13e9027e4a * Prompt to overwrite on put Change-Id: I110037204fcd1cd556d0336321e309cd8e8dff08 * improve csv builder Change-Id: I5b86694ea374ed2f78dfc049402f40e7de1fa508 * default to-path in an :fs put command to home directory and same filename as source Change-Id: I8e75b8aadd81625a1c4aff05851f1557496f4ef9 * fix class clast issue Change-Id: Ifb14d1ca93ca3702a06ff744b7e79f524ea0b5d8 * KNOX-2240 - address review comments Change-Id: Ibb4e4c3c3590aafb22d618b602de96e2cb4eb9f0 * KNOX-2240 - fix PMD failure Change-Id: I1d47c0aa60f6c409fb59b16c8b9e3f57070be8ca --- .../org/apache/knox/gateway/shell/KnoxSession.java | 38 ++ .../java/org/apache/knox/gateway/shell/Shell.java | 2 + .../shell/commands/AbstractKnoxShellCommand.java | 9 + .../shell/commands/AbstractSQLCommandSupport.java | 8 - .../knox/gateway/shell/commands/CSVCommand.java | 18 +- .../gateway/shell/commands/WebHDFSCommand.java | 426 +++++++++++++++++++++ .../org/apache/knox/gateway/shell/hdfs/Ls.java | 2 +- .../org/apache/knox/gateway/shell/hdfs/Mkdir.java | 2 +- .../org/apache/knox/gateway/shell/hdfs/Put.java | 2 +- .../org/apache/knox/gateway/shell/hdfs/Rm.java | 2 +- .../shell/table/CSVKnoxShellTableBuilder.java | 50 ++- .../knox/gateway/shell/table/KnoxShellTable.java | 10 +- .../shell/table/KnoxShellTableFileUtils.java | 47 +++ .../shell/table/KnoxShellTableJSONSerializer.java | 23 +- .../shell/table/KnoxShellTableRenderer.java | 16 +- .../gateway/shell/table/KnoxShellTableTest.java | 8 + .../org/apache/knox/gateway/util/JsonUtils.java | 15 + .../apache/knox/gateway/util/JsonUtilsTest.java | 45 ++- 18 files changed, 666 insertions(+), 57 deletions(-) diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java index 563dcd8..a3d691d 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java @@ -141,6 +141,7 @@ public class KnoxSession implements Closeable { private static final String KNOXSQLHISTORIES_JSON = "knoxsqlhistories.json"; private static final String KNOXDATASOURCES_JSON = "knoxdatasources.json"; + private static final String KNOXMOUNTPOINTS_JSON = "knoxmountpoints.json"; public Map<String, String> getHeaders() { return headers; @@ -607,6 +608,16 @@ public class KnoxSession implements Closeable { * @param map to persist */ public static <T> void persistDataSourcesToKnoxShell(String fileName, Map<String, T> map) { + persistMapToKnoxShell(fileName, map); + } + + /** + * Persist provided Map to a file within the {user.home}/.knoxshell directory + * @param <T> type of the value in the map + * @param fileName of persisted file + * @param map to persist + */ + public static <T> void persistMapToKnoxShell(String fileName, Map<String, T> map) { String s = JsonUtils.renderAsJsonString(map); String home = System.getProperty("user.home"); try { @@ -641,6 +652,10 @@ public class KnoxSession implements Closeable { persistDataSourcesToKnoxShell(KNOXDATASOURCES_JSON, datasources); } + public static void persistMountPoints(Map<String, String> mounts) { + persistMapToKnoxShell(KNOXMOUNTPOINTS_JSON, mounts); + } + /** * Load and return a map of datasource names to sql commands * from the {user.home}/.knoxshell/knoxsqlhistories.json file. @@ -661,6 +676,20 @@ public class KnoxSession implements Closeable { return sqlHistories; } + public static Map<String, String> loadMountPoints() throws IOException { + Map<String, String> mounts = new HashMap<>(); + String home = System.getProperty("user.home"); + + File mountFile = new File( + home + File.separator + + ".knoxshell" + File.separator + KNOXMOUNTPOINTS_JSON); + if (mountFile.exists()) { + String json = readFileToString(mountFile); + mounts = getMapFromJsonString(json); + } + return mounts; + } + private static String readFileToString(File file) throws IOException { String content = null; @@ -705,6 +734,15 @@ public class KnoxSession implements Closeable { return obj; } + public static <T> Map<String, T> getMapFromJsonString(String json) throws IOException { + Map<String, T> obj; + JsonFactory factory = new JsonFactory(); + ObjectMapper mapper = new ObjectMapper(factory); + TypeReference<Map<String, T>> typeRef = new TypeReference<Map<String, T>>() {}; + obj = mapper.readValue(json, typeRef); + return obj; + } + public static Map<String, KnoxDataSource> getMapOfDataSourcesFromJsonString(String json) throws IOException { Map<String, KnoxDataSource> obj; JsonFactory factory = new JsonFactory(); diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java index 6542dcb..c4a0be6 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java @@ -23,6 +23,7 @@ import org.apache.knox.gateway.shell.commands.AbstractSQLCommandSupport; import org.apache.knox.gateway.shell.commands.CSVCommand; import org.apache.knox.gateway.shell.commands.DataSourceCommand; import org.apache.knox.gateway.shell.commands.SelectCommand; +import org.apache.knox.gateway.shell.commands.WebHDFSCommand; import org.apache.knox.gateway.shell.hbase.HBase; import org.apache.knox.gateway.shell.hdfs.Hdfs; import org.apache.knox.gateway.shell.job.Job; @@ -96,6 +97,7 @@ public class Shell { shell.register(new SelectCommand(shell)); shell.register(new DataSourceCommand(shell)); shell.register(new CSVCommand(shell)); + shell.register(new WebHDFSCommand(shell)); shell.run( null ); } } diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java index 3fa400e..a24edad 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java @@ -18,6 +18,9 @@ package org.apache.knox.gateway.shell.commands; import java.util.List; + +import org.apache.knox.gateway.shell.CredentialCollectionException; +import org.apache.knox.gateway.shell.CredentialCollector; import org.codehaus.groovy.tools.shell.CommandSupport; import org.codehaus.groovy.tools.shell.Groovysh; @@ -43,4 +46,10 @@ public abstract class AbstractKnoxShellCommand extends CommandSupport { } return variableName; } + + protected CredentialCollector login() throws CredentialCollectionException { + KnoxLoginDialog dlg = new KnoxLoginDialog(); + dlg.collect(); + return dlg; + } } \ No newline at end of file diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java index eaeea01..d39699f 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java @@ -25,8 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.knox.gateway.shell.CredentialCollectionException; -import org.apache.knox.gateway.shell.CredentialCollector; import org.apache.knox.gateway.shell.KnoxDataSource; import org.apache.knox.gateway.shell.KnoxSession; import org.apache.knox.gateway.shell.jdbc.JDBCUtils; @@ -202,10 +200,4 @@ public abstract class AbstractSQLCommandSupport extends AbstractKnoxShellCommand } }); } - - protected CredentialCollector login() throws CredentialCollectionException { - KnoxLoginDialog dlg = new KnoxLoginDialog(); - dlg.collect(); - return dlg; - } } \ No newline at end of file diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java index 22aad2c..b8c3619 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java @@ -49,10 +49,24 @@ public class CSVCommand extends AbstractKnoxShellCommand { try { if (withHeaders) { - table = KnoxShellTable.builder().csv().withHeaders().url(url); + if (url.startsWith("$")) { + // a knoxshell variable is a csv file as a string + String csvString = (String) getVariables().get(url.substring(1)); + table = KnoxShellTable.builder().csv().withHeaders().string(csvString); + } + else { + table = KnoxShellTable.builder().csv().withHeaders().url(url); + } } else { - table = KnoxShellTable.builder().csv().url(url); + if (url.startsWith("$")) { + // a knoxshell variable is a csv file as a string + String csvString = (String) getVariables().get(url.substring(1)); + table = KnoxShellTable.builder().csv().string(csvString); + } + else { + table = KnoxShellTable.builder().csv().url(url); + } } } catch (IOException e) { e.printStackTrace(); diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java new file mode 100644 index 0000000..4dafc27 --- /dev/null +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.shell.commands; + +import java.io.Console; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +import org.apache.knox.gateway.shell.CredentialCollectionException; +import org.apache.knox.gateway.shell.CredentialCollector; +import org.apache.knox.gateway.shell.KnoxSession; +import org.apache.knox.gateway.shell.KnoxShellException; +import org.apache.knox.gateway.shell.hdfs.Hdfs; +import org.apache.knox.gateway.shell.hdfs.Status.Response; +import org.apache.knox.gateway.shell.table.KnoxShellTable; +import org.apache.knox.gateway.util.JsonUtils; +import org.codehaus.groovy.tools.shell.Groovysh; + +public class WebHDFSCommand extends AbstractKnoxShellCommand { + private Map<String, KnoxSession> sessions = new HashMap<>(); + + public WebHDFSCommand(Groovysh shell) { + super(shell, ":filesystem", ":fs"); + } + + @Override + public String getUsage() { + String usage = "Usage: \n" + + " :fs ls {target-path} \n" + + " :fs cat {target-path} \n" + + " :fs get {from-path} {to-path} \n" + + " :fs put {from-path} {tp-path} \n" + + " :fs rm {target-path} \n" + + " :fs mkdir {dir-path} \n"; + return usage; + } + + @Override + public Object execute(List<String> args) { + Map<String, String> mounts = getMountPoints(); + if (args.isEmpty()) { + args.add("ls"); + } + if (args.get(0).equalsIgnoreCase("mount")) { + String url = args.get(1); + String mountPoint = args.get(2); + return mount(mounts, url, mountPoint); + } + else if (args.get(0).equalsIgnoreCase("unmount")) { + String mountPoint = args.get(1); + unmount(mounts, mountPoint); + } + else if (args.get(0).equalsIgnoreCase("mounts")) { + return listMounts(mounts); + } + else if (args.get(0).equalsIgnoreCase("ls")) { + String path = args.get(1); + return listStatus(mounts, path); + } + else if (args.get(0).equalsIgnoreCase("put")) { + // Hdfs.put( session ).file( dataFile ).to( dataDir + "/" + dataFile ).now() + // :fs put from-path to-path + String localFile = args.get(1); + String path = args.get(2); + int permission = 755; + if (args.size() >= 4) { + permission = Integer.parseInt(args.get(3)); + } + + return put(mounts, localFile, path, permission); + } + else if (args.get(0).equalsIgnoreCase("rm")) { + // Hdfs.rm( session ).file( dataFile ).now() + // :fs rm target-path + String path = args.get(1); + return remove(mounts, path); + } + else if (args.get(0).equalsIgnoreCase("cat")) { + // println Hdfs.get( session ).from( dataDir + "/" + dataFile ).now().string + // :fs cat target-path + String path = args.get(1); + return cat(mounts, path); + } + else if (args.get(0).equalsIgnoreCase("mkdir")) { + // println Hdfs.mkdir( session ).dir( directoryPath ).perm( "777" ).now().string + // :fs mkdir target-path [perms] + String path = args.get(1); + String perms = null; + if (args.size() == 3) { + perms = args.get(2); + } + + return mkdir(mounts, path, perms); + } + else if (args.get(0).equalsIgnoreCase("get")) { + // println Hdfs.get( session ).from( dataDir + "/" + dataFile ).now().string + // :fs get from-path [to-path] + String path = args.get(1); + + String mountPoint = determineMountPoint(path); + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + String from = determineTargetPath(path, mountPoint); + String to = null; + if (args.size() > 2) { + to = args.get(2); + } + else { + to = System.getProperty("user.home") + File.separator + + path.substring(path.lastIndexOf(File.separator)); + } + return get(mountPoint, from, to); + } + else { + return "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + } + else { + System.out.println("Unknown filesystem command"); + System.out.println(getUsage()); + } + return ""; + } + + private String get(String mountPoint, String from, String to) { + String result = null; + try { + Hdfs.get(sessions.get(mountPoint)).from(from).file(to).now().getString(); + result = "Successfully copied: " + from + " to: " + to; + } catch (KnoxShellException | IOException e) { + e.printStackTrace(); + result = "Exception ocurred: " + e.getMessage(); + } + return result; + } + + private String mkdir(Map<String, String> mounts, String path, String perms) { + String result = null; + String mountPoint = determineMountPoint(path); + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + String targetPath = determineTargetPath(path, mountPoint); + if (!exists(session, targetPath)) { + try { + if (perms != null) { + Hdfs.mkdir(sessions.get(mountPoint)).dir(targetPath).now().getString(); + } + else { + Hdfs.mkdir(session).dir(targetPath).perm(perms).now().getString(); + } + result = "Successfully created directory: " + targetPath; + } catch (KnoxShellException | IOException e) { + e.printStackTrace(); + result = "Exception ocurred: " + e.getMessage(); + } + } + else { + result = targetPath + " already exists"; + } + } + else { + result = "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + return result; + } + + private String cat(Map<String, String> mounts, String path) { + String response = null; + String mountPoint = determineMountPoint(path); + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + String targetPath = determineTargetPath(path, mountPoint); + try { + String contents = Hdfs.get(session).from(targetPath).now().getString(); + response = contents; + } catch (KnoxShellException | IOException e) { + e.printStackTrace(); + response = "Exception ocurred: " + e.getMessage(); + } + } + else { + response = "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + return response; + } + + private String remove(Map<String, String> mounts, String path) { + String mountPoint = determineMountPoint(path); + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + String targetPath = determineTargetPath(path, mountPoint); + try { + Hdfs.rm(session).file(targetPath).now().getString(); + } catch (KnoxShellException | IOException e) { + e.printStackTrace(); + } + } + else { + return "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + return "Successfully removed: " + path; + } + + private String put(Map<String, String> mounts, String localFile, String path, int permission) { + String mountPoint = determineMountPoint(path); + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + String targetPath = determineTargetPath(path, mountPoint); + try { + boolean overwrite = false; + if (exists(session, targetPath)) { + if (collectClearInput(targetPath + " already exists would you like to overwrite (Y/n)").equalsIgnoreCase("y")) { + overwrite = true; + } + } + Hdfs.put(session).file(localFile).to(targetPath).overwrite(overwrite).permission(permission).now().getString(); + } catch (IOException e) { + e.printStackTrace(); + return "Exception ocurred: " + e.getMessage(); + } + } + else { + return "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + return "Successfully put: " + localFile + " to: " + path; + } + + private boolean exists(KnoxSession session, String path) { + boolean rc = false; + try { + Response response = Hdfs.status(session).file(path).now(); + rc = response.exists(); + } catch (KnoxShellException e) { + // NOP + } + return rc; + } + + private Object listStatus(Map<String, String> mounts, String path) { + Object response = null; + try { + String directory; + String mountPoint = determineMountPoint(path); + if (mountPoint != null) { + KnoxSession session = getSessionForMountPoint(mounts, mountPoint); + if (session != null) { + directory = determineTargetPath(path, mountPoint); + String json = Hdfs.ls(session).dir(directory).now().getString(); + Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map = + JsonUtils.getFileStatusesAsMap(json); + if (map != null) { + ArrayList<HashMap<String, String>> list = map.get("FileStatuses").get("FileStatus"); + KnoxShellTable table = buildTableFromListStatus(directory, list); + response = table; + } + } + else { + response = "No session established for mountPoint: " + mountPoint + " Use :fs mount {topology-url} {mountpoint-name}"; + } + } + else { + response = "No mountpoint found. Use ':fs mount {topologyURL} {mountpoint}'."; + } + } catch (KnoxShellException | IOException e) { + response = "Exception ocurred: " + e.getMessage(); + e.printStackTrace(); + } + return response; + } + + private KnoxShellTable listMounts(Map<String, String> mounts) { + KnoxShellTable table = new KnoxShellTable(); + table.header("Mount Point").header("Topology URL"); + for (String mountPoint : mounts.keySet()) { + table.row().value(mountPoint).value(mounts.get(mountPoint)); + } + return table; + } + + private void unmount(Map<String, String> mounts, String mountPoint) { + sessions.remove(mountPoint); + mounts.remove(mountPoint); + KnoxSession.persistMountPoints(mounts); + } + + private String mount(Map<String, String> mounts, String url, String mountPoint) { + KnoxSession session = establishSession(mountPoint, url); + if (session != null) { + mounts.put(mountPoint, url); + KnoxSession.persistMountPoints(mounts); + return url + " mounted as " + mountPoint; + } + return "Failed to mount " + url + " as " + mountPoint; + } + + private KnoxSession getSessionForMountPoint(Map<String, String> mounts, String mountPoint) { + KnoxSession session = sessions.get(mountPoint); + if (session == null) { + String url = mounts.get(mountPoint); + if (url != null) { + session = establishSession(mountPoint, url); + } + } + return session; + } + + private KnoxSession establishSession(String mountPoint, String url) { + CredentialCollector dlg; + try { + dlg = login(); + } catch (CredentialCollectionException e) { + e.printStackTrace(); + return null; + } + String username = dlg.name(); + String password = new String(dlg.chars()); + KnoxSession session = null; + try { + session = KnoxSession.login(url, username, password); + sessions.put(mountPoint, session); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return session; + } + + private String collectClearInput(String prompt) { + Console c = System.console(); + if (c == null) { + System.err.println("No console."); + System.exit(1); + } + + String value = c.readLine(prompt); + + return value; + } + + private String determineTargetPath(String path, String mountPoint) { + String directory = null; + if (path.startsWith("/")) { + directory = stripMountPoint(path, mountPoint); + } + return directory; + } + + private String stripMountPoint(String path, String mountPoint) { + String newPath = path.replace("/" + mountPoint, ""); + return newPath; + } + + private String determineMountPoint(String path) { + String mountPoint = null; + if (path.startsWith("/")) { + // does the user supplied path starts at a root + // if so check for a mountPoint based on the first element of the path + String[] pathElements = path.split("/"); + mountPoint = pathElements[1]; + } + return mountPoint; + } + + private KnoxShellTable buildTableFromListStatus(String directory, List<HashMap<String, String>> list) { + Calendar cal = Calendar.getInstance(TimeZone.getDefault(), Locale.getDefault()); + KnoxShellTable table = new KnoxShellTable(); + table.title(directory); + table.header("permission") + .header("owner") + .header("group") + .header("length") + .header("modtime") + .header("name"); + + for (Map<String, String> map : list) { + cal.setTimeInMillis(Long.parseLong(map.get("modificationTime"))); + table.row() + .value(map.get("permission")) + .value(map.get("owner")) + .value(map.get("group")) + .value(map.get("length")) + .value(cal.getTime()) + .value(map.get("pathSuffix")); + } + + return table; + } + + protected Map<String, String> getMountPoints() { + Map<String, String> mounts = null; + try { + mounts = KnoxSession.loadMountPoints(); + } catch (IOException e) { + e.printStackTrace(); + } + return mounts; + } + + public static void main(String[] args) { + WebHDFSCommand cmd = new WebHDFSCommand(new Groovysh()); + cmd.execute(new ArrayList<>(Arrays.asList(args))); + } +} diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java index 88d8d86..a219cdd 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java @@ -26,7 +26,7 @@ import org.apache.http.client.utils.URIBuilder; import java.util.concurrent.Callable; -class Ls { +public class Ls { public static class Request extends AbstractRequest<Response> { diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java index f927953..aca4479 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java @@ -26,7 +26,7 @@ import org.apache.http.client.utils.URIBuilder; import java.util.concurrent.Callable; -class Mkdir { +public class Mkdir { public static class Request extends AbstractRequest<Response> { diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java index 9d6e74e..82293be 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java @@ -34,7 +34,7 @@ import org.apache.http.util.EntityUtils; import java.io.File; import java.util.concurrent.Callable; -class Put { +public class Put { public static class Request extends AbstractRequest<Response> { diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java index 52f3a88..ee7026a 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java @@ -27,7 +27,7 @@ import org.apache.http.client.utils.URIBuilder; import java.io.IOException; import java.util.concurrent.Callable; -class Rm { +public class Rm { public static class Request extends AbstractRequest<Response> { diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java index bda2ab2..d3ceedc 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java @@ -18,7 +18,9 @@ package org.apache.knox.gateway.shell.table; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; @@ -39,33 +41,45 @@ public class CSVKnoxShellTableBuilder extends KnoxShellTableBuilder { } public KnoxShellTable url(String url) throws IOException { - int rowIndex = 0; URL urlToCsv = new URL(url); URLConnection connection = urlToCsv.openConnection(); try (Reader urlConnectionStreamReader = new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8); BufferedReader csvReader = new BufferedReader(urlConnectionStreamReader);) { - if (title != null) { - this.table.title(title); + buildTableFromCSVReader(csvReader); + } + return this.table; + } + + public KnoxShellTable string(String csvString) throws IOException { + try (InputStream is = new ByteArrayInputStream(csvString.getBytes(StandardCharsets.UTF_8)); Reader stringStreamReader = new InputStreamReader(is, StandardCharsets.UTF_8); + BufferedReader csvReader = new BufferedReader(stringStreamReader);) { + buildTableFromCSVReader(csvReader); + } + return this.table; + } + + private void buildTableFromCSVReader(BufferedReader csvReader) throws IOException { + int rowIndex = 0; + if (title != null) { + this.table.title(title); + } + String row = null; + while ((row = csvReader.readLine()) != null) { + boolean addingHeaders = (withHeaders && rowIndex == 0); + if (!addingHeaders) { + this.table.row(); } - String row = null; - while ((row = csvReader.readLine()) != null) { - boolean addingHeaders = (withHeaders && rowIndex == 0); - if (!addingHeaders) { - this.table.row(); - } - String[] data = row.split(","); + String[] data = row.split(",", -1); - for (String value : data) { - if (addingHeaders) { - this.table.header(value); - } else { - this.table.value(value); - } + for (String value : data) { + if (addingHeaders) { + this.table.header(value); + } else { + this.table.value(value); } - rowIndex++; } + rowIndex++; } - return this.table; } } diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java index 47439d4..4c5368a 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java @@ -137,6 +137,10 @@ public class KnoxShellTable { double[] colArray = new double[col.size()]; Conversions conversionMethod = null; for (int i = 0; i < col.size(); i++) { + Object v = col.get(i); + if (v instanceof String && ((String) v).trim().isEmpty()) { + col.set(i, "0"); + } if (i == 0) { conversionMethod = getConversion(col.get(i)); } @@ -435,6 +439,10 @@ public class KnoxShellTable { } public String toCSV() { - return new KnoxShellTableRenderer(this).toCSV(); + return toCSV(null); + } + + public String toCSV(String filePath) { + return new KnoxShellTableRenderer(this).toCSV(filePath); } } diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java new file mode 100644 index 0000000..5eb1c79 --- /dev/null +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.shell.table; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class KnoxShellTableFileUtils { + public static void persistToFile(String filePath, final String content) throws IOException { + final Path jsonFilePath = Paths.get(filePath); + if (!Files.exists(jsonFilePath.getParent())) { + Files.createDirectories(jsonFilePath.getParent()); + } + Files.deleteIfExists(jsonFilePath); + Files.createFile(jsonFilePath); + setPermissions(jsonFilePath); + Files.write(jsonFilePath, content.getBytes(StandardCharsets.UTF_8)); + } + + private static void setPermissions(Path path) throws IOException { + // clear all flags for everybody + path.toFile().setReadable(false, false); + path.toFile().setWritable(false, false); + path.toFile().setExecutable(false, false); + // allow owners to read/write + path.toFile().setReadable(true, true); + path.toFile().setWritable(true, true); + } +} diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java index 24850e2..bf10b28 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java @@ -18,10 +18,6 @@ package org.apache.knox.gateway.shell.table; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Locale; @@ -87,27 +83,10 @@ class KnoxShellTableJSONSerializer { } else { jsonResult = JsonUtils.renderAsJsonString(KnoxShellTableCallHistory.getInstance().getCallHistory(table.id), null, JSON_DATE_FORMAT.get()); } - final Path jsonFilePath = Paths.get(filePath); - if (!Files.exists(jsonFilePath.getParent())) { - Files.createDirectories(jsonFilePath.getParent()); - } - Files.deleteIfExists(jsonFilePath); - Files.createFile(jsonFilePath); - setPermissions(jsonFilePath); - Files.write(jsonFilePath, jsonResult.getBytes(StandardCharsets.UTF_8)); + KnoxShellTableFileUtils.persistToFile(filePath, jsonResult); return "Successfully saved into " + filePath; } catch (IOException e) { throw new KnoxShellException("Error while saving KnoxShellTable JSON into " + filePath, e); } } - - private static void setPermissions(Path path) throws IOException { - // clear all flags for everybody - path.toFile().setReadable(false, false); - path.toFile().setWritable(false, false); - path.toFile().setExecutable(false, false); - // allow owners to read/write - path.toFile().setReadable(true, true); - path.toFile().setWritable(true, true); - } } diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java index b38cac8..38f5bd2 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java @@ -17,6 +17,7 @@ */ package org.apache.knox.gateway.shell.table; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -38,6 +39,10 @@ class KnoxShellTableRenderer { } String toCSV() { + return toCSV(null); + } + + String toCSV(String filePath) { final StringBuilder csv = new StringBuilder(); String header; for (int i = 0; i < tableToRender.headers.size(); i++) { @@ -59,8 +64,17 @@ class KnoxShellTableRenderer { } } } + String content = csv.toString(); + if (filePath != null) { + try { + KnoxShellTableFileUtils.persistToFile(filePath, content); + } catch (IOException e) { + System.out.println("Persistence of CSV file has failed. " + e.getMessage()); + e.printStackTrace(); + } + } - return csv.toString(); + return content; } @Override diff --git a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java index e65278a..ac2f802 100644 --- a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java +++ b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java @@ -191,6 +191,14 @@ public class KnoxShellTableTest { } @Test + public void testCSVStringToTable() throws IOException { + String initialString = "colA, colB, colC\nvalue1, value2. value3\nvalue4, value5, value6"; + + KnoxShellTable table = KnoxShellTable.builder().csv().withHeaders().string(initialString); + assertEquals(table.rows.size(), 2); + } + + @Test public void testToAndFromJSON() throws IOException { KnoxShellTable table = new KnoxShellTable(); diff --git a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java index 4eabe2c..9d2e613 100644 --- a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java +++ b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java @@ -19,6 +19,7 @@ package org.apache.knox.gateway.util; import java.io.IOException; import java.text.DateFormat; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -97,4 +98,18 @@ public class JsonUtils { } return map; } + + public static Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> getFileStatusesAsMap(String json) { + Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map = null; + JsonFactory factory = new JsonFactory(); + ObjectMapper mapper = new ObjectMapper(factory); + TypeReference<HashMap<String,HashMap<String, ArrayList<HashMap<String, String>>>>> typeRef + = new TypeReference<HashMap<String,HashMap<String, ArrayList<HashMap<String, String>>>>>() {}; + try { + map = mapper.readValue(json, typeRef); + } catch (IOException e) { + //LOG.failedToGetMapFromJsonString( json, e ); + } + return map; + } } diff --git a/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java b/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java index 84fb7a1..05ae692 100644 --- a/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java +++ b/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java @@ -17,15 +17,16 @@ */ package org.apache.knox.gateway.util; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import org.junit.Test; - import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class JsonUtilsTest { private String expiresIn = "\"expires_in\":\"1364487943100\""; @@ -54,4 +55,46 @@ public class JsonUtilsTest { assertEquals("Bearer", map.get("token_type")); assertEquals("1364487943100", map.get("expires_in")); } + + @Test + public void testFileStatusesAsMap() { + String json = "{\n" + + " \"FileStatuses\":{\n" + + " \"FileStatus\":[\n" + + " {\n" + + " \"accessTime\":0,\n" + + " \"blockSize\":0,\n" + + " \"childrenNum\":3,\n" + + " \"fileId\":16389,\n" + + " \"group\":\"supergroup\",\n" + + " \"length\":0,\n" + + " \"modificationTime\":1581578495905,\n" + + " \"owner\":\"hdfs\",\n" + + " \"pathSuffix\":\"tmp\",\n" + + " \"permission\":\"1777\",\n" + + " \"replication\":0,\n" + + " \"storagePolicy\":0,\n" + + " \"type\":\"DIRECTORY\"\n" + + " },\n" + + " {\n" + + " \"accessTime\":0,\n" + + " \"blockSize\":0,\n" + + " \"childrenNum\":658,\n" + + " \"fileId\":16386,\n" + + " \"group\":\"supergroup\",\n" + + " \"length\":0,\n" + + " \"modificationTime\":1581578527580,\n" + + " \"owner\":\"hdfs\",\n" + + " \"pathSuffix\":\"user\",\n" + + " \"permission\":\"755\",\n" + + " \"replication\":0,\n" + + " \"storagePolicy\":0,\n" + + " \"type\":\"DIRECTORY\"\n" + + " } \n" + + " ]\n" + + " }\n" + + "}"; + Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map = JsonUtils.getFileStatusesAsMap(json); + assertNotNull(map); + } }