This is an automated email from the ASF dual-hosted git repository. lmccay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push: new 074a035 KNOX-2005 - Improvements to KnoxShellTable 074a035 is described below commit 074a0354b815968a162790f09189ff47ca04e169 Author: lmccay <lmc...@apache.org> AuthorDate: Tue Sep 3 22:53:34 2019 -0400 KNOX-2005 - Improvements to KnoxShellTable --- gateway-shell/pom.xml | 9 +- .../apache/knox/gateway/shell/KnoxShellTable.java | 334 ++++++++++++++++++++- .../java/org/apache/knox/gateway/shell/Shell.java | 3 +- .../knox/gateway/shell/KnoxShellTableTest.java | 119 +++++++- .../gateway/i18n/GatewayUtilCommonMessages.java | 2 + .../org/apache/knox/gateway/util/JsonUtils.java | 27 ++ 6 files changed, 487 insertions(+), 7 deletions(-) diff --git a/gateway-shell/pom.xml b/gateway-shell/pom.xml index 7f5f09d..139355b 100644 --- a/gateway-shell/pom.xml +++ b/gateway-shell/pom.xml @@ -110,6 +110,13 @@ <groupId>de.thetaphi</groupId> <artifactId>forbiddenapis</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> </dependencies> - </project> diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java index 9905c42..6c5e1b1 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java @@ -17,11 +17,27 @@ */ package org.apache.knox.gateway.shell; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.io.FileUtils; +import org.apache.knox.gateway.util.JsonUtils; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; /** * Simple table representation and text based rendering of a table via toString(). @@ -36,6 +52,12 @@ public class KnoxShellTable { private List<String> headers = new ArrayList<String>(); private List<List<String>> rows = new ArrayList<List<String>>(); + private String title; + + public KnoxShellTable title(String title) { + this.title = title; + return this; + } public KnoxShellTable header(String header) { headers.add(header); @@ -58,16 +80,106 @@ public class KnoxShellTable { return this; } + public KnoxShellTableCell cell(int colIndex, int rowIndex) { + return new KnoxShellTableCell(colIndex, rowIndex); + } + + public List<String> values(int colIndex) { + ArrayList<String> col = new ArrayList<String>(); + rows.forEach(row -> col.add(row.get(colIndex))); + return col; + } + + public KnoxShellTable apply(KnoxShellTableCell cell) { + if (!headers.isEmpty()) { + headers.set(cell.colIndex, cell.header); + } + if (!rows.isEmpty()) { + rows.get(cell.rowIndex).set(cell.colIndex, cell.value); + } + return this; + } + + public class KnoxShellTableCell { + private int colIndex; + private int rowIndex; + private String header; + private String value; + + KnoxShellTableCell(int colIndex, int rowIndex) { + this.colIndex = colIndex; + this.rowIndex = rowIndex; + if (!headers.isEmpty()) { + this.header = headers.get(colIndex); + } + if (!rows.isEmpty()) { + this.value = rows.get(rowIndex).get(colIndex); + } + } + + KnoxShellTableCell(String name, int rowIndex) { + this.rowIndex = rowIndex; + if (!headers.isEmpty()) { + this.header = name; + this.colIndex = headers.indexOf(name); + } + if (!rows.isEmpty()) { + this.value = rows.get(rowIndex).get(colIndex); + } + } + + public KnoxShellTableCell value(String value) { + this.value = value; + return this; + } + + public KnoxShellTableCell header(String name) { + this.header = name; + return this; + } + + public String value() { + return this.value; + } + + public String header() { + return this.header; + } + } + + public List<String> getHeaders() { + if (headers.isEmpty()) { + return null; + } + return headers; + } + + public List<List<String>> getRows() { + return rows; + } + + public String getTitle() { + return title; + } + @Override public String toString() { - if (!headers.isEmpty() && headers.size() != rows.get(0).size()) { - throw new IllegalStateException("Number of columns within rows and headers must be the same."); + if (!headers.isEmpty() && !rows.isEmpty() && headers.size() != rows.get(0).size()) { + throw new IllegalStateException("Number of columns and headers must be the same."); } StringBuilder sb = new StringBuilder(); Map<Integer, Integer> widthMap = getWidthMap(); - int colCount = rows.get(0).size(); + if (title != null && !title.isEmpty()) { + sb.append(this.title); + newLine(sb, 1); + } + int colCount = 0; + if (!rows.isEmpty()) { + colCount = rows.get(0).size(); + } if (!headers.isEmpty()) { + colCount = headers.size(); createBorder(sb, colCount, widthMap); newLine(sb, 1); @@ -145,7 +257,221 @@ public class KnoxShellTable { } } } - return map; } + + public static KnoxShellTableBuilder builder() { + return new KnoxShellTableBuilder(); + } + + public static class KnoxShellTableBuilder { + public CSVKnoxShellTableBuilder csv() { + return new CSVKnoxShellTableBuilder(); + } + + public JSONKnoxShellTableBuilder json() { + return new JSONKnoxShellTableBuilder(); + } + + public JoinKnoxShellTableBuilder join() { + return new JoinKnoxShellTableBuilder(); + } + } + + public static class JoinKnoxShellTableBuilder { + private KnoxShellTable left; + private KnoxShellTable right; + private int leftIndex = -1; + private int rightIndex = -1; + + public JoinKnoxShellTableBuilder() { + } + + public JoinKnoxShellTableBuilder left(KnoxShellTable left) { + this.left = left; + return this; + } + + public JoinKnoxShellTableBuilder right(KnoxShellTable right) { + this.right = right; + return this; + } + + public KnoxShellTable on(int leftIndex, int rightIndex) { + KnoxShellTable joined = new KnoxShellTable(); + + this.leftIndex = leftIndex; + this.rightIndex = rightIndex; + + joined.headers.addAll(new ArrayList<String>(left.headers)); + for (List<String> row : left.rows) { + joined.rows.add(new ArrayList<String>(row)); + } + List<String> col = right.values(leftIndex); + ArrayList<String> row; + String leftKey; + int matchedIndex; + + joined.headers.addAll(new ArrayList<String>(right.headers)); + for (Iterator<List<String>> it = joined.rows.iterator(); it.hasNext();) { + row = (ArrayList<String>) it.next(); + leftKey = row.get(leftIndex); + if (leftKey != null) { + matchedIndex = col.indexOf(leftKey); + if (matchedIndex > -1) { + row.addAll(right.rows.get(matchedIndex)); + } + else { + it.remove(); + } + } + } + return joined; + } + } + + public static class JSONKnoxShellTableBuilder { + boolean withHeaders; + + public JSONKnoxShellTableBuilder withHeaders() { + withHeaders = true; + return this; + } + + public KnoxShellTable string(String json) throws IOException { + KnoxShellTable table = getKnoxShellTableFromJsonString(json); + return table; + } + + public KnoxShellTable path(String path) throws IOException { + String json = FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8); + KnoxShellTable table = getKnoxShellTableFromJsonString(json); + return table; + } + + public static KnoxShellTable getKnoxShellTableFromJsonString(String json) { + KnoxShellTable table = null; + JsonFactory factory = new JsonFactory(); + ObjectMapper mapper = new ObjectMapper(factory); + TypeReference<KnoxShellTable> typeRef + = new TypeReference<KnoxShellTable>() {}; + try { + table = mapper.readValue(json, typeRef); + } catch (IOException e) { + //LOG.failedToGetMapFromJsonString( json, e ); + } + return table; + } + } + + public static class CSVKnoxShellTableBuilder { + boolean withHeaders; + + public CSVKnoxShellTableBuilder withHeaders() { + withHeaders = true; + return this; + } + + public KnoxShellTable url(String url) throws IOException { + int rowIndex = 0; + URLConnection connection; + BufferedReader csvReader = null; + KnoxShellTable table = null; + try { + URL urlToCsv = new URL(url); + connection = urlToCsv.openConnection(); + csvReader = new BufferedReader(new InputStreamReader( + connection.getInputStream(), StandardCharsets.UTF_8)); + table = new KnoxShellTable(); + String row = null; + while ((row = csvReader.readLine()) != null) { + boolean addingHeaders = (withHeaders && rowIndex == 0); + if (!addingHeaders) { + table.row(); + } + String[] data = row.split(","); + + for (String value : data) { + if (addingHeaders) { + table.header(value); + } + else { + table.value(value); + } + } + rowIndex++; + } + } + finally { + csvReader.close(); + } + return table; + } + } + + public String toJSON() { + return JsonUtils.renderAsJsonString(this); + } + + public String toCSV() { + StringBuilder csv = new StringBuilder(); + String header; + for(int i = 0; i < headers.size(); i++) { + header = headers.get(i); + csv.append(header); + if (i < headers.size() - 1) { + csv.append(','); + } + else { + csv.append('\n'); + } + } + for(List<String> row : rows) { + for(int ii = 0; ii < row.size(); ii++) { + csv.append(row.get(ii)); + if (ii < row.size() - 1) { + csv.append(','); + } + else { + csv.append('\n'); + } + } + } + + return csv.toString(); + } + + public KnoxShellTableFilter filter() { + return new KnoxShellTableFilter(); + } + + public class KnoxShellTableFilter { + String name; + int index; + + public KnoxShellTableFilter name(String name) { + this.name = name; + index = headers.indexOf(name); + return this; + } + + public KnoxShellTableFilter index(int index) { + this.index = index; + return this; + } + + public KnoxShellTable regex(String regex) { + KnoxShellTable table = new KnoxShellTable(); + table.headers.addAll(headers); + for (List<String> row : rows) { + if (Pattern.matches(regex, row.get(index))) { + table.row(); + row.forEach(value -> { + table.value(value); + }); + } + } + return table; + } + } } diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java index fa95af7..f0aaf41 100644 --- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java +++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java @@ -43,7 +43,8 @@ public class Shell { Workflow.class.getName(), Yarn.class.getName(), TimeUnit.class.getName(), - Manager.class.getName() + Manager.class.getName(), + KnoxShellTable.class.getName() }; static { diff --git a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java index e82fc45..c036c3f 100644 --- a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java +++ b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java @@ -20,6 +20,12 @@ package org.apache.knox.gateway.shell; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.FileUtils; +import org.apache.knox.gateway.shell.KnoxShellTable.KnoxShellTableCell; import org.junit.Test; public class KnoxShellTableTest { @@ -82,9 +88,11 @@ public class KnoxShellTableTest { table.row().value("789").value("012").value("844444444"); assertEquals(expectedResult, table.toString()); + + assertEquals(expectedResult, table.toString()); } - @Test (expected = IllegalStateException.class) + @Test(expected = IllegalStateException.class) public void testMultipleRowsTableMismatchedColAndHeadersCountError() { KnoxShellTable table = new KnoxShellTable(); @@ -94,5 +102,114 @@ public class KnoxShellTableTest { table.row().value("789").value("012").value("844444444"); assertNull(table.toString()); + table.toString(); + } + + @Test + public void testLoadCSVToAndFromURL() { +// table = KnoxShellTable.builder().db().driver("HiveDriver2").connect("sdklfjsdjflsd").sql("select * from test;"); + KnoxShellTable table = new KnoxShellTable(); + table.title("From URL"); + + table.header("Column A").header("Column B").header("Column C"); + + table.row().value("123").value("456").value("344444444"); + table.row().value("789").value("012").value("844444444"); + + String csv = table.toCSV(); + try { + // write file to /tmp to read back in + FileUtils.writeStringToFile(new File("/tmp/testtable.csv"), csv, StandardCharsets.UTF_8); + + KnoxShellTable urlTable = KnoxShellTable.builder().csv() + .withHeaders() + .url("file:///tmp/testtable.csv"); + //.url("http://samplecsvs.s3.amazonaws.com/Sacramentorealestatetransactions.csv"); + urlTable.title("From URL"); + assertEquals(urlTable.toString(), table.toString()); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testToAndFromJSON() throws IOException { + KnoxShellTable table = new KnoxShellTable(); + + table.header("Column A").header("Column B").header("Column C"); + + table.row().value("123").value("456").value("344444444"); + table.row().value("789").value("012").value("844444444"); + + String json = table.toJSON(); + + KnoxShellTable table2 = KnoxShellTable.builder().json().string(json); + assertEquals(table.toString(), table2.toString()); + } + + @Test + public void testCells() throws IOException { + KnoxShellTable table = new KnoxShellTable(); + + table.header("Column A").header("Column B").header("Column C"); + + table.row().value("123").value("456").value("344444444"); + table.row().value("789").value("012").value("844444444"); + + KnoxShellTableCell cell = table.cell(1, 1); + assertEquals(cell.header(), "Column B"); + assertEquals(cell.value(), "012"); + cell.header("Column Beeee"); + cell.value("234"); + table.apply(cell); + assertEquals(table.cell(1, 1).value(), "234"); + assertEquals(table.cell(1, 1).header(), "Column Beeee"); + } + + @Test + public void testFilterTable() { + KnoxShellTable table = new KnoxShellTable(); + + table.header("Column A").header("Column B").header("Column C"); + + table.row().value("123").value("456").value("344444444"); + table.row().value("789").value("012").value("844444444"); + + KnoxShellTable filtered = table.filter().name("Column A").regex("123"); + + assertEquals(filtered.getRows().size(), 1); + } + + @Test + public void testJoinTables() throws IOException { + KnoxShellTable table = new KnoxShellTable(); + + table.title("Left Table").header("Column A").header("Column B").header("Column C"); + + table.row().value("123").value("456").value("344444444"); + table.row().value("789").value("012").value("844444444"); + + KnoxShellTable table2 = new KnoxShellTable(); + + table2.title("Right Table").header("Column D").header("Column E").header("Column F"); + + table2.row().value("123").value("367").value("244444444"); + table2.row().value("780").value("908").value("944444444"); + + KnoxShellTable joined = KnoxShellTable.builder().join().left(table).right(table2).on(0, 0); + joined.title("Joined Table"); + + assertEquals(joined.getRows().size(), 1); + assertEquals(joined.getTitle(), "Joined Table"); + assertEquals(joined.cell(0, 0).value(), "123"); + String json = joined.toJSON(); + + KnoxShellTable zombie = KnoxShellTable.builder().json().string(json); + zombie.title("Zombie Table"); + + assertEquals(zombie.getRows().size(), 1); + assertEquals(zombie.getTitle(), "Zombie Table"); + assertEquals(zombie.cell(0, 0).value(), "123"); } } diff --git a/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java b/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java index 33572f5..96a8e32 100644 --- a/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java +++ b/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java @@ -36,4 +36,6 @@ public interface GatewayUtilCommonMessages { @Message( level = MessageLevel.ERROR, text = "Error in generating certificate: {0}" ) void failedToGenerateCertificate( @StackTrace( level = MessageLevel.ERROR ) Exception e ); + @Message(level = MessageLevel.ERROR, text = "Failed to serialize Object to Json string {0}: {1}" ) + void failedToSerializeObjectToJSON( Object obj, @StackTrace( level = MessageLevel.DEBUG ) Exception e ); } diff --git a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java index 6662767..28feea2 100644 --- a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java +++ b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java @@ -45,6 +45,33 @@ public class JsonUtils { return json; } + public static String renderAsJsonString(Object obj) { + String json = null; + ObjectMapper mapper = new ObjectMapper(); + + try { + // write JSON to a file + json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj); + } catch ( JsonProcessingException e ) { + LOG.failedToSerializeObjectToJSON( obj, e ); + } + return json; + } + + public static Object getObjectFromJsonString(String json) { + Map<String, String> obj = null; + JsonFactory factory = new JsonFactory(); + ObjectMapper mapper = new ObjectMapper(factory); + TypeReference<Object> typeRef + = new TypeReference<Object>() {}; + try { + obj = mapper.readValue(json, typeRef); + } catch (IOException e) { + LOG.failedToGetMapFromJsonString( json, e ); + } + return obj; + } + public static Map<String, String> getMapFromJsonString(String json) { Map<String, String> map = null; JsonFactory factory = new JsonFactory();