Repository: ambari Updated Branches: refs/heads/trunk 810a32abb -> 079293935
AMBARI-13747. Hive View : Add Database/Table creation from File. (Nitiraj Singh Rathore via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/07929393 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/07929393 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/07929393 Branch: refs/heads/trunk Commit: 079293935aed3650f5240c1b2dc2975d2875b82f Parents: 810a32a Author: Yusaku Sako <yus...@hortonworks.com> Authored: Fri Nov 6 16:50:15 2015 -0800 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Fri Nov 6 16:50:15 2015 -0800 ---------------------------------------------------------------------- .../view/hive/client/ColumnDescription.java | 18 ++ .../view/hive/client/ConnectionFactory.java | 19 +- .../org/apache/ambari/view/hive/client/Row.java | 24 ++ .../view/hive/resources/uploads/CSVParser.java | 185 ++++++++++++ .../uploads/ColumnDescriptionImpl.java | 95 ++++++ .../view/hive/resources/uploads/DataParser.java | 63 ++++ .../view/hive/resources/uploads/IParser.java | 37 +++ .../hive/resources/uploads/ParseOptions.java | 46 +++ .../view/hive/resources/uploads/ParseUtils.java | 103 +++++++ .../hive/resources/uploads/QueryGenerator.java | 66 +++++ .../view/hive/resources/uploads/TableInfo.java | 62 ++++ .../hive/resources/uploads/UploadService.java | 296 +++++++++++++++++++ .../ui/hive-web/app/adapters/file-upload.js | 31 ++ .../ui/hive-web/app/adapters/upload-table.js | 76 +++++ .../ui/hive-web/app/components/file-upload.js | 25 ++ .../ui/hive-web/app/components/navbar-widget.js | 5 +- .../ui/hive-web/app/controllers/upload-table.js | 261 ++++++++++++++++ .../ui/hive-web/app/initializers/i18n.js | 7 +- .../main/resources/ui/hive-web/app/router.js | 1 + .../resources/ui/hive-web/app/styles/app.scss | 3 +- .../ui/hive-web/app/templates/upload-table.hbs | 92 ++++++ .../ui/hive-web/app/utils/constants.js | 1 + .../src/main/resources/ui/hive-web/bower.json | 1 + .../src/main/resources/ui/hive-web/package.json | 1 + contrib/views/hive/src/main/resources/view.xml | 6 + .../hive/resources/upload/DataParserTest.java | 65 ++++ 26 files changed, 1584 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java index d7ea560..a25571f 100644 --- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java @@ -19,6 +19,24 @@ package org.apache.ambari.view.hive.client; public interface ColumnDescription { + enum DataTypes { + TINYINT, // + SMALLINT, // + INT, // + BIGINT, // + BOOLEAN, // + FLOAT, // + DOUBLE, // + STRING, // + BINARY, // -- (Note: Available in Hive 0.8.0 and later) + TIMESTAMP, // -- (Note: Available in Hive 0.8.0 and later) + DECIMAL, // -- (Note: Available in Hive 0.11.0 and later) + // DECIMAL,(precision, scale)Â -- (Note: Available in Hive 0.13.0 and later) Not included. + DATE, // -- (Note: Available in Hive 0.12.0 and later) + VARCHAR, // -- (Note: Available in Hive 0.12.0 and later) + CHAR, // -- (Note: Available in Hive 0.13.0 and later) + } + public abstract String getName(); public abstract void setName(String name); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java index 82ac1f5..1442748 100644 --- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java @@ -24,10 +24,11 @@ import org.apache.ambari.view.hive.utils.ServiceFormattedException; import org.apache.ambari.view.utils.UserLocalFactory; import org.apache.ambari.view.utils.ambari.AmbariApi; import org.apache.ambari.view.utils.ambari.AmbariApiException; +import org.apache.ambari.view.utils.hdfs.HdfsApi; +import org.apache.ambari.view.utils.hdfs.HdfsUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,6 +39,7 @@ public class ConnectionFactory implements UserLocalFactory<Connection> { private ViewContext context; private HiveAuthCredentials credentials; private AmbariApi ambariApi; + private HdfsApi hdfsApi = null; public ConnectionFactory(ViewContext context, HiveAuthCredentials credentials) { this.context = context; @@ -45,6 +47,21 @@ public class ConnectionFactory implements UserLocalFactory<Connection> { this.ambariApi = new AmbariApi(context); } + /** + * Get HdfsApi instance + * @return HdfsApi business delegate + */ + public synchronized HdfsApi getHDFSApi() { + if (hdfsApi == null) { + try { + hdfsApi = HdfsUtil.connectToHDFSApi(context); + } catch (Exception ex) { + throw new ServiceFormattedException("HdfsApi connection failed. Check \"webhdfs.url\" property", ex); + } + } + return hdfsApi; + } + @Override public Connection create() { try { http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java index 306530a..35f216b 100644 --- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java @@ -18,6 +18,7 @@ package org.apache.ambari.view.hive.client; +import java.util.Arrays; import java.util.HashSet; public class Row { @@ -47,4 +48,27 @@ public class Row { public void setRow(Object[] row) { this.row = row; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Row row1 = (Row) o; + + // Probably incorrect - comparing Object[] arrays with Arrays.equals + return Arrays.equals(row, row1.row); + } + + @Override + public int hashCode() { + return Arrays.hashCode(row); + } + + @Override + public String toString() { + return "Row{" + + "row=" + Arrays.toString(row) + + '}'; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java new file mode 100644 index 0000000..388cf53 --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.apache.ambari.view.hive.client.ColumnDescriptionShort; +import org.apache.ambari.view.hive.client.Row; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; + +import java.io.*; +import java.util.*; + +/** + * Parses the given Reader and extracts headers and rows, and detect datatypes of columns + */ +public class CSVParser implements IParser { + + static class CSVIterator implements Iterator<Row> { + + private Iterator<CSVRecord> iterator; + + public CSVIterator(Iterator<CSVRecord> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Row next() { + CSVRecord row = iterator.next(); + Object[] values = new Object[row.size()]; + for (int i = 0; i < values.length; i++) { + values[i] = row.get(i); + } + Row r = new Row(values); + return r; + } + + @Override + public void remove() { + this.iterator.remove(); + } + } + + private Reader originalReader; // same as CSV reader in this case + private ParseOptions parseOptions; + private CSVIterator iterator; + private List<Row> previewRows; + private List<ColumnDescription> header; + private boolean isHeaderFirstRow = false; + private int numberOfPreviewRows = 10; + private org.apache.commons.csv.CSVParser parser; + + public CSVParser(Reader reader, ParseOptions parseOptions) throws IOException { + this.originalReader = reader; + this.parseOptions = parseOptions; + // always create without headers + parser = new org.apache.commons.csv.CSVParser(reader, CSVFormat.EXCEL); + iterator = new CSVIterator(parser.iterator()); + } + + public void parsePreview() { + try { + numberOfPreviewRows = (Integer) parseOptions.getOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS); + } catch (Exception e) { + } + + int numberOfRows = numberOfPreviewRows; + previewRows = new ArrayList<Row>(numberOfPreviewRows); // size including the header. + + Row headerRow = null; + if (parseOptions.getOption(ParseOptions.OPTIONS_HEADER).equals(ParseOptions.HEADER_FIRST_RECORD)) { + if (!this.iterator().hasNext()) { + throw new NoSuchElementException("Cannot parse Header"); + } + isHeaderFirstRow = true; + headerRow = iterator().next(); + previewRows.add(headerRow); + } + + // find data types. + int[][] typeCounts = null; + Row r = null; + int numOfCols = 0; + if (iterator().hasNext()) { + r = iterator().next(); + numOfCols = r.getRow().length; + typeCounts = new int[numOfCols][ColumnDescription.DataTypes.values().length]; + } else { + throw new NoSuchElementException("No rows in the csv."); + } + + while (true) { + // create Header definition from row + Object[] values = r.getRow(); + previewRows.add(r); + + if (values.length != numOfCols) + throw new IllegalArgumentException("Illegal number of cols for row : " + r); + + for (int colNum = 0; colNum < values.length; colNum++) { + // detect type + ColumnDescription.DataTypes type = ParseUtils.detectHiveDataType(values[colNum]); + typeCounts[colNum][type.ordinal()]++; + } + numberOfRows--; + if (numberOfRows <= 0 || !iterator().hasNext()) + break; + + r = iterator().next(); + } + ; + + if (previewRows.size() <= 0) + throw new NoSuchElementException("Does not contain any rows."); + + header = new ArrayList<ColumnDescription>(numOfCols); + for (int colNum = 0; colNum < numOfCols; colNum++) { + int dataTypeId = getLikelyDataType(typeCounts, colNum); + ColumnDescription.DataTypes type = ColumnDescription.DataTypes.values()[dataTypeId]; + String colName = "Column" + colNum; + if (null != headerRow) + colName = (String) headerRow.getRow()[colNum]; + + ColumnDescription cd = new ColumnDescriptionImpl(colName, type.toString(), colNum); + header.add(cd); + } + } + + /** + * returns which datatype was detected for the maximum number of times in the given column + * @param typeCounts + * @param colNum + * @return + */ + private int getLikelyDataType(int[][] typeCounts, int colNum) { + int[] colArray = typeCounts[colNum]; + int maxIndex = 0; + int i = 1; + for (; i < colArray.length; i++) { + if (colArray[i] > colArray[maxIndex]) + maxIndex = i; + } + + return maxIndex; + } + + @Override + public Reader getCSVReader() { + return originalReader; + } + + @Override + public List<ColumnDescription> getHeader() { + return header; + } + + @Override + public List<Row> getPreviewRows() { + return this.previewRows; + } + + public Iterator<Row> iterator() { + return iterator; // only one iterator per parser. + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java new file mode 100644 index 0000000..50f5036 --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; + +import java.io.Serializable; + +public class ColumnDescriptionImpl implements ColumnDescription, Serializable { + private String name; + private String type; + private int position; + + public ColumnDescriptionImpl(String name, String type, int position) { + this.name = name; + this.type = type; + this.position = position; + } + + @Override + public String getName() { + return name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getType() { + return type; + } + + @Override + public void setType(String type) { + this.type = type; + } + + @Override + public int getPosition() { + return this.position; + } + + @Override + public void setPosition(int position) { + this.position = position; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ColumnDescriptionImpl that = (ColumnDescriptionImpl) o; + + if (position != that.position) return false; + if (!name.equals(that.name)) return false; + return type.equals(that.type); + + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + type.hashCode(); + result = 31 * result + position; + return result; + } + + @Override + public String toString() { + return new StringBuilder().append("ColumnDescriptionImpl[") + .append("name : ").append(name) + .append("type : " + type) + .append("position : " + position) + .append("]").toString(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java new file mode 100644 index 0000000..5f2db55 --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.apache.ambari.view.hive.client.Row; + +import java.io.IOException; +import java.io.Reader; +import java.util.Iterator; +import java.util.List; + +public class DataParser implements IParser { + + private IParser parser; + + public DataParser(Reader reader, ParseOptions parseOptions) throws IOException { + if (parseOptions.getOption(ParseOptions.OPTIONS_FILE_TYPE).equals(ParseOptions.FILE_TYPE_CSV)) { + parser = new CSVParser(reader, parseOptions); + } + } + + @Override + public Reader getCSVReader() { + return parser.getCSVReader(); + } + + @Override + public List<ColumnDescription> getHeader() { + return parser.getHeader(); + } + + @Override + public List<Row> getPreviewRows() { + return parser.getPreviewRows(); + } + + @Override + public void parsePreview() { + parser.parsePreview(); + } + + @Override + public Iterator<Row> iterator() { + return parser.iterator(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java new file mode 100644 index 0000000..c478b70 --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.apache.ambari.view.hive.client.Row; + +import java.io.File; +import java.io.InputStream; +import java.io.Reader; +import java.util.List; + +public interface IParser extends Iterable<Row> { + public Reader getCSVReader(); + + public List<ColumnDescription> getHeader(); + + public List<Row> getPreviewRows(); + + public void parsePreview(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java new file mode 100644 index 0000000..2ec3b1b --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import java.util.HashMap; + +public class ParseOptions { + final public static String OPTIONS_FILE_TYPE = "FILE_TYPE"; + final public static String OPTIONS_HEADER = "HEADER"; + final public static String OPTIONS_NUMBER_OF_PREVIEW_ROWS = "NUMBER_OF_PREVIEW_ROWS"; + + final public static String FILE_TYPE_CSV = "CSV"; + final public static String FILE_TYPE_JSON = "JSON"; + final public static String XML = "XML"; + + final public static String HEADER_FIRST_RECORD = "FIRST_RECORD"; + final public static String HEADER_PROVIDED_BY_USER = "PROVIDED_BY_USER"; + + final public static String HEADERS = "HEADERS"; + + private HashMap<String, Object> options = new HashMap<String, Object>(); + + public void setOption(String key, Object value) { + this.options.put(key, value); + } + + public Object getOption(String key) { + return this.options.get(key); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java new file mode 100644 index 0000000..aea370e --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class ParseUtils { + + final public static String[] DATE_FORMATS = {"mm/dd/yyyy", "dd/mm/yyyy", "mm-dd-yyyy" /*add more formatss*/}; + + public static boolean isInteger(Object object) { + if (object == null) + return false; + + if (object instanceof Integer) + return true; + + try { + Integer i = Integer.parseInt(object.toString()); + return true; + } catch (NumberFormatException nfe) { + return false; + } + } + + public static boolean isDouble(Object object) { + if (object == null) + return false; + + if (object instanceof Double) + return true; + + try { + Double i = Double.parseDouble(object.toString()); + return true; + } catch (NumberFormatException nfe) { + return false; + } + } + + public static boolean isChar(Object object) { + if (object == null) + return false; + + if (object instanceof Character) + return true; + + String str = object.toString().trim(); + if (str.length() == 1) + return true; + + return false; + } + + public static boolean isDate(Object object) { + if (object == null) + return false; + + if (object instanceof Date) + return true; + + String str = object.toString(); + for (String format : DATE_FORMATS) { + try { + Date i = new SimpleDateFormat(format).parse(str); + return true; + } catch (Exception e) { + } + } + + return false; + } + + public static ColumnDescription.DataTypes detectHiveDataType(Object object) { + // detect Integer + if (isInteger(object)) return ColumnDescription.DataTypes.INT; + if (isDouble(object)) return ColumnDescription.DataTypes.DOUBLE; + if (isDate(object)) return ColumnDescription.DataTypes.DATE; + if (isChar(object)) return ColumnDescription.DataTypes.CHAR; + + return ColumnDescription.DataTypes.STRING; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java new file mode 100644 index 0000000..98616cf --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * generates the sql query from given data + */ +public class QueryGenerator { + protected final static Logger LOG = + LoggerFactory.getLogger(QueryGenerator.class); + + public String generateCreateQuery(TableInfo tableInfo) { + String tableName = tableInfo.getTableName(); + List<ColumnDescription> cdList = tableInfo.getColumns(); + + StringBuilder query = new StringBuilder(); + query.append("create table " + tableName + " ("); + Collections.sort(cdList, new Comparator<ColumnDescription>() { + @Override + public int compare(ColumnDescription o1, ColumnDescription o2) { + return o1.getPosition() - o2.getPosition(); + } + }); + + boolean first = true; + for (ColumnDescription cd : cdList) { + if (first) { + first = false; + } else { + query.append(", "); + } + + query.append(cd.getName() + " " + cd.getType()); + } + + query.append(") ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE;"); + + String queryString = query.toString(); + LOG.info("Query : %S", queryString); + return queryString; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java new file mode 100644 index 0000000..ed4943d --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import org.apache.ambari.view.hive.client.ColumnDescription; + +import java.util.List; + +public class TableInfo { + private String tableName; + private String databaseName; + private List<ColumnDescription> columns; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getDatabaseName() { + return databaseName; + } + + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + + public List<ColumnDescription> getColumns() { + return columns; + } + + public void setColumns(List<ColumnDescription> columns) { + this.columns = columns; + } + + public TableInfo(String databaseName, String tableName, List<ColumnDescription> columns) { + this.tableName = tableName; + this.databaseName = databaseName; + this.columns = columns; + } + + public TableInfo() { + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java new file mode 100644 index 0000000..2098a5f --- /dev/null +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java @@ -0,0 +1,296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.uploads; + +import com.sun.jersey.core.header.FormDataContentDisposition; +import com.sun.jersey.multipart.FormDataParam; +import org.apache.ambari.view.hive.BaseService; +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.apache.ambari.view.hive.client.HiveClientException; +import org.apache.ambari.view.hive.persistence.utils.ItemNotFound; +import org.apache.ambari.view.hive.resources.jobs.NoOperationStatusSetException; +import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job; +import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobController; +import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl; +import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobResourceManager; +import org.apache.ambari.view.hive.utils.ServiceFormattedException; +import org.apache.ambari.view.hive.utils.SharedObjectsFactory; +import org.apache.ambari.view.utils.ambari.AmbariApi; +import org.apache.commons.io.input.ReaderInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.*; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Servlet for queries + * API: + * POST /preview + * POST /upload + * POST /createTable + * GET /createTable/status + */ +public class UploadService extends BaseService { + + private AmbariApi ambariApi; + + protected JobResourceManager resourceManager; + + final private String HIVE_META_STORE_LOCATION_KEY = "hive.metastore.warehouse.dir"; + final private String HIVE_SITE = "hive-site"; + + @POST + @Path("/preview") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response uploadForPreview( + @FormDataParam("file") InputStream uploadedInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail) { + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER_FIRST_RECORD); + + try { + DataParser dataParser = new DataParser(new InputStreamReader(uploadedInputStream), parseOptions); + + dataParser.parsePreview(); + + Map<String, Object> retData = new HashMap<String, Object>(); + retData.put("header", dataParser.getHeader()); + retData.put("rows", dataParser.getPreviewRows()); + retData.put("isFirstRowHeader", true); + + JSONObject jsonObject = new JSONObject(retData); + return Response.ok(jsonObject).build(); + } catch (IOException e) { + throw new ServiceFormattedException(e.getMessage(), e); + } + } + + public static class TableInput { + public Boolean isFirstRowHeader; + public String header; + public String tableName; + public String databaseName; + + public TableInput() { + } + + public Boolean getIsFirstRowHeader() { + return isFirstRowHeader; + } + + public void setIsFirstRowHeader(Boolean isFirstRowHeader) { + this.isFirstRowHeader = isFirstRowHeader; + } + + public String getHeader() { + return header; + } + + public void setHeader(String header) { + this.header = header; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getDatabaseName() { + return databaseName; + } + + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + } + + @Path("/createTable") + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response createTable(TableInput tableInput) throws IllegalAccessException, InvocationTargetException, ItemNotFound, NoSuchMethodException { + String header = tableInput.getHeader(); + String databaseName = tableInput.getDatabaseName(); + String tableName = tableInput.getTableName(); + Boolean isFirstRowHeader = (Boolean) tableInput.getIsFirstRowHeader(); + + Object headerObj = JSONValue.parse(header); + JSONArray array = (JSONArray) headerObj; + List<ColumnDescription> cdList = new ArrayList<ColumnDescription>(array.size()); + for (Object o : array) { + JSONObject jo = (JSONObject) o; + String name = (String) jo.get("name"); + String type = (String) jo.get("type"); + Long p = (Long) jo.get("position"); + Integer position = p != null ? p.intValue() : 0; + + ColumnDescriptionImpl cdi = new ColumnDescriptionImpl(name, type, position); + cdList.add(cdi); + } + + Map jobInfo = new HashMap<String, String>();//PropertyUtils.describe(request.job); + jobInfo.put("title", "Internal Table Creation"); + jobInfo.put("forcedContent", generateCreateQuery(databaseName, tableName, cdList)); + jobInfo.put("dataBase", databaseName); + + LOG.info("jobInfo : " + jobInfo); + Job job = new JobImpl(jobInfo); + LOG.info("job : " + job); + getResourceManager().create(job); + + JobController createdJobController = getResourceManager().readController(job.getId()); + createdJobController.submit(); + getResourceManager().saveIfModified(createdJobController); + + String filePath = databaseName + ".db/" + tableName + "/" + tableName + ".csv"; + + JSONObject jobObject = new JSONObject(); + jobObject.put("jobId", job.getId()); + jobObject.put("filePath", filePath); + + LOG.info("Create table query submitted : file should be uploaded at location : {}", filePath); + return Response.ok(jobObject).status(201).build(); + } + + @Path("/createTable/status") + @GET + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response isTableCreated(@QueryParam("jobId") int jobId) throws IllegalAccessException, NoSuchMethodException, InvocationTargetException, ItemNotFound, HiveClientException, NoOperationStatusSetException { + JobController jobController = getResourceManager().readController(jobId + ""); + LOG.info("jobController.getStatus().status : {} for job : {}", jobController.getStatus().status, jobController.getJob().getId()); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status", jobController.getStatus().status); + return Response.ok(jsonObject).build(); + } + + @Path("/upload") + @POST + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response uploadFile( + @FormDataParam("file") InputStream uploadedInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @FormDataParam("isFirstRowHeader") Boolean isFirstRowHeader, + @FormDataParam("filePath") String filePath + + ) throws IOException, InterruptedException { + LOG.info("inside uploadFile : isFirstRowHeader : {} , filePath : {}", isFirstRowHeader, filePath); +/* This is not working as expected. + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV); + parseOptions.setOption(ParseOptions.HEADERS,cdList); + + if(isFirstRowHeader) + parseOptions.setOption(ParseOptions.OPTIONS_HEADER,ParseOptions.HEADER_FIRST_RECORD); + else + parseOptions.setOption(ParseOptions.OPTIONS_HEADER,ParseOptions.HEADER_PROVIDED_BY_USER); + + DataParser dataParser = new DataParser(new InputStreamReader(uploadedInputStream),parseOptions); + + // remove first row if it is header and send the rest to HDFS + if(isFirstRowHeader){ + if( dataParser.iterator().hasNext() ){ + dataParser.iterator().next(); + } + } + + Reader csvReader = dataParser.getCSVReader(); +*/ + + // TODO : workaround alert as above method is not working properly + // remove first row if it is header and send the rest to HDFS + Reader r = new InputStreamReader(uploadedInputStream); + if (isFirstRowHeader) { + BufferedReader br = new BufferedReader(r, 1); // + br.readLine(); // TODO : remove the header line. Wrong if first record is beyond first endline + } + + + String basePath = getHiveMetaStoreLocation(); + if (null == basePath) + basePath = "/apps/hive/warehouse"; + + if (!basePath.endsWith("/")) + basePath = basePath + "/"; + + String fullPath = basePath + filePath; + + uploadTable(new ReaderInputStream(r), fullPath); + + LOG.info("create the table successfully at : {}", fullPath); + return Response.ok().build(); + } + + private String getHiveMetaStoreLocation() { + return this.getAmbariApi().getCluster().getConfigurationValue(HIVE_SITE, HIVE_META_STORE_LOCATION_KEY); + } + + private void uploadTable(InputStream is, String path) throws IOException, InterruptedException { + if (!path.endsWith("/")) { + path = path + "/"; + } + + uploadFile(path, is); + } + + private void uploadFile(final String filePath, InputStream uploadedInputStream) + throws IOException, InterruptedException { + byte[] chunk = new byte[1024]; + FSDataOutputStream out = getSharedObjectsFactory().getHdfsApi().create(filePath, false); + while (uploadedInputStream.read(chunk) != -1) { + out.write(chunk); + } + out.close(); + } + + + protected synchronized JobResourceManager getResourceManager() { + if (resourceManager == null) { + SharedObjectsFactory connectionsFactory = getSharedObjectsFactory(); + resourceManager = new JobResourceManager(connectionsFactory, context); + } + return resourceManager; + } + + protected synchronized AmbariApi getAmbariApi() { + if (null == ambariApi) { + ambariApi = new AmbariApi(this.context); + } + return ambariApi; + } + + private String generateCreateQuery(String databaseName, String tableName, List<ColumnDescription> cdList) { + return new QueryGenerator().generateCreateQuery(new TableInfo(databaseName, tableName, cdList)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js new file mode 100644 index 0000000..1bd8eee --- /dev/null +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import EmberUploader from 'ember-uploader'; +import Ember from 'ember'; + +export default EmberUploader.Uploader.extend({ + headers: {}, + + // Override + _ajax: function(settings) { + settings = Ember.merge(settings, this.getProperties('headers')); + console.log("_ajax : settings: " + JSON.stringify(settings)); + return this._super(settings); + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js new file mode 100644 index 0000000..6a9c54b --- /dev/null +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import EmberUploader from 'ember-uploader'; +import Ember from 'ember'; +import application from './application'; +import FileUploader from './file-upload'; + +export default application.extend({ + hdrs : function(){ + console.log("inside hdrs : headers : ",this.get('headers')); + var h = Ember.$.extend(true, {},this.get('headers')); + delete h['Content-Type']; + return h; + }.property('headers'), + + buildUploadURL: function (path) { + return this.buildURL() + "/resources/upload/" + path; + }, + + uploadFiles: function (path, files, extras) { + var uploadUrl = this.buildUploadURL(path); + + console.log("uplaoder : uploadURL : ", uploadUrl); + console.log("uploader : extras : ", extras); + console.log("uploader : files : ", files); + + console.log("hdrs : ", this.get('hdrs')); + var uploader = FileUploader.create({ + headers: this.get('hdrs'), + url: uploadUrl + }); + + if (!Ember.isEmpty(files)) { + var promise = uploader.upload(files[0], extras); + return promise; + } + }, + + createTable: function (tableData) { + var _this = this; + var postHeader = JSON.parse(JSON.stringify(this.get('headers'))); + console.log("headers postHeadesfsfdfsfsfss : : " , postHeader); + return Ember.$.ajax( { + url : this.buildUploadURL("createTable"), + type : 'post', + data: JSON.stringify(tableData), + headers: this.get('headers'), + dataType : 'json' + } + ); + }, + + getCreateTableResult : function(jobId){ + return Ember.$.ajax(this.buildUploadURL("createTable/status"),{ + data : {"jobId":jobId}, + type: "get", + headers: this.get('headers') + }); + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js new file mode 100644 index 0000000..1cd05ae --- /dev/null +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import EmberUploader from 'ember-uploader'; + +export default EmberUploader.FileField.extend({ + filesDidChange: function(files) { + this.sendAction('filesUploaded',files); // sends this action to controller. + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js index c3659cf..11333d0 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js @@ -34,6 +34,9 @@ export default Ember.Component.extend({ path: constants.namingConventions.routes.history}), Ember.Object.create({text: 'menus.udfs', - path: constants.namingConventions.routes.udfs}) + path: constants.namingConventions.routes.udfs}), + + Ember.Object.create({text: 'menus.uploadTable', + path: constants.namingConventions.routes.uploadTable}) ]) }); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js new file mode 100644 index 0000000..ab7b934 --- /dev/null +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; +import Uploader from 'hive/adapters/upload-table' +import constants from 'hive/utils/constants'; + + +export default Ember.Controller.extend({ + notifyService: Ember.inject.service(constants.namingConventions.notify), + needs : ['databases'], + showErrors : false, + uploader: Uploader.create(), + baseUrl: "/resources/upload", + isFirstRowHeader: null, // is first row header + header: null, // header received from server + files: null, // files that need to be uploaded only file[0] is relevant + firstRow: [], // the actual first row of the table. + rows: null, // preview rows received from server + databaseName:null, + selectedDatabase : null, + filePath : null, + tableName: null, + dataTypes : [ + "TINYINT", // + "SMALLINT", // + "INT", // + "BIGINT", // + "BOOLEAN", // + "FLOAT", // + "DOUBLE", // + "STRING", // + "BINARY", // -- (Note: Available in Hive 0.8.0 and later) + "TIMESTAMP", // -- (Note: Available in Hive 0.8.0 and later) + "DECIMAL", // -- (Note: Available in Hive 0.11.0 and later) + "DATE", // -- (Note: Available in Hive 0.12.0 and later) + "VARCHAR", // -- (Note: Available in Hive 0.12.0 and later) + "CHAR" // -- (Note: Available in Hive 0.13.0 and later) + ], + isFirstRowHeaderDidChange: function () { + console.log("inside onFirstRowHeader : isFirstRowHeader : " + this.get('isFirstRowHeader')); + if (this.get('isFirstRowHeader') != null && typeof this.get('isFirstRowHeader') !== 'undefined') { + if (this.get('isFirstRowHeader') == false) { + if (this.get('rows')) { + this.get('rows').unshiftObject({row: this.get('firstRow')}); + } + } else { + // take first row of + this.get('header').forEach(function (item, index) { + console.log("item : ", item); + console.log("this.get('firstRow').objectAt(index) : ", this.get('firstRow').objectAt(index)); + Ember.set(item, 'name', this.get('firstRow')[index]); + }, this); + + this.get('rows').removeAt(0); + } + + this.printValues(); + } + }.observes('isFirstRowHeader'), + + uploadForPreview: function (files) { + console.log("uploaderForPreview called."); + return this.get('uploader').uploadFiles('preview', files); + }, + + clearFields: function () { + this.set("header"); + this.set("rows"); + this.set("error"); + this.set('isFirstRowHeader'); + this.set('files'); + this.set("firstRow"); + this.set("selectedDatabase"); + this.set("databaseName"); + this.set("filePath"); + this.set('tableName'); + + this.printValues(); + }, + + printValues: function () { + console.log("printing all values : "); + console.log("header : ", this.get('header')); + console.log("rows : ", this.get('rows')); + console.log("error : ", this.get('error')); + console.log("isFirstRowHeader : ", this.get('isFirstRowHeader')); + console.log("files : ", this.get('files')); + console.log("firstRow : ", this.get('firstRow')); + }, + previewTable: function (data) { + console.log('inside previewTable'); + this.set("header", data.header); + this.set("rows", data.rows); + this.set("firstRow", data.rows[0].row); + console.log("firstRow : ", this.get('firstRow')); + this.set('isFirstRowHeader', data.isFirstRowHeader); + }, + + fetchCreateTableStatus: function (jobId, resolve, reject) { + var self = this; + this.get('uploader').getCreateTableResult(jobId).then(function (data) { + console.log("fetchCreateTableStatus : data : ", data); + var status = data.status; + if (status == "Succeeded") { + console.log("resolving fetchCreateTableStatus with : " + data); + resolve(status); + } else if (status == "Canceled" || status == "Closed" || status == "Error") { + console.log("rejecting fetchCreateTableStatus with : " + status); + reject(new Error(status)); + } else { + console.log("retrying fetchCreateTableStatus : "); + self.fetchCreateTableStatus(jobId, resolve, reject); + } + }, function (error) { + console.log("rejecting fetchCreateTableStatus with : " + error); + reject(error); + }) + }, + + waitForResult: function (jobId) { + var self = this; + return new Ember.RSVP.Promise(function (resolve, reject) { + self.fetchCreateTableStatus(jobId,resolve,reject); + }); + }, + + createTable: function () { + var headers = JSON.stringify(this.get('header')); + + var selectedDatabase = this.get('selectedDatabase'); + if( null == selectedDatabase || typeof selectedDatabase === 'undefined'){ + throw new Error(constants.hive.errors.emptyDatabase); + } + + this.set('databaseName',this.get('selectedDatabase').get('name')); + var databaseName = this.get('databaseName'); + var tableName = this.get('tableName'); + var isFirstRowHeader = this.get('isFirstRowHeader'); + console.log("databaseName : " , databaseName, ", tableName : ", tableName, ", isFirstRowHeader : " , isFirstRowHeader , ", headers : ", headers); + + if( null == databaseName || typeof databaseName === 'undefined'){ + throw new Error(constants.hive.errors.emptyDatabase); + } + if( null == tableName || typeof tableName === 'undefined'){ + throw new Error(constants.hive.errors.emptyTableName); + } + if( null == isFirstRowHeader || typeof isFirstRowHeader === 'undefined'){ + throw new Error(constants.hive.errors.emptyIsFirstRow); + } + + this.validateColumns(); + + return this.get('uploader').createTable({ + "isFirstRowHeader": isFirstRowHeader, + "header": headers, + "tableName": tableName, + "databaseName": databaseName + }); + }, + + validateColumns: function(){ + // TODO :check validation of columnames. + // throw exception if invalid. + }, + setError: function(error){ + this.set('error',JSON.stringify(error)); + console.log("upload table error : ",error); + this.get('notifyService').error(error); + }, + + previewError: function (error) { + this.setError(error); + }, + + uploadTable: function () { + this.printValues(); + return this.get('uploader').uploadFiles('upload', this.get('files'), { + "isFirstRowHeader": this.get("isFirstRowHeader"), + "filePath": this.get('filePath') + }); + }, + + onUploadSuccessfull: function (data) { + console.log("onUploadSuccessfull : ", data); + this.get('notifyService').success( "Uploaded Successfully", "Table " + this.get('tableName') + " created in database " + this.get("databaseName")); + this.clearFields(); + }, + + onUploadError: function (error) { + console.log("onUploadError : ", error); + this.setError(error); + }, + + actions: { + toggleErrors : function(){ + this.toggleProperty('showErrors'); + }, + filesUploaded: function (files) { + console.log("upload-table.js : uploaded new files : ", files); + + this.clearFields(); + + this.set('files', files); + var name = files[0].name; + var i = name.indexOf("."); + var tableName = name.substr(0, i); + this.set('tableName', tableName); + var self = this; + return this.uploadForPreview(files).then(function (data) { + self.previewTable(data); + }, function (error) { + self.previewError(error); + }); + }, + + createTableAndUploadFile: function () { + var self = this; + + try { + this.createTable() + .then(function (jobData) { + console.log("jobData : ", jobData); + self.set('filePath', jobData.filePath); + self.waitForResult(jobData.jobId) + .then(function (successStatus) { + console.log("successStatus : ", successStatus); + self.uploadTable().then(function (operationData) { + console.log("operation successfull operationData : ", operationData); + self.onUploadSuccessfull(operationData); + }, function (error) { + self.onUploadError(error); + }); + }, function (error) { + self.onUploadError(error); + }) + }, function (error) { + self.onUploadError(error); + }) + }catch(e){ + self.onUploadError(e); + } + } + + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js index 5ae9b7e..bd0e6e6 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js @@ -158,6 +158,7 @@ TRANSLATIONS = { savedQueries: 'Saved Queries', history: 'History', udfs: 'UDFs', + uploadTable: 'Upload Table', logs: 'Logs', results: 'Results', explain: 'Explain' @@ -199,6 +200,7 @@ TRANSLATIONS = { expand: 'Expand message', collapse: 'Collapse message', previousPage: 'previous', + uploadTable: 'Upload Table', nextPage: 'next', loadMore: 'Load more...', saveHdfs: 'Save to HDFS', @@ -237,7 +239,10 @@ TRANSLATIONS = { hive: { errors: { - 'no.query': "No query to process." + 'no.query': "No query to process.", + 'emptyDatabase' : "Please select Database.", + 'emptyTableName' : "Please enter tableName.", + 'emptyIsFirstRow' : "Please select is First Row Header?" } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js index 5a51b11..56e87d9 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js @@ -31,6 +31,7 @@ Router.map(function () { this.route(constants.namingConventions.routes.queries); this.route(constants.namingConventions.routes.history); this.route(constants.namingConventions.routes.udfs); + this.route(constants.namingConventions.routes.uploadTable); this.resource(constants.namingConventions.routes.index, { path: '/' }, function () { this.route(constants.namingConventions.routes.savedQuery, { path: savedQueryPath}); http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss index 7e23d46..f37a057 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss @@ -550,11 +550,10 @@ tree-view ul li { margin: 0 0 10px; } -#query-results { +#query-results, #upload-table { .table { display: inline-block; overflow: auto; - max-width: 830px; } .query-results-tools { http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs new file mode 100644 index 0000000..0ef4a7d --- /dev/null +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs @@ -0,0 +1,92 @@ +{{! +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +}} + +<div class="pull-right"> + <i class="query-menu-tab fa queries-icon query-context-tab fa-envelope" {{ action 'toggleErrors'}}></i> +</div> + +<div style="width : 90%"> +<div class="main-content"> + {{#if showErrors}} + {{render 'messages'}} + {{/if}} +</div> +</div> + +{{#unless showErrors}} +<div> + {{file-upload filesUploaded="filesUploaded"}} +</div> +<div id="upload-table"> + <!--<div class='fa query-menu-tab fa queries-icon query-context-tab fa-envelope'></div>--> + + {{#if rows}} + <div class="query-results-tools"> + <div class="pull-right"> + <button type="button" {{action + "createTableAndUploadFile"}} + {{bind-attr class=":btn :btn-sm :btn-default"}}>{{t "buttons.uploadTable"}}</button> + </div> + </div> + + <div> + <div class="col-md-3">Database : + {{typeahead-widget + content=controllers.databases.databases + optionValuePath="id" + optionLabelPath="name" + selection=selectedDatabase + placeholder="Select a Database" + }} + </div> + <div class="col-md-3">Table Name : {{input type="text" class="form-control" placeHolder="Table Name" value=tableName }} + </div> + <div class="col-md-3">Is First Row Header? :{{input type="checkbox" class="form-control" checked=isFirstRowHeader }} + </div> + </div> + + <table class="table table-expandable"> + <thead> + <tr> + {{#each column in header}} + <th> {{input type="text" class="form-control" value=column.name}}</th> + {{/each}} + </tr> + <tr> + {{#each column in header}} + <th> {{typeahead-widget + content=dataTypes + selection=column.type + }} + </th> + {{/each}} + </tr> + </thead> + <tbody> + {{#each row in rows}} + <tr> + {{#each item in row.row}} + <td>{{item}}</td> + {{/each}} + </tr> + {{/each}} + </tbody> + </table> + {{/if}} +</div> +{{/unless}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js index a349d51..199677d 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js +++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js @@ -39,6 +39,7 @@ export default Ember.Object.create({ logs: 'logs', results: 'results', explain: 'explain', + uploadTable :'upload-table', visualization: 'visualization' }, http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/bower.json ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/bower.json b/contrib/views/hive/src/main/resources/ui/hive-web/bower.json index d43881f..72cdf2a 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/bower.json +++ b/contrib/views/hive/src/main/resources/ui/hive-web/bower.json @@ -18,6 +18,7 @@ "jquery-ui": "~1.11.2", "selectize": "~0.12.0", "pretender": "0.1.0", + "ember-uploader": "0.3.9", "polestar": "https://github.com/pallavkul/polestar.git", "voyager": "https://github.com/pallavkul/voyager.git" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/package.json ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/package.json b/contrib/views/hive/src/main/resources/ui/hive-web/package.json index 6ecdcb6..595b1f2 100644 --- a/contrib/views/hive/src/main/resources/ui/hive-web/package.json +++ b/contrib/views/hive/src/main/resources/ui/hive-web/package.json @@ -38,6 +38,7 @@ "ember-cli-qunit": "0.3.14", "ember-cli-selectize": "0.0.19", "ember-cli-uglify": "1.0.1", + "ember-cli-uploader": "^0.3.9", "ember-data": "1.0.0-beta.16.1", "ember-dynamic-component": "0.0.1", "ember-export-application-global": "^1.0.0", http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/view.xml ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/resources/view.xml b/contrib/views/hive/src/main/resources/view.xml index fdc32d7..23f901b 100644 --- a/contrib/views/hive/src/main/resources/view.xml +++ b/contrib/views/hive/src/main/resources/view.xml @@ -255,6 +255,12 @@ </resource> <resource> + <name>upload</name> + <plural-name>uploads</plural-name> + <service-class>org.apache.ambari.view.hive.resources.uploads.UploadService</service-class> + </resource> + + <resource> <name>file</name> <service-class>org.apache.ambari.view.hive.resources.files.FileService</service-class> </resource> http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java new file mode 100644 index 0000000..3d77d29 --- /dev/null +++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive.resources.upload; + +import org.apache.ambari.view.hive.client.ColumnDescription; +import org.apache.ambari.view.hive.client.ColumnDescriptionShort; +import org.apache.ambari.view.hive.client.Row; +import org.apache.ambari.view.hive.resources.uploads.ColumnDescriptionImpl; +import org.apache.ambari.view.hive.resources.uploads.DataParser; +import org.apache.ambari.view.hive.resources.uploads.ParseOptions; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; + +public class DataParserTest { + + @Test + public void testDataParser() throws IOException { + String str = "1,a\n" + + "2,b\n" + + "3,c\n"; + StringReader sr = new StringReader(str); + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER_FIRST_RECORD); + + DataParser dp = new DataParser(sr, parseOptions); + dp.parsePreview(); + Assert.assertNotNull(dp.getPreviewRows()); + Assert.assertNotNull(dp.getHeader()); + Assert.assertEquals(3, dp.getPreviewRows().size()); + Assert.assertEquals(2, dp.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.INT.toString(), 0), + new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)}; + + Assert.assertArrayEquals("Header Not Correct.", cd, dp.getHeader().toArray()); + + // TODO : include testing of each row element. Below comparison does not work properly. + // Object[] rows = {new Row(new Object[]{'1','a'}),new Row(new Object[]{'2','b'}),new Row(new Object[]{'3','c'})}; + // Assert.assertArrayEquals("Rows Not Correct.", rows, dp.getPreviewRows().toArray()); + + sr.close(); + } +}