This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 24af593954 Issue #8614 : Preview a Parquet file in the Hop GUI (#8619)
24af593954 is described below
commit 24af593954ed22617f8646e5a15c066f2b9bccc5
Author: Matt Casters <[email protected]>
AuthorDate: Sat Sep 26 15:24:14 2026 +0200
Issue #8614 : Preview a Parquet file in the Hop GUI (#8619)
* Issue #8614 : Preview a Parquet file in the Hop GUI
* Issue #8614 : Number the first column of the Parquet preview tables
* Issue #8614 : Confirm opening a large remote Parquet file
---
.../hop/parquet/explorer/ParquetColumnView.java | 32 ++
.../parquet/explorer/ParquetFileInspection.java | 81 +++++
.../hop/parquet/explorer/ParquetFileInspector.java | 313 +++++++++++++++++
.../parquet/transforms/input/ParquetInputMeta.java | 3 +-
.../file/ParquetExplorerFileTypeHandler.java | 391 +++++++++++++++++++++
.../perspective/explorer/file/ParquetFileType.java | 132 +++----
.../file/messages/messages_en_US.properties | 45 +++
.../parquet/explorer/ParquetFileInspectorTest.java | 293 +++++++++++++++
.../explorer/file/ParquetFileTypeTest.java | 84 +++++
9 files changed, 1298 insertions(+), 76 deletions(-)
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetColumnView.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetColumnView.java
new file mode 100644
index 0000000000..c523cdaad4
--- /dev/null
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetColumnView.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.parquet.explorer;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/** One leaf column of a Parquet schema, as shown in the explorer structure
table. */
+@Getter
+@RequiredArgsConstructor
+public class ParquetColumnView {
+ private final String name;
+ private final String parquetType;
+ private final String hopType;
+ private final int length;
+ private final int precision;
+}
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspection.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspection.java
new file mode 100644
index 0000000000..15313d9f98
--- /dev/null
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspection.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.parquet.explorer;
+
+import java.util.List;
+import lombok.Getter;
+import org.apache.hop.core.row.IRowMeta;
+
+/**
+ * What the explorer shows for one Parquet file. Sizes that the file does not
record are null.
+ * {@link #previewError} is set when the footer could be read but the rows
could not.
+ */
+@Getter
+public class ParquetFileInspection {
+ private final String fileName;
+ private final String folder;
+ private final long sizeBytes;
+ private final String compression;
+ private final String version;
+ private final Long rowGroupSize;
+ private final Long dataPageSize;
+ private final Long dictionaryPageSize;
+ private final long rowCount;
+ private final int rowGroupCount;
+ private final String createdBy;
+ private final String schemaJson;
+ private final List<ParquetColumnView> columns;
+ private final IRowMeta rowMeta;
+ private final List<Object[]> previewRows;
+ private final String previewError;
+
+ public ParquetFileInspection(
+ String fileName,
+ String folder,
+ long sizeBytes,
+ String compression,
+ String version,
+ Long rowGroupSize,
+ Long dataPageSize,
+ Long dictionaryPageSize,
+ long rowCount,
+ int rowGroupCount,
+ String createdBy,
+ String schemaJson,
+ List<ParquetColumnView> columns,
+ IRowMeta rowMeta,
+ List<Object[]> previewRows,
+ String previewError) {
+ this.fileName = fileName;
+ this.folder = folder;
+ this.sizeBytes = sizeBytes;
+ this.compression = compression;
+ this.version = version;
+ this.rowGroupSize = rowGroupSize;
+ this.dataPageSize = dataPageSize;
+ this.dictionaryPageSize = dictionaryPageSize;
+ this.rowCount = rowCount;
+ this.rowGroupCount = rowGroupCount;
+ this.createdBy = createdBy;
+ this.schemaJson = schemaJson;
+ this.columns = List.copyOf(columns);
+ this.rowMeta = rowMeta;
+ this.previewRows = List.copyOf(previewRows);
+ this.previewError = previewError;
+ }
+}
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspector.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspector.java
new file mode 100644
index 0000000000..b0ae82a870
--- /dev/null
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/explorer/ParquetFileInspector.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.parquet.explorer;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.TreeSet;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.RowMetaAndData;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.parquet.transforms.input.ParquetField;
+import org.apache.hop.parquet.transforms.input.ParquetInputMeta;
+import org.apache.hop.parquet.transforms.input.ParquetReadSupport;
+import org.apache.hop.parquet.transforms.input.ParquetReaderBuilder;
+import org.apache.hop.parquet.transforms.input.ParquetStream;
+import org.apache.hop.parquet.transforms.output.ParquetVersion;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.format.PageHeader;
+import org.apache.parquet.format.PageType;
+import org.apache.parquet.format.Util;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.io.SeekableInputStream;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+/** Reads the footer, the first page header and a bounded row preview of one
Parquet file. */
+public final class ParquetFileInspector {
+
+ /** How many rows the explorer preview reads. */
+ public static final int PREVIEW_ROW_LIMIT = 1000;
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private ParquetFileInspector() {}
+
+ /**
+ * Reads {@code filename} through Hop VFS. The file itself is not loaded:
the footer, one page
+ * header and at most {@link #PREVIEW_ROW_LIMIT} rows.
+ */
+ public static ParquetFileInspection inspect(String filename, IVariables
variables)
+ throws HopException {
+ try {
+ FileObject fileObject = HopVfs.getFileObject(filename, variables);
+ String fileName = fileObject.getName().getBaseName();
+ FileObject parent = fileObject.getParent();
+ String folder = parent == null ? "" : HopVfs.getFilename(parent);
+ long sizeBytes = fileObject.getContent().getSize();
+
+ try (ParquetStream parquetStream = new ParquetStream(fileObject,
filename)) {
+ ParquetMetadata footer;
+ try (ParquetFileReader reader = ParquetFileReader.open(parquetStream))
{
+ footer = reader.getFooter();
+ }
+ FileMetaData fileMetaData = footer.getFileMetaData();
+ MessageType schema = fileMetaData.getSchema();
+ PageFacts pages = readPageFacts(parquetStream, footer);
+ Preview preview = readPreview(parquetStream, schema);
+ return new ParquetFileInspection(
+ fileName,
+ folder,
+ sizeBytes,
+ compression(footer),
+ pages.version,
+ rowGroupSize(footer),
+ pages.dataPageSize,
+ pages.dictionaryPageSize,
+ rowCount(footer),
+ footer.getBlocks().size(),
+ fileMetaData.getCreatedBy() == null ? "" :
fileMetaData.getCreatedBy(),
+ schemaJson(schema),
+ leafColumns(schema),
+ preview.rowMeta,
+ preview.rows,
+ preview.error);
+ }
+ } catch (HopException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HopException("Unable to read Parquet file '" + filename + "'",
e);
+ }
+ }
+
+ /** Pretty-printed JSON of the schema tree. Groups stay nested; this is not
the file footer. */
+ static String schemaJson(MessageType schema) throws IOException {
+ ObjectNode root = MAPPER.createObjectNode();
+ root.put("name", schema.getName());
+ ArrayNode fields = root.putArray("fields");
+ for (Type field : schema.getFields()) {
+ fields.add(typeNode(field));
+ }
+ return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root);
+ }
+
+ /** Leaf columns in file order. A nested field is named with a dotted path.
*/
+ static List<ParquetColumnView> leafColumns(MessageType schema) throws
HopException {
+ List<ParquetColumnView> columns = new ArrayList<>();
+ for (Type field : schema.getFields()) {
+ collectLeaves(field, "", columns);
+ }
+ return columns;
+ }
+
+ private static void collectLeaves(Type type, String prefix,
List<ParquetColumnView> columns)
+ throws HopException {
+ String name = prefix.isEmpty() ? type.getName() : prefix + "." +
type.getName();
+ if (type.isPrimitive()) {
+ PrimitiveType primitive = type.asPrimitiveType();
+ IValueMeta valueMeta = ParquetInputMeta.hopValueMeta(name, primitive);
+ columns.add(
+ new ParquetColumnView(
+ name,
+ parquetType(primitive),
+ valueMeta.getTypeDesc(),
+ valueMeta.getLength(),
+ valueMeta.getPrecision()));
+ return;
+ }
+ for (Type child : type.asGroupType().getFields()) {
+ collectLeaves(child, name, columns);
+ }
+ }
+
+ private static ObjectNode typeNode(Type type) {
+ ObjectNode node = MAPPER.createObjectNode();
+ node.put("name", type.getName());
+ node.put("repetition", type.getRepetition().name());
+ LogicalTypeAnnotation logicalType = type.getLogicalTypeAnnotation();
+ if (type.isPrimitive()) {
+ PrimitiveType primitive = type.asPrimitiveType();
+ node.put("type", primitive.getPrimitiveTypeName().name());
+ if (primitive.getTypeLength() > 0) {
+ node.put("length", primitive.getTypeLength());
+ }
+ } else {
+ node.put("type", "group");
+ ArrayNode children = node.putArray("fields");
+ for (Type child : type.asGroupType().getFields()) {
+ children.add(typeNode(child));
+ }
+ }
+ if (logicalType != null) {
+ node.put("logicalType", logicalType.toString());
+ }
+ return node;
+ }
+
+ private static String parquetType(PrimitiveType primitive) {
+ String physical = primitive.getPrimitiveTypeName().name();
+ LogicalTypeAnnotation logicalType = primitive.getLogicalTypeAnnotation();
+ if (logicalType == null) {
+ return physical;
+ }
+ return physical + " (" + logicalType + ")";
+ }
+
+ private static String compression(ParquetMetadata footer) {
+ TreeSet<String> codecs = new TreeSet<>();
+ for (BlockMetaData block : footer.getBlocks()) {
+ for (ColumnChunkMetaData column : block.getColumns()) {
+ if (column.getCodec() != null) {
+ codecs.add(column.getCodec().name());
+ }
+ }
+ }
+ return String.join(", ", codecs);
+ }
+
+ private static long rowCount(ParquetMetadata footer) {
+ long rows = 0;
+ for (BlockMetaData block : footer.getBlocks()) {
+ rows += block.getRowCount();
+ }
+ return rows;
+ }
+
+ /** Largest row group, in uncompressed bytes. The writer limit is not stored
in the file. */
+ private static Long rowGroupSize(ParquetMetadata footer) {
+ Long largest = null;
+ for (BlockMetaData block : footer.getBlocks()) {
+ long size = block.getTotalByteSize();
+ if (largest == null || size > largest) {
+ largest = size;
+ }
+ }
+ return largest;
+ }
+
+ /**
+ * Version and page sizes from the first column chunk only. The dictionary
page and the first data
+ * page headers are a few dozen bytes; the column values are not read.
+ */
+ private static PageFacts readPageFacts(ParquetStream parquetStream,
ParquetMetadata footer)
+ throws IOException {
+ if (footer.getBlocks().isEmpty() ||
footer.getBlocks().get(0).getColumns().isEmpty()) {
+ return PageFacts.empty();
+ }
+ ColumnChunkMetaData column = footer.getBlocks().get(0).getColumns().get(0);
+ Long dictionaryPageSize = null;
+ Long dataPageSize = null;
+ String version = "";
+ try (SeekableInputStream in = parquetStream.newStream()) {
+ if (column.hasDictionaryPage() && column.getDictionaryPageOffset() > 0) {
+ PageHeader header = readHeader(in, column.getDictionaryPageOffset());
+ if (header.getType() == PageType.DICTIONARY_PAGE) {
+ dictionaryPageSize = (long) header.getUncompressed_page_size();
+ }
+ }
+ if (column.getFirstDataPageOffset() > 0) {
+ PageHeader header = readHeader(in, column.getFirstDataPageOffset());
+ dataPageSize = (long) header.getUncompressed_page_size();
+ if (header.getType() == PageType.DATA_PAGE_V2) {
+ version = ParquetVersion.Version2.getDescription();
+ } else if (header.getType() == PageType.DATA_PAGE) {
+ version = ParquetVersion.Version1.getDescription();
+ }
+ }
+ }
+ return new PageFacts(version, dataPageSize, dictionaryPageSize);
+ }
+
+ private static PageHeader readHeader(SeekableInputStream in, long offset)
throws IOException {
+ in.seek(offset);
+ return Util.readPageHeader(in);
+ }
+
+ /**
+ * Top-level primitive columns, which is what {@link ParquetReadSupport} can
convert. Nested
+ * values stay in the schema JSON and the leaf table.
+ */
+ private static Preview readPreview(ParquetStream parquetStream, MessageType
schema) {
+ IRowMeta rowMeta = new RowMeta();
+ List<ParquetField> fields = new ArrayList<>();
+ try {
+ for (ColumnDescriptor column : schema.getColumns()) {
+ if (column.getPath().length != 1) {
+ continue;
+ }
+ String name = column.getPath()[0];
+ IValueMeta valueMeta = ParquetInputMeta.hopValueMeta(name,
column.getPrimitiveType());
+ rowMeta.addValueMeta(valueMeta);
+ fields.add(
+ new ParquetField(
+ name,
+ name,
+ valueMeta.getTypeDesc(),
+ valueMeta.getFormatMask(),
+ Integer.toString(valueMeta.getLength()),
+ Integer.toString(valueMeta.getPrecision())));
+ }
+ if (fields.isEmpty()) {
+ return new Preview(rowMeta, List.of(), null);
+ }
+ List<Object[]> rows = new ArrayList<>();
+ ParquetReadSupport readSupport = new ParquetReadSupport(fields);
+ try (ParquetReader<RowMetaAndData> reader =
+ new ParquetReaderBuilder<>(readSupport, parquetStream).build()) {
+ RowMetaAndData row = reader.read();
+ while (row != null && rows.size() < PREVIEW_ROW_LIMIT) {
+ Object[] data = row.getData();
+ rows.add(data == null ? new Object[rowMeta.size()] :
Arrays.copyOf(data, data.length));
+ if (rows.size() >= PREVIEW_ROW_LIMIT) {
+ break;
+ }
+ row = reader.read();
+ }
+ }
+ return new Preview(rowMeta, rows, null);
+ } catch (Exception e) {
+ String message = e.getMessage() == null ? e.toString() : e.getMessage();
+ return new Preview(rowMeta, List.of(), message);
+ }
+ }
+
+ private record PageFacts(String version, Long dataPageSize, Long
dictionaryPageSize) {
+ static PageFacts empty() {
+ return new PageFacts("", null, null);
+ }
+ }
+
+ private record Preview(IRowMeta rowMeta, List<Object[]> rows, String error)
{}
+}
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
index 52747de5e2..7a7e68db12 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
@@ -169,7 +169,8 @@ public class ParquetInputMeta extends
BaseTransformMeta<ParquetInput, ParquetInp
* @see <a
href="https://parquet.apache.org/docs/file-format/types/logicaltypes/";>Parquet
logical
* types</a>
*/
- static IValueMeta hopValueMeta(String name, PrimitiveType primitiveType)
throws HopException {
+ public static IValueMeta hopValueMeta(String name, PrimitiveType
primitiveType)
+ throws HopException {
LogicalTypeAnnotation logicalType =
primitiveType.getLogicalTypeAnnotation();
int length = -1;
int precision = -1;
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetExplorerFileTypeHandler.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetExplorerFileTypeHandler.java
new file mode 100644
index 0000000000..11752dd371
--- /dev/null
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetExplorerFileTypeHandler.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.perspective.explorer.file;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.parquet.explorer.ParquetColumnView;
+import org.apache.hop.parquet.explorer.ParquetFileInspection;
+import org.apache.hop.parquet.explorer.ParquetFileInspector;
+import org.apache.hop.ui.core.FormDataBuilder;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.dialog.ErrorDialog;
+import org.apache.hop.ui.core.dialog.RowPreviewSupport;
+import org.apache.hop.ui.core.gui.GuiResource;
+import org.apache.hop.ui.core.widget.ColumnInfo;
+import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.core.widget.editor.IContentEditorWidget;
+import org.apache.hop.ui.hopgui.ContentEditorFacade;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.perspective.explorer.ExplorerFile;
+import org.apache.hop.ui.hopgui.perspective.explorer.ExplorerPerspective;
+import
org.apache.hop.ui.hopgui.perspective.explorer.file.types.base.BaseExplorerFileTypeHandler;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
+import org.eclipse.swt.custom.SashForm;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.TableItem;
+
+/** Read-only explorer tab for a Parquet file: details, schema and a short
data preview. */
+public class ParquetExplorerFileTypeHandler extends
BaseExplorerFileTypeHandler {
+
+ private static final Class<?> PKG = ParquetExplorerFileTypeHandler.class;
+
+ private Label note;
+ private TableView detailsView;
+ private IContentEditorWidget schemaEditor;
+ private TableView columnsView;
+ private Composite previewComposite;
+ private Label previewLabel;
+ private TableView previewView;
+
+ public ParquetExplorerFileTypeHandler(
+ HopGui hopGui, ExplorerPerspective perspective, ExplorerFile
explorerFile) {
+ super(hopGui, perspective, explorerFile);
+ }
+
+ @Override
+ public void renderFile(Composite composite) {
+ PropsUi.setLook(composite);
+
+ CTabFolder folder = new CTabFolder(composite, SWT.BORDER);
+ PropsUi.setLook(folder, PropsUi.WIDGET_STYLE_TAB);
+ folder.setLayoutData(new FormDataBuilder().fullSize().result());
+
+ CTabItem detailsTab = new CTabItem(folder, SWT.NONE);
+ detailsTab.setText(BaseMessages.getString(PKG,
"ParquetExplorer.Tab.Details"));
+ detailsTab.setControl(buildDetails(folder));
+
+ CTabItem structureTab = new CTabItem(folder, SWT.NONE);
+ structureTab.setText(BaseMessages.getString(PKG,
"ParquetExplorer.Tab.Structure"));
+ structureTab.setControl(buildStructure(folder));
+
+ CTabItem previewTab = new CTabItem(folder, SWT.NONE);
+ previewTab.setText(BaseMessages.getString(PKG,
"ParquetExplorer.Tab.Preview"));
+ previewTab.setControl(buildPreview(folder));
+
+ folder.setSelection(0);
+
+ String filename = explorerFile.getFilename();
+ IVariables variables = getVariables();
+ Thread reader = new Thread(() -> load(composite, filename, variables),
"Parquet file preview");
+ reader.setDaemon(true);
+ reader.start();
+ }
+
+ private Composite buildDetails(Composite parent) {
+ Composite details = new Composite(parent, SWT.NONE);
+ details.setLayout(new FormLayout());
+ PropsUi.setLook(details);
+
+ note = new Label(details, SWT.WRAP);
+ PropsUi.setLook(note);
+ note.setText(BaseMessages.getString(PKG, "ParquetExplorer.Note"));
+ note.setToolTipText(note.getText());
+ // A wrapping label only keeps its wrapped height when the layout gives it
one.
+ int noteHeight = note.computeSize(SWT.DEFAULT, SWT.DEFAULT).y * 3;
+ note.setLayoutData(new
FormDataBuilder().top().fullWidth().height(noteHeight).result());
+
+ detailsView =
+ readOnlyTable(
+ details,
+ new ColumnInfo[] {
+ column("ParquetExplorer.Details.Column.Property"),
+ column("ParquetExplorer.Details.Column.Value")
+ });
+ detailsView.setLayoutData(
+ new FormDataBuilder().top(note,
PropsUi.getMargin()).bottom().fullWidth().result());
+ return details;
+ }
+
+ private Composite buildStructure(Composite parent) {
+ Composite structure = new Composite(parent, SWT.NONE);
+ structure.setLayout(new FormLayout());
+ PropsUi.setLook(structure);
+
+ SashForm sash = new SashForm(structure, SWT.VERTICAL);
+ PropsUi.setLook(sash);
+ sash.setLayoutData(new FormDataBuilder().fullSize().result());
+
+ Composite jsonParent = new Composite(sash, SWT.NONE);
+ jsonParent.setLayout(new FormLayout());
+ PropsUi.setLook(jsonParent);
+ schemaEditor = ContentEditorFacade.createContentEditor(jsonParent, "json");
+ schemaEditor.getControl().setLayoutData(new
FormDataBuilder().fullSize().result());
+ schemaEditor.setReadOnly(true);
+ schemaEditor.setText(BaseMessages.getString(PKG,
"ParquetExplorer.Reading"));
+
+ columnsView =
+ readOnlyTable(
+ sash,
+ new ColumnInfo[] {
+ column("ParquetExplorer.Structure.Column.Name"),
+ column("ParquetExplorer.Structure.Column.ParquetType"),
+ column("ParquetExplorer.Structure.Column.HopType"),
+ column("ParquetExplorer.Structure.Column.Length"),
+ column("ParquetExplorer.Structure.Column.Precision")
+ });
+ sash.setWeights(1, 1);
+ return structure;
+ }
+
+ private Composite buildPreview(Composite parent) {
+ previewComposite = new Composite(parent, SWT.NONE);
+ previewComposite.setLayout(new FormLayout());
+ PropsUi.setLook(previewComposite);
+
+ previewLabel = new Label(previewComposite, SWT.WRAP);
+ PropsUi.setLook(previewLabel);
+ previewLabel.setText(BaseMessages.getString(PKG,
"ParquetExplorer.Reading"));
+ previewLabel.setLayoutData(new
FormDataBuilder().top().fullWidth().result());
+ return previewComposite;
+ }
+
+ private TableView readOnlyTable(Composite parent, ColumnInfo[] columns) {
+ TableView table =
+ new TableView(
+ getVariables(),
+ parent,
+ SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
+ columns,
+ 0,
+ true,
+ null,
+ PropsUi.getInstance(),
+ false,
+ null,
+ false,
+ false);
+ table.setReadonly(true);
+ PropsUi.setLook(table);
+ return table;
+ }
+
+ private static ColumnInfo column(String key) {
+ return new ColumnInfo(
+ BaseMessages.getString(PKG, key), ColumnInfo.COLUMN_TYPE_TEXT, false,
true);
+ }
+
+ private void load(Composite composite, String filename, IVariables
variables) {
+ try {
+ ParquetFileInspection inspection =
ParquetFileInspector.inspect(filename, variables);
+ async(
+ composite,
+ () -> {
+ if (!composite.isDisposed()) {
+ show(inspection);
+ }
+ });
+ } catch (Exception e) {
+ async(composite, () -> showError(composite, e));
+ }
+ }
+
+ private void async(Composite composite, Runnable action) {
+ if (hopGui == null || hopGui.getDisplay() == null ||
hopGui.getDisplay().isDisposed()) {
+ return;
+ }
+ hopGui
+ .getDisplay()
+ .asyncExec(
+ () -> {
+ if (!composite.isDisposed()) {
+ action.run();
+ }
+ });
+ }
+
+ private void show(ParquetFileInspection inspection) {
+ if (note.isDisposed()) {
+ return;
+ }
+ fillRows(
+ detailsView,
+ List.of(
+ detail("ParquetExplorer.Details.Name", inspection.getFileName()),
+ detail("ParquetExplorer.Details.Folder", inspection.getFolder()),
+ detail("ParquetExplorer.Details.Size",
grouped(inspection.getSizeBytes())),
+ detail("ParquetExplorer.Details.Compression",
inspection.getCompression()),
+ detail("ParquetExplorer.Details.Version", inspection.getVersion()),
+ detail("ParquetExplorer.Details.RowGroupSize",
grouped(inspection.getRowGroupSize())),
+ detail("ParquetExplorer.Details.DataPageSize",
grouped(inspection.getDataPageSize())),
+ detail(
+ "ParquetExplorer.Details.DictionaryPageSize",
+ grouped(inspection.getDictionaryPageSize())),
+ detail("ParquetExplorer.Details.Rows",
grouped(inspection.getRowCount())),
+ detail(
+ "ParquetExplorer.Details.RowGroups",
+ Integer.toString(inspection.getRowGroupCount())),
+ detail("ParquetExplorer.Details.CreatedBy",
inspection.getCreatedBy())));
+ detailsView.table.setToolTipText(note.getText());
+
+ schemaEditor.setText(inspection.getSchemaJson());
+ List<String[]> columnRows = new ArrayList<>();
+ for (ParquetColumnView column : inspection.getColumns()) {
+ columnRows.add(
+ new String[] {
+ column.getName(),
+ column.getParquetType(),
+ column.getHopType(),
+ number(column.getLength()),
+ number(column.getPrecision())
+ });
+ }
+ fillRows(columnsView, columnRows);
+ showPreview(inspection);
+ }
+
+ private void showPreview(ParquetFileInspection inspection) {
+ if (previewLabel.isDisposed()) {
+ return;
+ }
+ if (inspection.getPreviewError() != null) {
+ previewLabel.setText(
+ BaseMessages.getString(
+ PKG, "ParquetExplorer.Preview.Failed",
inspection.getPreviewError()));
+ return;
+ }
+ int shown = inspection.getPreviewRows().size();
+ if (shown >= ParquetFileInspector.PREVIEW_ROW_LIMIT) {
+ previewLabel.setText(
+ BaseMessages.getString(
+ PKG,
+ "ParquetExplorer.Preview.First",
+ Integer.toString(ParquetFileInspector.PREVIEW_ROW_LIMIT)));
+ } else {
+ previewLabel.setText(
+ BaseMessages.getString(PKG, "ParquetExplorer.Preview.Showing",
Integer.toString(shown)));
+ }
+
+ IRowMeta rowMeta = inspection.getRowMeta();
+ if (rowMeta == null || rowMeta.isEmpty()) {
+ return;
+ }
+ ColumnInfo[] columns = new ColumnInfo[rowMeta.size()];
+ for (int i = 0; i < rowMeta.size(); i++) {
+ IValueMeta valueMeta = rowMeta.getValueMeta(i);
+ columns[i] =
+ new ColumnInfo(valueMeta.getName(), ColumnInfo.COLUMN_TYPE_TEXT,
valueMeta.isNumeric());
+ RowPreviewSupport.applyColumnMeta(columns[i], valueMeta);
+ }
+ previewView =
+ new TableView(
+ getVariables(),
+ previewComposite,
+ SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
+ columns,
+ 0,
+ true,
+ null,
+ PropsUi.getInstance());
+ previewView.setShowingBlueNullValues(true);
+ previewView.setShortenDisplayedValues(true);
+ previewView.setSortable(true);
+ previewView.setReadonly(true);
+ previewView.setLayoutData(
+ new FormDataBuilder().top(previewLabel,
PropsUi.getMargin()).bottom().fullWidth().result());
+ RowPreviewSupport.installCellTooltips(previewView, rowMeta);
+
+ List<Object[]> rows = inspection.getPreviewRows();
+ for (int r = 0; r < rows.size(); r++) {
+ TableItem item =
+ r == 0 ? previewView.table.getItem(0) : new
TableItem(previewView.table, SWT.NONE);
+ item.setText(0, Integer.toString(r + 1));
+ Object[] row = rows.get(r);
+ if (row == null) {
+ continue;
+ }
+ for (int c = 0; c < rowMeta.size(); c++) {
+ String display;
+ try {
+ display = RowPreviewSupport.formatCell(rowMeta.getValueMeta(c),
row[c]);
+ } catch (HopValueException | ArrayIndexOutOfBoundsException e) {
+ display = null;
+ }
+ if (display == null) {
+ item.setText(c + 1, "<null>");
+ item.setForeground(c + 1, GuiResource.getInstance().getColorBlue());
+ } else {
+ previewView.setCellValue(item, c + 1, display);
+ }
+ }
+ }
+ if (rows.isEmpty()) {
+ previewView.table.removeAll();
+ }
+ if (!previewView.isDisposed()) {
+ previewView.optWidth(true, 200);
+ }
+ previewComposite.layout(true, true);
+ }
+
+ private void showError(Composite composite, Exception e) {
+ if (note != null && !note.isDisposed()) {
+ note.setText(e.getMessage() == null ? e.toString() : e.getMessage());
+ }
+ if (hopGui.getShell() == null || hopGui.getShell().isDisposed()) {
+ return;
+ }
+ new ErrorDialog(
+ hopGui.getShell(),
+ BaseMessages.getString(PKG, "ParquetExplorer.Error.Title"),
+ BaseMessages.getString(PKG, "ParquetExplorer.Error.Message"),
+ e);
+ }
+
+ private static String[] detail(String key, String value) {
+ return new String[] {BaseMessages.getString(PKG, key), value == null ? ""
: value};
+ }
+
+ private static String grouped(Long value) {
+ return value == null ? "" : String.format(Locale.getDefault(), "%,d",
value);
+ }
+
+ private static String number(int value) {
+ return value < 0 ? "" : Integer.toString(value);
+ }
+
+ private static void fillRows(TableView table, List<String[]> rows) {
+ if (table == null || table.isDisposed()) {
+ return;
+ }
+ table.clearAll(false);
+ if (rows.isEmpty()) {
+ table.table.removeAll();
+ return;
+ }
+ for (int i = 0; i < rows.size(); i++) {
+ TableItem item = i == 0 ? table.table.getItem(0) : new
TableItem(table.table, SWT.NONE);
+ String[] values = rows.get(i);
+ // Column 0 is the row number. optimizeTableView() writes it.
+ for (int c = 0; c < values.length; c++) {
+ item.setText(c + 1, values[c] == null ? "" : values[c]);
+ }
+ }
+ table.optimizeTableView();
+ }
+}
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileType.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileType.java
index 39b06328a8..507ed33124 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileType.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileType.java
@@ -13,110 +13,92 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
package org.apache.hop.ui.hopgui.perspective.explorer.file;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Properties;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.file.IHasFilename;
-import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.ui.hopgui.HopGui;
-import org.apache.hop.ui.hopgui.context.IGuiContextHandler;
-import org.apache.hop.ui.hopgui.file.HopFileTypeBase;
import org.apache.hop.ui.hopgui.file.HopFileTypePlugin;
import org.apache.hop.ui.hopgui.file.IHopFileType;
import org.apache.hop.ui.hopgui.file.IHopFileTypeHandler;
import org.apache.hop.ui.hopgui.file.empty.EmptyHopFileTypeHandler;
+import org.apache.hop.ui.hopgui.perspective.explorer.ExplorerFile;
+import org.apache.hop.ui.hopgui.perspective.explorer.ExplorerPerspective;
+import
org.apache.hop.ui.hopgui.perspective.explorer.file.capabilities.FileTypeCapabilities;
+import
org.apache.hop.ui.hopgui.perspective.explorer.file.types.base.BaseExplorerFileType;
@HopFileTypePlugin(
id = "ParquetFileType",
name = "Parquet File Type",
description = "Apache Parquet file handling in the explorer perspective",
image = "parquet.svg")
-public class ParquetFileType implements IHopFileType {
-
- private static final String[] EXTENSIONS = new String[] {"*.parquet"};
- private static final String[] FILTER_EXTENSIONS = new String[] {"*.parquet"};
- private static final String[] FILTER_NAMES = new String[] {"Parquet files"};
-
- @Override
- public String getName() {
- return "Parquet";
- }
-
- @Override
- public String getDefaultFileExtension() {
- return "parquet";
- }
-
- @Override
- public String[] getFilterExtensions() {
- return FILTER_EXTENSIONS;
+public class ParquetFileType extends
BaseExplorerFileType<ParquetExplorerFileTypeHandler> {
+
+ public ParquetFileType() {
+ super(
+ "Parquet file",
+ ".parquet",
+ new String[] {"*.parquet;*.parq"},
+ new String[] {"Parquet files"},
+ FileTypeCapabilities.getCapabilities(
+ IHopFileType.CAPABILITY_CLOSE,
IHopFileType.CAPABILITY_FILE_HISTORY));
}
+ /**
+ * Opens a local file without the large-file confirmation. The preview seeks
to the footer and
+ * reads at most 1000 rows. Any other VFS scheme keeps {@link
BaseExplorerFileType#openFile}'s
+ * confirmation: a seek there re-reads the file from the start.
+ */
@Override
- public String[] getFilterNames() {
- return FILTER_NAMES;
- }
-
- @Override
- public Properties getCapabilities() {
- return new Properties();
- }
-
- @Override
- public boolean hasCapability(String capability) {
- return false;
- }
-
- @Override
- public IHopFileTypeHandler openFile(HopGui hopGui, String filename,
IVariables variables)
- throws HopException {
- return new EmptyHopFileTypeHandler();
- }
-
- @Override
- public IHopFileTypeHandler newFile(HopGui hopGui, IVariables variables)
throws HopException {
- return new EmptyHopFileTypeHandler();
- }
-
- @Override
- public boolean isHandledBy(String filename, boolean checkContent) throws
HopException {
- String fileExtension = HopFileTypeBase.extractExtension(filename);
- if (Utils.isEmpty(fileExtension)) {
- return false;
- }
- for (String typeExtension : EXTENSIONS) {
- if (typeExtension.toLowerCase(Locale.ROOT).endsWith(fileExtension)) {
- return true;
+ public ParquetExplorerFileTypeHandler openFile(
+ HopGui hopGui, String filename, IVariables variables) throws
HopException {
+ try {
+ FileObject fileObject = HopVfs.getFileObject(filename, variables);
+ if (keepsLargeFileConfirmation(fileObject)) {
+ fileObject.close();
+ return super.openFile(hopGui, filename, variables);
}
+ String name = fileObject.getName().getBaseName();
+ filename = HopVfs.getFilename(fileObject);
+
+ ExplorerFile explorerFile = new ExplorerFile();
+ explorerFile.setName(Const.NVL(name, ""));
+ explorerFile.setFilename(filename);
+ explorerFile.setFileType(this);
+
+ ExplorerPerspective perspective = ExplorerPerspective.getInstance();
+ ParquetExplorerFileTypeHandler fileTypeHandler =
+ createFileTypeHandler(hopGui, perspective, explorerFile);
+ perspective.addFile(fileTypeHandler);
+ return fileTypeHandler;
+ } catch (Exception e) {
+ throw new HopException(
+ "Error opening file '" + filename + "' in a new tab in the Explorer
perspective", e);
}
- return false;
- }
-
- @Override
- public boolean supportsFile(IHasFilename metaObject) {
- return false;
}
- @Override
- public List<IGuiContextHandler> getContextHandlers() {
- return Collections.emptyList();
+ /**
+ * Remote and non-file locations re-stream the object on every seek, so a
large open still asks. A
+ * plain {@code file} location can seek and skips the confirmation.
+ */
+ static boolean keepsLargeFileConfirmation(FileObject fileObject) {
+ return fileObject.getName() == null ||
!"file".equals(fileObject.getName().getScheme());
}
@Override
- public String getFileTypeImage() {
- return getClass().getAnnotation(HopFileTypePlugin.class).image();
+ public ParquetExplorerFileTypeHandler createFileTypeHandler(
+ HopGui hopGui, ExplorerPerspective perspective, ExplorerFile file) {
+ return new ParquetExplorerFileTypeHandler(hopGui, perspective, file);
}
@Override
- public boolean supportsOpening() {
- return false;
+ public IHopFileTypeHandler newFile(HopGui hopGui, IVariables
parentVariableSpace)
+ throws HopException {
+ return new EmptyHopFileTypeHandler();
}
}
diff --git
a/plugins/tech/parquet/src/main/resources/org/apache/hop/ui/hopgui/perspective/explorer/file/messages/messages_en_US.properties
b/plugins/tech/parquet/src/main/resources/org/apache/hop/ui/hopgui/perspective/explorer/file/messages/messages_en_US.properties
new file mode 100644
index 0000000000..4478eb6e2e
--- /dev/null
+++
b/plugins/tech/parquet/src/main/resources/org/apache/hop/ui/hopgui/perspective/explorer/file/messages/messages_en_US.properties
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ParquetExplorer.Tab.Details=Details
+ParquetExplorer.Tab.Structure=Structure
+ParquetExplorer.Tab.Preview=Preview
+ParquetExplorer.Reading=Reading...
+ParquetExplorer.Note=Sizes are the values stored in the file. Row group size
is the largest row group. Data page size and dictionary page size come from the
first page of the first column.
+ParquetExplorer.Error.Title=Error opening Parquet file
+ParquetExplorer.Error.Message=The Parquet file could not be read.
+ParquetExplorer.Details.Column.Property=Property
+ParquetExplorer.Details.Column.Value=Value
+ParquetExplorer.Details.Name=Name
+ParquetExplorer.Details.Folder=Folder
+ParquetExplorer.Details.Size=Size
+ParquetExplorer.Details.Compression=Compression
+ParquetExplorer.Details.Version=Version
+ParquetExplorer.Details.RowGroupSize=Row group size
+ParquetExplorer.Details.DataPageSize=Data page size
+ParquetExplorer.Details.DictionaryPageSize=Dictionary page size
+ParquetExplorer.Details.Rows=Rows
+ParquetExplorer.Details.RowGroups=Row groups
+ParquetExplorer.Details.CreatedBy=Created by
+ParquetExplorer.Structure.Column.Name=Name
+ParquetExplorer.Structure.Column.ParquetType=Parquet type
+ParquetExplorer.Structure.Column.HopType=Hop type
+ParquetExplorer.Structure.Column.Length=Length
+ParquetExplorer.Structure.Column.Precision=Precision
+ParquetExplorer.Preview.Showing=Showing {0} rows
+ParquetExplorer.Preview.First=Showing the first {0} rows
+ParquetExplorer.Preview.Failed=The rows could not be read: {0}
diff --git
a/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/explorer/ParquetFileInspectorTest.java
b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/explorer/ParquetFileInspectorTest.java
new file mode 100644
index 0000000000..c2eebd6c29
--- /dev/null
+++
b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/explorer/ParquetFileInspectorTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.parquet.explorer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.parquet.transforms.input.ParquetStream;
+import org.apache.hop.parquet.transforms.output.ParquetOutputFile;
+import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+class ParquetFileInspectorTest {
+
+ private static final MessageType SCHEMA =
+ Types.buildMessage()
+ .required(PrimitiveTypeName.INT64)
+ .named("id")
+ .optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("name")
+ .optional(PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.decimalType(2, 10))
+ .named("price")
+ .named("row");
+
+ @TempDir private Path tempDir;
+
+ @BeforeAll
+ static void init() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void readsFileDetailsSchemaAndRows() throws Exception {
+ Path path = tempDir.resolve("sample.parquet");
+ write(
+ path,
+ SCHEMA,
+ CompressionCodecName.UNCOMPRESSED,
+ WriterVersion.PARQUET_1_0,
+ (factory, writer) -> {
+ writer.write(row(factory, 1L, "alpha"));
+ writer.write(row(factory, 2L, "beta"));
+ });
+
+ ParquetFileInspection inspection =
+ ParquetFileInspector.inspect(path.toString(), new Variables());
+
+ assertEquals("sample.parquet", inspection.getFileName());
+ assertTrue(
+ inspection.getFolder().contains(tempDir.getFileName().toString()),
inspection.getFolder());
+ assertEquals(Files.size(path), inspection.getSizeBytes());
+ assertEquals("UNCOMPRESSED", inspection.getCompression());
+ assertEquals("Parquet 1.0", inspection.getVersion());
+ assertEquals(2L, inspection.getRowCount());
+ assertEquals(1, inspection.getRowGroupCount());
+ assertFalse(inspection.getCreatedBy().isBlank());
+ assertNotNull(inspection.getDataPageSize());
+ assertTrue(inspection.getDataPageSize() > 0);
+ if (inspection.getDictionaryPageSize() != null) {
+ assertTrue(inspection.getDictionaryPageSize() > 0);
+ }
+ assertNull(inspection.getPreviewError());
+
+ try (FileObject fileObject = HopVfs.getFileObject(path.toString());
+ ParquetStream stream = new ParquetStream(fileObject, path.toString());
+ ParquetFileReader reader = ParquetFileReader.open(stream)) {
+ assertEquals(
+ reader.getFooter().getBlocks().get(0).getTotalByteSize(),
inspection.getRowGroupSize());
+ }
+
+ JsonNode schema = new ObjectMapper().readTree(inspection.getSchemaJson());
+ assertEquals("row", schema.get("name").asText());
+ JsonNode price = field(schema, "price");
+ assertEquals("INT64", price.get("type").asText());
+ assertEquals("OPTIONAL", price.get("repetition").asText());
+ assertEquals("DECIMAL(10,2)", price.get("logicalType").asText());
+
+ ParquetColumnView name = column(inspection, "name");
+ assertEquals("String", name.getHopType());
+ assertEquals(-1, name.getLength());
+ assertEquals(-1, name.getPrecision());
+ assertEquals("BINARY (STRING)", name.getParquetType());
+
+ ParquetColumnView priceColumn = column(inspection, "price");
+ assertEquals("BigNumber", priceColumn.getHopType());
+ assertEquals(10, priceColumn.getLength());
+ assertEquals(2, priceColumn.getPrecision());
+ assertEquals("INT64 (DECIMAL(10,2))", priceColumn.getParquetType());
+
+ assertEquals(2, inspection.getPreviewRows().size());
+ assertEquals(1L, inspection.getPreviewRows().get(0)[0]);
+ assertEquals("alpha", inspection.getPreviewRows().get(0)[1]);
+ assertEquals("beta", inspection.getPreviewRows().get(1)[1]);
+ assertNull(inspection.getPreviewRows().get(0)[2]);
+ assertEquals(List.of("id", "name", "price"),
List.of(inspection.getRowMeta().getFieldNames()));
+ }
+
+ @Test
+ void reportsParquet2WhenTheDataPageIsV2() throws Exception {
+ Path path = tempDir.resolve("v2.parquet");
+ MessageType schema =
+
Types.buildMessage().required(PrimitiveTypeName.INT64).named("id").named("row");
+ write(
+ path,
+ schema,
+ CompressionCodecName.UNCOMPRESSED,
+ WriterVersion.PARQUET_2_0,
+ (factory, writer) -> writer.write(factory.newGroup().append("id",
1L)));
+
+ ParquetFileInspection inspection =
+ ParquetFileInspector.inspect(path.toString(), new Variables());
+
+ assertEquals("Parquet 2.0", inspection.getVersion());
+ }
+
+ @Test
+ void reportsTheCompressionCodec() throws Exception {
+ Path path = tempDir.resolve("snappy.parquet");
+ MessageType schema =
+
Types.buildMessage().required(PrimitiveTypeName.INT64).named("id").named("row");
+ write(
+ path,
+ schema,
+ CompressionCodecName.SNAPPY,
+ WriterVersion.PARQUET_1_0,
+ (factory, writer) -> writer.write(factory.newGroup().append("id",
5L)));
+
+ ParquetFileInspection inspection =
+ ParquetFileInspector.inspect(path.toString(), new Variables());
+
+ assertEquals("SNAPPY", inspection.getCompression());
+ }
+
+ @Test
+ void previewStopsAtOneThousandRows() throws Exception {
+ Path path = tempDir.resolve("many.parquet");
+ MessageType schema =
+
Types.buildMessage().required(PrimitiveTypeName.INT64).named("id").named("row");
+ write(
+ path,
+ schema,
+ CompressionCodecName.UNCOMPRESSED,
+ WriterVersion.PARQUET_1_0,
+ (factory, writer) -> {
+ for (int i = 0; i < ParquetFileInspector.PREVIEW_ROW_LIMIT + 5; i++)
{
+ writer.write(factory.newGroup().append("id", (long) i));
+ }
+ });
+
+ ParquetFileInspection inspection =
+ ParquetFileInspector.inspect(path.toString(), new Variables());
+
+ assertEquals(ParquetFileInspector.PREVIEW_ROW_LIMIT,
inspection.getPreviewRows().size());
+ assertEquals(0L, inspection.getPreviewRows().get(0)[0]);
+ assertEquals(
+ (long) ParquetFileInspector.PREVIEW_ROW_LIMIT - 1,
+ inspection.getPreviewRows().get(ParquetFileInspector.PREVIEW_ROW_LIMIT
- 1)[0]);
+ }
+
+ @Test
+ void rejectsAFileThatIsNotParquet() throws Exception {
+ Path path = tempDir.resolve("not.parquet");
+ try (OutputStream out = HopVfs.getOutputStream(path.toString(), false)) {
+ out.write("not parquet".getBytes(StandardCharsets.UTF_8));
+ }
+
+ HopException exception =
+ assertThrows(
+ HopException.class,
+ () -> ParquetFileInspector.inspect(path.toString(), new
Variables()));
+
+ assertTrue(exception.getMessage().contains("not.parquet"),
exception.getMessage());
+ }
+
+ @Test
+ void schemaJsonKeepsGroupsAndTheTableListsLeaves() throws Exception {
+ MessageType schema =
+ Types.buildMessage()
+ .required(PrimitiveTypeName.INT64)
+ .named("id")
+ .optionalGroup()
+ .optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("city")
+ .named("address")
+ .named("row");
+
+ JsonNode root = new
ObjectMapper().readTree(ParquetFileInspector.schemaJson(schema));
+ assertEquals("group", field(root, "address").get("type").asText());
+ assertEquals("city", field(root,
"address").get("fields").get(0).get("name").asText());
+
+ List<ParquetColumnView> leaves = ParquetFileInspector.leafColumns(schema);
+ assertEquals(
+ List.of("id", "address.city"),
leaves.stream().map(ParquetColumnView::getName).toList());
+ assertEquals("String", leaves.get(1).getHopType());
+ assertEquals("Integer", leaves.get(0).getHopType());
+ }
+
+ private static JsonNode field(JsonNode schema, String name) {
+ for (JsonNode field : schema.get("fields")) {
+ if (name.equals(field.get("name").asText())) {
+ return field;
+ }
+ }
+ throw new AssertionError("missing field " + name);
+ }
+
+ private static ParquetColumnView column(ParquetFileInspection inspection,
String name) {
+ return inspection.getColumns().stream()
+ .filter(column -> name.equals(column.getName()))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ private static Group row(SimpleGroupFactory factory, long id, String name) {
+ return factory.newGroup().append("id", id).append("name",
Binary.fromString(name));
+ }
+
+ private static void write(
+ Path path,
+ MessageType schema,
+ CompressionCodecName codec,
+ WriterVersion version,
+ WriterBody body)
+ throws Exception {
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ try (OutputStream outputStream = HopVfs.getOutputStream(path.toString(),
false);
+ ParquetWriter<Group> writer =
+ ExampleParquetWriter.builder(new ParquetOutputFile(outputStream))
+ .withType(schema)
+ .withCompressionCodec(codec)
+ .withWriterVersion(version)
+ .withWriteMode(ParquetFileWriter.Mode.CREATE)
+ .build()) {
+ body.write(factory, writer);
+ }
+ }
+
+ @FunctionalInterface
+ private interface WriterBody {
+ void write(SimpleGroupFactory factory, ParquetWriter<Group> writer) throws
Exception;
+ }
+}
diff --git
a/plugins/tech/parquet/src/test/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileTypeTest.java
b/plugins/tech/parquet/src/test/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileTypeTest.java
new file mode 100644
index 0000000000..16f201a71d
--- /dev/null
+++
b/plugins/tech/parquet/src/test/java/org/apache/hop/ui/hopgui/perspective/explorer/file/ParquetFileTypeTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.perspective.explorer.file;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Path;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.ui.hopgui.file.IHopFileType;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+class ParquetFileTypeTest {
+
+ private final ParquetFileType fileType = new ParquetFileType();
+
+ @TempDir private Path tempDir;
+
+ @BeforeAll
+ static void init() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void opensParquetAndParqExtensions() throws Exception {
+ assertEquals(".parquet", fileType.getDefaultFileExtension());
+ assertTrue(fileType.supportsOpening());
+ assertTrue(fileType.isHandledBy("/data/orders.PARQUET", false));
+ assertTrue(fileType.isHandledBy("folder/part.parq", false));
+ assertTrue(fileType.isHandledBy("folder/part.Parq", false));
+ assertFalse(fileType.isHandledBy("folder/part.csv", false));
+ assertFalse(fileType.isHandledBy("folder/part.parquet.bak", false));
+ }
+
+ @Test
+ void isReadOnlyInTheExplorer() {
+ assertTrue(fileType.hasCapability(IHopFileType.CAPABILITY_CLOSE));
+ assertTrue(fileType.hasCapability(IHopFileType.CAPABILITY_FILE_HISTORY));
+ assertFalse(fileType.hasCapability(IHopFileType.CAPABILITY_SAVE));
+ assertFalse(fileType.hasCapability(IHopFileType.CAPABILITY_SAVE_AS));
+ assertFalse(fileType.hasCapability(IHopFileType.CAPABILITY_NEW));
+ }
+
+ @Test
+ void keepsTheLargeFileConfirmationForRemoteFiles() throws Exception {
+ Path local = tempDir.resolve("sample.parquet");
+ try (FileObject file = HopVfs.getFileObject(local.toString())) {
+ assertFalse(ParquetFileType.keepsLargeFileConfirmation(file));
+ }
+ try (FileObject remote =
HopVfs.getFileObject("ram:///parquet-preview/sample.parquet")) {
+ assertTrue(ParquetFileType.keepsLargeFileConfirmation(remote));
+ }
+ }
+
+ @Test
+ void createsTheExplorerHandler() {
+ assertInstanceOf(
+ ParquetExplorerFileTypeHandler.class,
fileType.createFileTypeHandler(null, null, null));
+ }
+}
