hansva commented on code in PR #9607: URL: https://github.com/apache/hop/pull/9607#discussion_r4652543351
########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); Review Comment: `Integer.parseInt` throws `NumberFormatException` on a large length such as `varchar(4294967295)` (how MySQL LONGTEXT can show up). `CatalogFieldsDialog.accept()` only catches `HopException`, so the exception goes to the SWT event loop: the user clicks OK, gets no error dialog, and the field table stays empty. Could we treat an out-of-range length/scale as "no length" (same for line 91), and also catch `RuntimeException` in `accept()`? ########## plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java: ########## @@ -3220,6 +3241,38 @@ private Vector<ITextFileInputField> getFields(TextFileInputMeta info, List<Strin return fields; } + /** Fills the fields from the schema a data catalog has for the first file. */ + private void getFieldsFromCatalog() { + CatalogFieldsDialog.getFields( + shell, + variables, + metadataProvider, + new CatalogDataset(CatalogDatasetType.FILE, null, null, firstFileForCatalog()), + wFields, + 1, + 2, + 3, + 5, + 6); + } + + /** + * The first file the transform reads, resolved the way lineage names it, so a folder with a + * wildcard gives the first matching file. Falls back to the first file row as entered when + * nothing matches yet. + */ + private String firstFileForCatalog() { + if (wFilenameList.getItemCount() <= 0) { + return null; + } + String[] fileRecord = wFilenameList.getItem(0); + String entered = fileRecord == null || fileRecord.length == 0 ? null : fileRecord[0]; + TextFileInputMeta meta = new TextFileInputMeta(); + getInfo(meta, true); + String[] files = meta.getFilePaths(variables); Review Comment: `meta.getFilePaths(variables)` runs a full `FileInputList` listing (every folder, wildcard and subfolder over VFS) on the UI thread, only to pick the first path, and before the catalog dialog even opens. With e.g. `s3://bucket/landing/` + `.*\.csv` + include subfolders, or a slow provider, Hop GUI freezes. Could this resolve only the first row's path, and fall back to the typed dataset name if that fails? ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); + int scale = matcher.group(3) == null ? -1 : Integer.parseInt(matcher.group(3)); + return switch (typeName) { + case "varchar", + "varchar2", + "nvarchar", + "nvarchar2", + "char", + "nchar", + "character", + "character varying", + "bpchar", + "text", + "string", + "clob", + "nclob", + "uuid" -> + withLength(new ValueMetaString(name), length, -1); + case "int", + "integer", + "bigint", + "smallint", + "tinyint", + "mediumint", + "long", + "int2", + "int4", + "int8", + "serial", + "bigserial" -> + new ValueMetaInteger(name); + case "decimal", "numeric", "number", "bignumber" -> + length > 0 && length <= MAX_INTEGER_DIGITS && scale <= 0 + ? withLength(new ValueMetaInteger(name), length, 0) + : withLength(new ValueMetaBigNumber(name), length, scale); + case "double", "double precision", "float", "float4", "float8", "real", "binary_double" -> + new ValueMetaNumber(name); + case "boolean", "bool", "bit" -> new ValueMetaBoolean(name); + case "date" -> new ValueMetaDate(name); + case "datetime", "datetime2", "smalldatetime" -> new ValueMetaDate(name); + case "time" -> new ValueMetaDate(name); + case "timestamp", "timestamptz", "timestamp with time zone", "timestamp without time zone" -> + new ValueMetaTimestamp(name); + case "binary", "varbinary", "blob", "longblob", "bytea", "bytes", "raw" -> + new ValueMetaBinary(name); + default -> null; + }; + } + + /** DataHub's generic field types, the names of SchemaFieldDataType. */ + private static IValueMeta fromCatalogType(String name, String type) { + String typeName = StringUtils.defaultString(type).trim().toUpperCase(Locale.ROOT); + return switch (typeName) { + case "NUMBER" -> new ValueMetaNumber(name); + case "BOOLEAN" -> new ValueMetaBoolean(name); + case "DATE" -> new ValueMetaDate(name); + case "TIME" -> new ValueMetaTimestamp(name); Review Comment: Native `time` maps to Date (line 130) but the catalog-type fallback maps `TIME` to Timestamp, so the same column can get a different Hop type depending on what the catalog sends. Could both use the same mapping? ########## ui/src/main/java/org/apache/hop/ui/datacatalog/CatalogFieldsDialog.java: ########## @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.datacatalog; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.datacatalog.CatalogDataset; +import org.apache.hop.datacatalog.CatalogFieldTypes; +import org.apache.hop.datacatalog.CatalogSchemas; +import org.apache.hop.datacatalog.DataCatalog; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Shell; + +/** + * Asks for a data catalog and a dataset in it, and reads the fields of that dataset, for a + * transform dialog to fill its field table with. The transform keeps its own field definitions: + * nothing is read from the catalog when the pipeline runs. + */ +public class CatalogFieldsDialog { + + private static final Class<?> PKG = CatalogFieldsDialog.class; + + /** The catalog picked last in this session, to start from the next time. */ + private static String lastCatalogName; + + private final Shell parent; + private final IVariables variables; + private final IHopMetadataProvider metadataProvider; + private final CatalogFieldsSource source; + private IRowMeta fields; + + /** + * @param dataset the dataset to start from, typically the file or table of the transform + */ + public CatalogFieldsDialog( + Shell parent, + IVariables variables, + IHopMetadataProvider metadataProvider, + CatalogDataset dataset) { + this.parent = parent; + this.variables = variables; + this.metadataProvider = metadataProvider; + this.source = new CatalogFieldsSource(); + source.setCatalogName(defaultCatalogName(metadataProvider)); + if (dataset != null) { + source.setDatasetType(dataset.getType()); + source.setConnection(dataset.getConnection()); + source.setSchema(dataset.getSchema()); + source.setName(dataset.getName()); + } + } + + /** True when the project has a data catalog to get fields from. */ + public static boolean isAvailable(IHopMetadataProvider metadataProvider) { + return !catalogNames(metadataProvider).isEmpty(); + } + + /** + * Fills a field table with the fields of a dataset in a data catalog. The person picks the + * catalog and dataset, and then whether to add the new fields, add all of them or replace the + * table. + * + * @param formatColumn the column of the format, or -1 when the table has none + */ + public static void getFields( + Shell parent, + IVariables variables, + IHopMetadataProvider metadataProvider, + CatalogDataset dataset, + TableView table, + int nameColumn, + int typeColumn, + int formatColumn, + int lengthColumn, + int precisionColumn) { + IRowMeta rowMeta = new CatalogFieldsDialog(parent, variables, metadataProvider, dataset).open(); + if (rowMeta == null) { + return; + } + BaseTransformDialog.getFieldsFromPrevious( + rowMeta, + table, + nameColumn, + new int[] {nameColumn}, + new int[] {typeColumn}, + lengthColumn, + precisionColumn, + true, + (item, valueMeta) -> { + if (formatColumn > 0 && valueMeta.isDate()) { Review Comment: The conversion mask is always null on the value metas built from the catalog, so this only writes an empty string into an already empty cell. The `formatColumn` parameter passed by both dialogs has no effect; either drop it or set a real format to write. ########## plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/GetFieldsFromCatalogDialogTest.java: ########## @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.csvinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import org.apache.hop.core.fileinput.InputFile; +import org.apache.hop.core.gui.plugin.GuiRegistry; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.plugins.TransformPluginType; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.datacatalog.CatalogAnswer; +import org.apache.hop.datacatalog.CatalogEntityId; +import org.apache.hop.datacatalog.DataCatalog; +import org.apache.hop.datacatalog.DatasetRef; +import org.apache.hop.datacatalog.IDataCatalog; +import org.apache.hop.datacatalog.ISchemaCapability; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.ITransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.fileinput.text.TextFileInputDialog; +import org.apache.hop.pipeline.transforms.fileinput.text.TextFileInputMeta; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.datacatalog.CatalogFieldsDialog; +import org.apache.hop.ui.datacatalog.CatalogFieldsSource; +import org.apache.hop.ui.hopgui.HopGui; +import org.apache.hop.ui.hopgui.HopGuiImpl; +import org.apache.hop.ui.testing.SwtBotTestBase; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swtbot.swt.finder.SWTBot; +import org.eclipse.swtbot.swt.finder.utils.SWTUtils; +import org.eclipse.swtbot.swt.finder.waits.DefaultCondition; +import org.eclipse.swtbot.swt.finder.widgets.SWTBotShell; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * The CSV file input and Text file input dialogs fill their field table from the schema a data + * catalog has for their file. + */ +@Tag("uitest") +class GetFieldsFromCatalogDialogTest extends SwtBotTestBase { + + private static final String FILE = "/data/in/customers.csv"; + + private static final String CATALOG_TITLE = + BaseMessages.getString(CatalogFieldsDialog.class, "CatalogFieldsDialog.Title"); + + /** Name, type, format, length and precision of the three fields of the dataset. */ + private static final List<List<String>> EXPECTED = + List.of( + List.of("id", "Integer", "", "", "0"), + List.of("email", "String", "", "255", ""), + List.of("signed_up", "Date", "", "", "")); + + private static MemoryMetadataProvider metadataProvider; + + /** The annotated widgets are looked up by class name; register them as the plugin scan does. */ + @BeforeAll + static void registerWidgets() { + GuiRegistry registry = GuiRegistry.getInstance(); + if (registry.findGuiElements(CatalogFieldsSource.class.getName(), CatalogFieldsSource.PARENT_ID) + != null) { + return; + } + for (Field field : CatalogFieldsSource.class.getDeclaredFields()) { + GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class); + if (element != null) { + registry.addGuiWidgetElement(CatalogFieldsSource.class.getName(), element, field); + } + } + } + + /** The metadata selection lines read their provider from the running Hop GUI. */ + @BeforeAll + static void installHopGui() throws Exception { + metadataProvider = new MemoryMetadataProvider(); + metadataProvider + .getSerializer(DataCatalog.class) + .save(new DataCatalog("catalog", null, new SchemaCatalog())); + Variables variables = new Variables(); + HopGui hopGui = mock(HopGui.class); + when(hopGui.getMetadataProvider()) + .thenReturn(new MultiMetadataProvider(variables, metadataProvider)); + when(hopGui.getVariables()).thenReturn(variables); + setHopGui(hopGui); + } + + @AfterAll + static void removeHopGui() throws Exception { + setHopGui(null); + } + + private static void setHopGui(HopGui hopGui) throws Exception { + Field instance = HopGuiImpl.class.getDeclaredField("instance"); + instance.setAccessible(true); + instance.set(null, hopGui); + } + + @BeforeEach + void forget() { + SchemaCatalog.asked = null; + } + + @Test + void csvFileInputFillsItsFieldsFromTheCatalog() { + CsvInputMeta meta = new CsvInputMeta(); + meta.setDefault(); + meta.setFilename(FILE); + PipelineMeta pipelineMeta = pipelineWith(meta, "CSV file input"); + List<List<String>> rows = new ArrayList<>(); + + withDialog( + parent -> new CsvInputDialog(parent, new Variables(), meta, pipelineMeta).open(), + bot -> { + SWTBotShell dialog = + bot.shell(BaseMessages.getString(CsvInput.class, "CsvInputDialog.Shell.Title")); + dialog.activate(); + dialog + .bot() + .button( + BaseMessages.getString( + CsvInput.class, "CsvInputDialog.GetFieldsFromCatalog.Button")) + .click(); + getFieldsFromCatalog(bot, "csv"); + rows.addAll(readFields(bot, dialog, shell -> onlyTable(shell))); + dialog.bot().button(buttonLabel("System.Button.Cancel")).click(); + }); + + assertEquals(EXPECTED, rows); + assertNotNull(SchemaCatalog.asked, "the catalog was not asked"); + assertTrue( + SchemaCatalog.asked.name().endsWith(FILE), + "the dataset starts as the file: " + SchemaCatalog.asked); + } + + @Test + void textFileInputFillsItsFieldsFromTheCatalog() { + TextFileInputMeta meta = new TextFileInputMeta(); + meta.setDefault(); + InputFile file = new InputFile(); + file.setFileName(FILE); + meta.getFileInput().getInputFiles().add(file); + PipelineMeta pipelineMeta = pipelineWith(meta, "Text file input"); + String fieldsTab = + BaseMessages.getString(TextFileInputMeta.class, "TextFileInputDialog.FieldsTab.TabTitle"); + List<List<String>> rows = new ArrayList<>(); + + withDialog( + parent -> new TextFileInputDialog(parent, new Variables(), meta, pipelineMeta).open(), + bot -> { + SWTBotShell dialog = + bot.shell( + BaseMessages.getString( + TextFileInputMeta.class, "TextFileInputDialog.DialogTitle")); + dialog.activate(); + display.syncExec(() -> selectTab(dialog.widget, fieldsTab)); + dialog + .bot() + .button( + BaseMessages.getString( + TextFileInputMeta.class, "TextFileInputDialog.GetFieldsFromCatalog.Button")) + .click(); + getFieldsFromCatalog(bot, "text"); + rows.addAll(readFields(bot, dialog, shell -> tableOnTab(shell, fieldsTab))); + dialog.bot().button(buttonLabel("System.Button.Cancel")).click(); + }); + + // Text file input has a position column between the format and the length + assertEquals(EXPECTED, rows); + assertTrue(SchemaCatalog.asked.name().endsWith(FILE), String.valueOf(SchemaCatalog.asked)); + } + + @Test + void csvFileInputDisablesTheCatalogButtonWhenItIgnoresTheFields() { + CsvInputMeta meta = new CsvInputMeta(); + meta.setDefault(); + meta.setFilename(FILE); + meta.setIgnoreFields(true); + PipelineMeta pipelineMeta = pipelineWith(meta, "CSV file input"); + boolean[] enabled = new boolean[1]; + + withDialog( + parent -> new CsvInputDialog(parent, new Variables(), meta, pipelineMeta).open(), + bot -> { + SWTBotShell dialog = + bot.shell(BaseMessages.getString(CsvInput.class, "CsvInputDialog.Shell.Title")); + dialog.activate(); + enabled[0] = + dialog + .bot() + .button( + BaseMessages.getString( + CsvInput.class, "CsvInputDialog.GetFieldsFromCatalog.Button")) + .isEnabled(); + dialog.bot().button(buttonLabel("System.Button.Cancel")).click(); + }); + + assertFalse(enabled[0], "the button does nothing while the fields are ignored"); + } + + @Test + void textFileInputLooksUpTheFirstFileAFolderWildcardMatches(@TempDir Path folder) + throws Exception { + Files.writeString(folder.resolve("customers.csv"), "id;email;signed_up\n"); Review Comment: Minor: our convention is to read/write files through VFS (`HopVfs.getOutputStream(...)` / `FileObject`) rather than `java.nio.file.Files`; could the fixtures be written that way? ########## ui/src/main/java/org/apache/hop/ui/datacatalog/CatalogFieldsDialog.java: ########## @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.datacatalog; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.datacatalog.CatalogDataset; +import org.apache.hop.datacatalog.CatalogFieldTypes; +import org.apache.hop.datacatalog.CatalogSchemas; +import org.apache.hop.datacatalog.DataCatalog; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Shell; + +/** + * Asks for a data catalog and a dataset in it, and reads the fields of that dataset, for a + * transform dialog to fill its field table with. The transform keeps its own field definitions: + * nothing is read from the catalog when the pipeline runs. + */ +public class CatalogFieldsDialog { + + private static final Class<?> PKG = CatalogFieldsDialog.class; + + /** The catalog picked last in this session, to start from the next time. */ + private static String lastCatalogName; Review Comment: `lastCatalogName` is a JVM-wide static, so in Hop Web all sessions share it and one user's choice becomes another user's default. Could this be kept per session (session-scoped singleton or the audit/props state)? ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); + int scale = matcher.group(3) == null ? -1 : Integer.parseInt(matcher.group(3)); + return switch (typeName) { + case "varchar", + "varchar2", + "nvarchar", + "nvarchar2", + "char", + "nchar", + "character", + "character varying", + "bpchar", + "text", + "string", + "clob", + "nclob", + "uuid" -> + withLength(new ValueMetaString(name), length, -1); + case "int", + "integer", + "bigint", + "smallint", + "tinyint", + "mediumint", + "long", + "int2", + "int4", + "int8", + "serial", + "bigserial" -> + new ValueMetaInteger(name); + case "decimal", "numeric", "number", "bignumber" -> + length > 0 && length <= MAX_INTEGER_DIGITS && scale <= 0 + ? withLength(new ValueMetaInteger(name), length, 0) + : withLength(new ValueMetaBigNumber(name), length, scale); + case "double", "double precision", "float", "float4", "float8", "real", "binary_double" -> + new ValueMetaNumber(name); + case "boolean", "bool", "bit" -> new ValueMetaBoolean(name); Review Comment: `bit(n)` with n > 1 (MySQL `BIT(8)`, PostgreSQL `bit(n)`) is a bit string rather than a flag, so mapping it to Boolean will fail at runtime on values like `10110000`. Suggest Boolean only for `bit` / `bit(1)`, and Binary or String otherwise. ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogSchemas.java: ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; + +/** Reads the fields a data catalog has for a dataset. */ +public final class CatalogSchemas { + private static final Class<?> PKG = CatalogSchemas.class; + + private CatalogSchemas() {} + + /** + * The fields a data catalog has for a dataset. + * + * @throws HopException when there is no such catalog or dataset, or the catalog doesn't answer + */ + public static List<CatalogField> read( + IVariables variables, + IHopMetadataProvider metadataProvider, + String catalogName, + CatalogDataset dataset) + throws HopException { + String resolvedName = variables.resolve(catalogName); + if (StringUtils.isBlank(resolvedName)) { + throw new HopException(BaseMessages.getString(PKG, "CatalogSchemas.Error.NoCatalog")); + } + DataCatalog dataCatalog = metadataProvider.getSerializer(DataCatalog.class).load(resolvedName); + if (dataCatalog == null) { + throw new HopException( + BaseMessages.getString(PKG, "CatalogSchemas.Error.MissingCatalog", resolvedName)); + } + DatasetRef ref = dataset.toRef(variables, metadataProvider); + try (IDataCatalog catalog = dataCatalog.connect(variables, metadataProvider)) { + ISchemaCapability schemaCapability = + catalog + .capability(ISchemaCapability.class) + .orElseThrow( + () -> + new HopException( + BaseMessages.getString( + PKG, "CatalogSchemas.Error.NoSchemaSupport", resolvedName))); + CatalogEntityId id = known(catalog.resolve(ref), resolvedName, ref); + return known(schemaCapability.schema(id), resolvedName, ref); + } + } + + private static <T> T known(CatalogAnswer<T> answer, String catalogName, DatasetRef ref) + throws HopException { + return switch (answer) { + case CatalogAnswer.Known<T> known -> known.value(); Review Comment: A `Known` answer with an empty field list passes through here. The dialog then builds an empty row, `getFieldsFromPrevious` returns early, and the dialog closes without changing anything or telling the user why. Could an empty list be treated as not found here, or reported in `accept()` with the dialog kept open? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
