This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9dd73ebc2c Issues 7043 and 7049 (#7052)
9dd73ebc2c is described below
commit 9dd73ebc2cda4cd2689d8aba2c4bb69304d400ca
Author: Matt Casters <[email protected]>
AuthorDate: Wed Apr 29 09:17:20 2026 +0200
Issues 7043 and 7049 (#7052)
* Issue #7003 (Combination Lookup importer fix)
* Issue #7043 (MySQL bulk loader fixes)
* Issue #7049 (parallel processing bug)
* Issue #7049 (parallel processing bug)
* Issue #7043 (MySQL bulk loader fixes)
---
.../row/value/timestamp/SimpleTimestampFormat.java | 4 +-
.../mysqlbulkloader/MySqlBulkLoader.java | 153 ++++++----
.../mysqlbulkloader/MySqlBulkLoaderData.java | 5 +-
.../mysqlbulkloader/MySqlBulkLoaderDialog.java | 74 ++---
.../mysqlbulkloader/MySqlBulkLoaderMeta.java | 326 ++++++++++++---------
.../messages/messages_en_US.properties | 9 +-
6 files changed, 317 insertions(+), 254 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormat.java
b/core/src/main/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormat.java
index 91d21f46e7..fa4409ead6 100644
---
a/core/src/main/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormat.java
+++
b/core/src/main/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormat.java
@@ -70,10 +70,10 @@ public class SimpleTimestampFormat extends SimpleDateFormat
{
private static final String DEFAULT_MILLISECOND_DATE_FORMAT = "SSS";
- private static final SimpleDateFormat defaultTimestampFormat =
+ private final SimpleDateFormat defaultTimestampFormat =
new SimpleDateFormat(DEFAULT_TIMESTAMP_FORMAT_FOR_TIMESTAMP, Locale.US);
- private static final SimpleDateFormat defaultMillisecondDateFormat =
+ private final SimpleDateFormat defaultMillisecondDateFormat =
new SimpleDateFormat(DEFAULT_MILLISECOND_DATE_FORMAT, Locale.US);
/**
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java
index 0f474c4fdf..213568569d 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java
+++
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java
@@ -17,6 +17,9 @@
package org.apache.hop.pipeline.transforms.mysqlbulkloader;
+import static
org.apache.hop.pipeline.transforms.mysqlbulkloader.MySqlBulkLoaderMeta.Field;
+import static
org.apache.hop.pipeline.transforms.mysqlbulkloader.MySqlBulkLoaderMeta.FieldFormatType;
+
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -24,6 +27,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
+import java.sql.Timestamp;
import java.util.Date;
import org.apache.hop.core.Const;
import org.apache.hop.core.database.Database;
@@ -33,6 +37,7 @@ import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaTimestamp;
import org.apache.hop.core.util.StreamLogger;
import org.apache.hop.core.util.Utils;
import org.apache.hop.i18n.BaseMessages;
@@ -263,19 +268,24 @@ public class MySqlBulkLoader extends
BaseTransform<MySqlBulkLoaderMeta, MySqlBul
data.bulkFormatMeta = new IValueMeta[data.keynrs.length];
for (int i = 0; i < data.keynrs.length; i++) {
IValueMeta sourceMeta =
getInputRowMeta().getValueMeta(data.keynrs[i]);
- if (sourceMeta.isDate()) {
- if
(MySqlBulkLoaderMeta.getFieldFormatType(meta.getFields().get(i).getFieldFormatType())
- == MySqlBulkLoaderMeta.FIELD_FORMAT_TYPE_DATE) {
+ Field field = meta.getFields().get(i);
+
+ if (field.getFieldFormatType() == FieldFormatType.AUTO_FORMAT) {
+ data.bulkFormatMeta[i] =
+ switch (sourceMeta.getType()) {
+ case IValueMeta.TYPE_DATE -> data.bulkDateMeta.clone();
+ case IValueMeta.TYPE_TIMESTAMP ->
data.bulkTimestampMeta.clone();
+ case IValueMeta.TYPE_NUMBER -> data.bulkNumberMeta.clone();
+ default -> null;
+ };
+ } else if (sourceMeta.isDate()) {
+ if (field.getFieldFormatType() == FieldFormatType.DATE) {
data.bulkFormatMeta[i] = data.bulkDateMeta.clone();
- } else if (MySqlBulkLoaderMeta.getFieldFormatType(
- meta.getFields().get(i).getFieldFormatType())
- == MySqlBulkLoaderMeta.FIELD_FORMAT_TYPE_TIMESTAMP) {
+ } else if (field.getFieldFormatType() ==
FieldFormatType.TIMESTAMP) {
data.bulkFormatMeta[i] = data.bulkTimestampMeta.clone(); //
default to timestamp
}
} else if (sourceMeta.isNumeric()
- && MySqlBulkLoaderMeta.getFieldFormatType(
- meta.getFields().get(i).getFieldFormatType())
- == MySqlBulkLoaderMeta.FIELD_FORMAT_TYPE_NUMBER) {
+ && field.getFieldFormatType() == FieldFormatType.NUMBER) {
data.bulkFormatMeta[i] = data.bulkNumberMeta.clone();
}
@@ -357,6 +367,7 @@ public class MySqlBulkLoader extends
BaseTransform<MySqlBulkLoaderMeta, MySqlBul
int index = data.keynrs[i];
IValueMeta valueMeta = rowMeta.getValueMeta(index);
Object valueData = r[index];
+ Field field = meta.getFields().get(i);
if (valueData == null) {
data.fifoStream.write("NULL".getBytes());
@@ -365,17 +376,14 @@ public class MySqlBulkLoader extends
BaseTransform<MySqlBulkLoaderMeta, MySqlBul
case IValueMeta.TYPE_STRING:
data.fifoStream.write(data.quote);
if (valueMeta.isStorageBinaryString()
- && MySqlBulkLoaderMeta.getFieldFormatType(
- meta.getFields().get(i).getFieldFormatType())
- == MySqlBulkLoaderMeta.FIELD_FORMAT_TYPE_OK) {
+ && field.getFieldFormatType() == FieldFormatType.OK) {
// We had a string, just dump it back.
data.fifoStream.write((byte[]) valueData);
} else {
String string = valueMeta.getString(valueData);
if (string != null) {
- if (MySqlBulkLoaderMeta.getFieldFormatType(
- meta.getFields().get(i).getFieldFormatType())
- == MySqlBulkLoaderMeta.FIELD_FORMAT_TYPE_STRING_ESCAPE) {
+ if (field.getFieldFormatType() ==
FieldFormatType.STRING_ESCAPE
+ || field.getFieldFormatType() ==
FieldFormatType.AUTO_FORMAT) {
string =
Const.replace(
string,
@@ -412,13 +420,33 @@ public class MySqlBulkLoader extends
BaseTransform<MySqlBulkLoaderMeta, MySqlBul
}
}
break;
+ case IValueMeta.TYPE_TIMESTAMP:
+ if (valueMeta.isStorageBinaryString() && data.bulkFormatMeta[i]
== null) {
+ data.fifoStream.write(valueMeta.getBinaryString(valueData));
+ } else {
+ Timestamp timestamp = ((ValueMetaTimestamp)
valueMeta).getTimestamp(valueData);
+ if (timestamp != null) {
+
data.fifoStream.write(data.bulkFormatMeta[i].getString(timestamp).getBytes());
+ }
+ }
+ break;
case IValueMeta.TYPE_BOOLEAN:
if (valueMeta.isStorageBinaryString() && data.bulkFormatMeta[i]
== null) {
data.fifoStream.write(valueMeta.getBinaryString(valueData));
} else {
- Boolean b = valueMeta.getBoolean(valueData);
- if (b != null) {
-
data.fifoStream.write(data.bulkFormatMeta[i].getString(b).getBytes());
+ if (data.db.getDatabaseMeta().supportsBooleanDataType()) {
+ // For MySQL this is the bit data type so 1, 0 or null
+ //
+ Long bit = valueMeta.getInteger(valueData);
+ if (bit != null) {
+ data.fifoStream.write(bit.toString().getBytes());
+ }
+ } else {
+ // Write Y/N as a character
+ Boolean b = valueMeta.getBoolean(valueData);
+ if (b != null) {
+
data.fifoStream.write(data.bulkFormatMeta[i].getString(b).getBytes());
+ }
}
}
break;
@@ -501,54 +529,63 @@ public class MySqlBulkLoader extends
BaseTransform<MySqlBulkLoaderMeta, MySqlBul
@Override
public boolean init() {
- if (super.init()) {
+ if (!super.init()) {
+ return false;
+ }
- // Confirming Database Connection is defined.
- try {
- verifyDatabaseConnection();
- } catch (HopException ex) {
- logError(ex.getMessage());
- return false;
- }
+ // Confirming Database Connection is defined.
+ try {
+ verifyDatabaseConnection();
+ } catch (HopException ex) {
+ logError(ex.getMessage());
+ return false;
+ }
- if (Utils.isEmpty(meta.getEnclosure())) {
- data.quote = new byte[] {};
- } else {
- data.quote = resolve(meta.getEnclosure()).getBytes();
- }
- if (Utils.isEmpty(meta.getDelimiter())) {
- data.separator = "\t".getBytes();
- } else {
- data.separator = resolve(meta.getDelimiter()).getBytes();
- }
- data.newline = Const.CR.getBytes();
+ if (Utils.isEmpty(meta.getEnclosure())) {
+ data.quote = new byte[] {};
+ } else {
+ data.quote = resolve(meta.getEnclosure()).getBytes();
+ }
+ if (Utils.isEmpty(meta.getDelimiter())) {
+ data.separator = "\t".getBytes();
+ } else {
+ data.separator = resolve(meta.getDelimiter()).getBytes();
+ }
+ data.newline = Const.CR.getBytes();
- String realEncoding = resolve(meta.getEncoding());
- data.bulkTimestampMeta = new ValueMetaDate("timestampMeta");
- data.bulkTimestampMeta.setConversionMask("yyyy-MM-dd HH:mm:ss");
- data.bulkTimestampMeta.setStringEncoding(realEncoding);
+ String realEncoding = resolve(meta.getEncoding());
- data.bulkDateMeta = new ValueMetaDate("dateMeta");
- data.bulkDateMeta.setConversionMask("yyyy-MM-dd");
- data.bulkDateMeta.setStringEncoding(realEncoding);
+ data.bulkDateMeta = new ValueMetaDate("dateMeta");
+ data.bulkDateMeta.setConversionMask("yyyy-MM-dd");
+ data.bulkDateMeta.setStringEncoding(realEncoding);
- data.bulkNumberMeta = new ValueMetaNumber("numberMeta");
- data.bulkNumberMeta.setConversionMask("#.#");
- data.bulkNumberMeta.setGroupingSymbol(",");
- data.bulkNumberMeta.setDecimalSymbol(".");
- data.bulkNumberMeta.setStringEncoding(realEncoding);
+ data.bulkTimestampMeta = new ValueMetaDate("timestampMeta");
+ data.bulkTimestampMeta.setConversionMask("yyyy-MM-dd HH:mm:ss");
+ data.bulkTimestampMeta.setStringEncoding(realEncoding);
- data.bulkSize = Const.toLong(resolve(meta.getBulkSize()), -1L);
+ data.bulkTimestampMsMeta = new ValueMetaDate("timestampMsMeta");
+ data.bulkTimestampMsMeta.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS");
+ data.bulkTimestampMsMeta.setStringEncoding(realEncoding);
- // Schema-table combination...
- DatabaseMeta databaseMeta =
getPipelineMeta().findDatabase(meta.getConnection(), variables);
- data.schemaTable =
- databaseMeta.getQuotedSchemaTableCombination(
- variables, resolve(meta.getSchemaName()),
resolve(meta.getTableName()));
+ data.bulkTimestampNsMeta = new ValueMetaDate("timestampNsMeta");
+ data.bulkTimestampNsMeta.setConversionMask("yyyy-MM-dd
HH:mm:ss.SSSSSSSSS");
+ data.bulkTimestampNsMeta.setStringEncoding(realEncoding);
- return true;
- }
- return false;
+ data.bulkNumberMeta = new ValueMetaNumber("numberMeta");
+ data.bulkNumberMeta.setConversionMask("#.#");
+ data.bulkNumberMeta.setGroupingSymbol(",");
+ data.bulkNumberMeta.setDecimalSymbol(".");
+ data.bulkNumberMeta.setStringEncoding(realEncoding);
+
+ data.bulkSize = Const.toLong(resolve(meta.getBulkSize()), -1L);
+
+ // Schema-table combination...
+ DatabaseMeta databaseMeta =
getPipelineMeta().findDatabase(meta.getConnection(), variables);
+ data.schemaTable =
+ databaseMeta.getQuotedSchemaTableCombination(
+ variables, resolve(meta.getSchemaName()),
resolve(meta.getTableName()));
+
+ return true;
}
@Override
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java
index 0ef3c10c32..afefd010e7 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java
+++
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java
@@ -38,8 +38,11 @@ public class MySqlBulkLoaderData extends BaseTransformData
implements ITransform
public byte[] separator;
public byte[] newline;
- public IValueMeta bulkTimestampMeta;
public IValueMeta bulkDateMeta;
+ public IValueMeta bulkTimestampMeta;
+ public IValueMeta bulkTimestampMsMeta;
+ public IValueMeta bulkTimestampIsMeta;
+ public IValueMeta bulkTimestampNsMeta;
public IValueMeta bulkNumberMeta;
protected String dbDescription;
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java
index f157be764a..3baf7cf6f8 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java
+++
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java
@@ -523,7 +523,7 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
wTabFolder.addListener(
SWT.Selection,
e -> {
- if (wTabFolder != null && !wTabFolder.isDisposed()) {
+ if (!wTabFolder.isDisposed()) {
wTabFolder.layout(true, true);
}
});
@@ -630,16 +630,11 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
return;
}
- String[] inputNames = new String[sourceFields.size()];
- for (int i = 0; i < sourceFields.size(); i++) {
- IValueMeta value = sourceFields.getValueMeta(i);
- inputNames[i] = value.getName() + "-" + value.getOrigin() + ")";
- }
-
// Create the existing mapping list...
//
List<SourceToTargetMapping> mappings = new ArrayList<>();
StringBuilder missingSourceFields = new StringBuilder();
+ //noinspection MismatchedQueryAndUpdateOfStringBuilder
StringBuilder missingTargetFields = new StringBuilder();
int nrFields = wReturn.nrNonEmpty();
@@ -727,14 +722,11 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
wReturn.table.setItemCount(mappings.size());
for (int i = 0; i < mappings.size(); i++) {
SourceToTargetMapping mapping = mappings.get(i);
+ MySqlBulkLoaderMeta.Field field = input.getFields().get(i);
TableItem item = wReturn.table.getItem(i);
- item.setText(2,
sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
item.setText(1,
targetFields.getValueMeta(mapping.getTargetPosition()).getName());
- item.setText(
- 3,
- MySqlBulkLoaderMeta.getFieldFormatTypeDescription(
- MySqlBulkLoaderMeta.getFieldFormatType(
- input.getFields().get(i).getFieldFormatType())));
+ item.setText(2,
sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
+ item.setText(3, field.getFieldFormatType().getDescription());
}
wReturn.setRowNums();
wReturn.optWidth(true);
@@ -760,17 +752,13 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
if (input.getFields() != null) {
for (int i = 0; i < input.getFields().size(); i++) {
TableItem item = wReturn.table.getItem(i);
- if (input.getFields().get(i).getFieldTable() != null) {
- item.setText(1, input.getFields().get(i).getFieldTable());
- }
- if (input.getFields().get(i).getFieldStream() != null) {
- item.setText(2, input.getFields().get(i).getFieldStream());
+ MySqlBulkLoaderMeta.Field field = input.getFields().get(i);
+ if (field == null) {
+ continue;
}
- item.setText(
- 3,
- MySqlBulkLoaderMeta.getFieldFormatTypeDescription(
- MySqlBulkLoaderMeta.getFieldFormatType(
- input.getFields().get(i).getFieldFormatType())));
+ item.setText(1, Const.NVL(field.getFieldTable(), ""));
+ item.setText(2, Const.NVL(field.getFieldStream(), ""));
+ item.setText(3, field.getFieldFormatType().getDescription());
}
}
@@ -793,18 +781,15 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
protected void setComboBoxes() {
// Something was changed in the row.
- //
- final Map<String, Integer> fields = new HashMap<>();
-
// Add the currentMeta fields...
- fields.putAll(inputFields);
+ final Map<String, Integer> fields = new HashMap<>(inputFields);
Set<String> keySet = fields.keySet();
List<String> entries = new ArrayList<>(keySet);
- String[] fieldNames = entries.toArray(new String[entries.size()]);
+ String[] fieldNames = entries.toArray(new String[0]);
Const.sortStrings(fieldNames);
- // return fields
+ // Set the field names
ciReturn[1].setComboValues(fieldNames);
}
@@ -815,7 +800,6 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
}
private void getInfo(MySqlBulkLoaderMeta inf) {
- int nrfields = wReturn.nrNonEmpty();
inf.setEnclosure(wEnclosure.getText());
inf.setDelimiter(wDelimiter.getText());
@@ -827,22 +811,14 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
inf.setLocalFile(wLocal.getSelection());
inf.setBulkSize(wBulkSize.getText());
- if (log.isDebug()) {
- logDebug(BaseMessages.getString(PKG,
"MySqlBulkLoaderDialog.Log.FoundFields", "" + nrfields));
- }
- // CHECKSTYLE:Indentation:OFF
- List<MySqlBulkLoaderMeta.Field> fields = new ArrayList<>();
- for (int i = 0; i < nrfields; i++) {
- TableItem item = wReturn.getNonEmpty(i);
+ inf.getFields().clear();
+ for (TableItem item : wReturn.getNonEmptyItems()) {
MySqlBulkLoaderMeta.Field field = new MySqlBulkLoaderMeta.Field();
field.setFieldTable(item.getText(1));
field.setFieldStream(item.getText(2));
- field.setFieldFormatType(
- MySqlBulkLoaderMeta.getFieldFormatTypeCode(
- MySqlBulkLoaderMeta.getFieldFormatType(item.getText(3))));
- fields.add(field);
+ field.setFieldFormatTypeWithDescription(item.getText(3));
+ inf.getFields().add(field);
}
- inf.setFields(fields);
inf.setSchemaName(wSchema.getText());
inf.setTableName(wTable.getText());
inf.setConnection(wConnection.getText());
@@ -872,22 +848,16 @@ public class MySqlBulkLoaderDialog extends
BaseTransformDialog {
}
private void getTableName() {
- DatabaseMeta inf = null;
+ DatabaseMeta databaseMeta = null;
if (!wConnection.getText().isEmpty()) {
- inf = pipelineMeta.findDatabase(wConnection.getText(), variables);
+ databaseMeta = pipelineMeta.findDatabase(wConnection.getText(),
variables);
}
- if (inf != null) {
- if (log.isDebug()) {
- logDebug(
- BaseMessages.getString(PKG,
"MySqlBulkLoaderDialog.Log.LookingAtConnection")
- + inf.toString());
- }
-
+ if (databaseMeta != null) {
DatabaseExplorerDialog std =
new DatabaseExplorerDialog(
- shell, SWT.NONE, variables, inf, pipelineMeta.getDatabases(),
false, true);
+ shell, SWT.NONE, variables, databaseMeta,
pipelineMeta.getDatabases(), false, true);
std.setSelectedSchemaAndTable(wSchema.getText(), wTable.getText());
if (std.open()) {
wSchema.setText(Const.NVL(std.getSchemaName(), ""));
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
index 8e8c80dfcb..dec0d3f8bb 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
+++
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.mysqlbulkloader;
+import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
@@ -37,6 +38,8 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IEnumHasCode;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
@@ -58,23 +61,6 @@ import org.w3c.dom.Node;
public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader,
MySqlBulkLoaderData> {
private static final Class<?> PKG = MySqlBulkLoaderMeta.class;
- public static final int FIELD_FORMAT_TYPE_OK = 0;
- public static final int FIELD_FORMAT_TYPE_DATE = 1;
- public static final int FIELD_FORMAT_TYPE_TIMESTAMP = 2;
- public static final int FIELD_FORMAT_TYPE_NUMBER = 3;
- public static final int FIELD_FORMAT_TYPE_STRING_ESCAPE = 4;
-
- private static final String[] fieldFormatTypeCodes = {
- "OK", "DATE", "TIMESTAMP", "NUMBER", "STRING_ESC"
- };
- private static final String[] fieldFormatTypeDescriptions = {
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.OK.Description"),
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Date.Description"),
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description"),
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Number.Description"),
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.StringEscape.Description"),
- };
-
@HopMetadataProperty(key = "schema")
private String schemaName;
@@ -117,29 +103,8 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
@HopMetadataProperty(key = "bulk_size")
private String bulkSize;
- /** Added for backwards compatibility with older XML "mapping" blocks. */
- @Override
- public void convertLegacyXml(Node node) throws HopException {
- if (node == null) {
- return;
- }
-
- if (fields == null) {
- fields = new java.util.ArrayList<>();
- }
-
- int nrvalues = XmlHandler.countNodes(node, "mapping");
- for (int i = 0; i < nrvalues; i++) {
- Node vnode = XmlHandler.getSubNodeByNr(node, "mapping", i);
- if (vnode == null) {
- continue;
- }
- Field field = new Field();
- field.setFieldStream(XmlHandler.getTagValue(vnode, "field_name"));
- field.setFieldTable(XmlHandler.getTagValue(vnode, "stream_name"));
- field.setFieldFormatType(XmlHandler.getTagValue(vnode,
"field_format_ok"));
- fields.add(field);
- }
+ public MySqlBulkLoaderMeta() {
+ fields = new ArrayList<>();
}
@Override
@@ -149,7 +114,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
tableName = BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.DefaultTableName");
encoding = "UTF8";
loadCharSet = "UTF8MB4";
- fifoFileName = "/tmp/fifo";
+ fifoFileName = "${java.io.tmpdir}/fifo.${Internal.Transform.CopyNr}";
delimiter = "\t";
enclosure = "\"";
escapeChar = "\\";
@@ -171,7 +136,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
IVariables variables,
IHopMetadataProvider metadataProvider) {
CheckResult cr;
- String errorMessage = "";
+ StringBuilder errorMessage = new StringBuilder();
if (connection != null) {
DatabaseMeta databaseMeta =
@@ -189,7 +154,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
boolean first = true;
boolean errorFound = false;
- errorMessage = "";
+ errorMessage = new StringBuilder();
// Check fields in table
String schemaTable =
@@ -205,9 +170,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
remarks.add(cr);
// How about the fields to insert/dateMask in the table?
- first = true;
- errorFound = false;
- errorMessage = "";
+ errorMessage = new StringBuilder();
for (Field value : fields) {
String field = value.getFieldTable();
@@ -216,18 +179,21 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
if (v == null) {
if (first) {
first = false;
- errorMessage +=
- BaseMessages.getString(
+ errorMessage
+ .append(
+ BaseMessages.getString(
PKG,
-
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsToLoadInTargetTable")
- + Const.CR;
+
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsToLoadInTargetTable"))
+ .append(Const.CR);
}
errorFound = true;
- errorMessage += "\t\t" + field + Const.CR;
+ errorMessage.append("\t\t").append(field).append(Const.CR);
}
}
if (errorFound) {
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR,
errorMessage, transformMeta);
+ cr =
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR, errorMessage.toString(),
transformMeta);
} else {
cr =
new CheckResult(
@@ -239,9 +205,12 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
remarks.add(cr);
} else {
errorMessage =
- BaseMessages.getString(
- PKG,
"MySqlBulkLoaderMeta.CheckResult.CouldNotReadTableInfo");
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage,
transformMeta);
+ new StringBuilder(
+ BaseMessages.getString(
+ PKG,
"MySqlBulkLoaderMeta.CheckResult.CouldNotReadTableInfo"));
+ cr =
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR, errorMessage.toString(),
transformMeta);
remarks.add(cr);
}
}
@@ -259,7 +228,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
remarks.add(cr);
boolean first = true;
- errorMessage = "";
+ errorMessage = new StringBuilder();
boolean errorFound = false;
for (Field field : fields) {
@@ -267,17 +236,20 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
if (v == null) {
if (first) {
first = false;
- errorMessage +=
- BaseMessages.getString(
- PKG,
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsInInput")
- + Const.CR;
+ errorMessage
+ .append(
+ BaseMessages.getString(
+ PKG,
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsInInput"))
+ .append(Const.CR);
}
errorFound = true;
- errorMessage += "\t\t" + field.getFieldStream() + Const.CR;
+
errorMessage.append("\t\t").append(field.getFieldStream()).append(Const.CR);
}
}
if (errorFound) {
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage,
transformMeta);
+ cr =
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR, errorMessage.toString(),
transformMeta);
} else {
cr =
new CheckResult(
@@ -289,22 +261,29 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
remarks.add(cr);
} else {
errorMessage =
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsInInput3")
- + Const.CR;
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage,
transformMeta);
+ new StringBuilder(
+ BaseMessages.getString(
+ PKG,
"MySqlBulkLoaderMeta.CheckResult.MissingFieldsInInput3")
+ + Const.CR);
+ cr =
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR, errorMessage.toString(),
transformMeta);
remarks.add(cr);
}
} catch (HopException e) {
errorMessage =
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.CheckResult.DatabaseErrorOccurred")
- + e.getMessage();
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage,
transformMeta);
+ new StringBuilder(
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.CheckResult.DatabaseErrorOccurred")
+ + e.getMessage());
+ cr =
+ new CheckResult(ICheckResult.TYPE_RESULT_ERROR,
errorMessage.toString(), transformMeta);
remarks.add(cr);
}
} else {
errorMessage =
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.CheckResult.InvalidConnection");
- cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage,
transformMeta);
+ new StringBuilder(
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.CheckResult.InvalidConnection"));
+ cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR,
errorMessage.toString(), transformMeta);
remarks.add(cr);
}
@@ -335,65 +314,67 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
IHopMetadataProvider metadataProvider)
throws HopTransformException {
- DatabaseMeta databaseMeta =
-
getParentTransformMeta().getParentPipelineMeta().findDatabase(connection,
variables);
+ try {
+ DatabaseMeta databaseMeta =
+
metadataProvider.getSerializer(DatabaseMeta.class).load(variables.resolve(connection));
- SqlStatement retval =
- new SqlStatement(transformMeta.getName(), databaseMeta, null); //
default: nothing to do!
+ SqlStatement sqlStatement =
+ new SqlStatement(transformMeta.getName(), databaseMeta, null); //
default: nothing to do!
- if (databaseMeta != null) {
- if (prev != null && !prev.isEmpty()) {
- // Copy the row
- IRowMeta tableFields = new RowMeta();
-
- // Now change the field names
- for (Field field : fields) {
- IValueMeta v = prev.searchValueMeta(field.getFieldStream());
- if (v != null) {
- IValueMeta tableField = v.clone();
- tableField.setName(field.getFieldStream());
- tableFields.addValueMeta(tableField);
- } else {
- throw new HopTransformException(
- "Unable to find field [" + field.getFieldStream() + "] in the
input rows");
- }
- }
+ if (databaseMeta != null) {
+ if (prev != null && !prev.isEmpty()) {
+ // Copy the row
+ IRowMeta tableFields = new RowMeta();
- if (!Utils.isEmpty(tableName)) {
- Database db = new Database(loggingObject, variables, databaseMeta);
- try {
- db.connect();
-
- String schemaTable =
- databaseMeta.getQuotedSchemaTableCombination(
- variables, variables.resolve(schemaName),
variables.resolve(tableName));
- String crTable = db.getDDL(schemaTable, tableFields, null, false,
null, true);
-
- String sql = crTable;
- if (sql.isEmpty()) {
- retval.setSql(null);
+ // Now change the field names
+ for (Field field : fields) {
+ IValueMeta v = prev.searchValueMeta(field.getFieldStream());
+ if (v != null) {
+ IValueMeta tableField = v.clone();
+ tableField.setName(field.getFieldStream());
+ tableFields.addValueMeta(tableField);
} else {
- retval.setSql(sql);
+ throw new HopTransformException(
+ "Unable to find field [" + field.getFieldStream() + "] in
the input rows");
}
- } catch (HopException e) {
- retval.setError(
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.ErrorOccurred")
- + e.getMessage());
+ }
+
+ if (!Utils.isEmpty(tableName)) {
+ try (Database db = new Database(loggingObject, variables,
databaseMeta)) {
+ db.connect();
+
+ String schemaTable =
+ databaseMeta.getQuotedSchemaTableCombination(
+ variables, variables.resolve(schemaName),
variables.resolve(tableName));
+ String sql = db.getDDL(schemaTable, tableFields, null, false,
null, true);
+ if (sql.isEmpty()) {
+ sqlStatement.setSql(null);
+ } else {
+ sqlStatement.setSql(sql);
+ }
+ } catch (HopException e) {
+ sqlStatement.setError(
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.ErrorOccurred")
+ + e.getMessage());
+ }
+ } else {
+ sqlStatement.setError(
+ BaseMessages.getString(
+ PKG,
"MySqlBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection"));
}
} else {
- retval.setError(
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection"));
+ sqlStatement.setError(
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields"));
}
} else {
- retval.setError(
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields"));
+ sqlStatement.setError(
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.NoConnectionDefined"));
}
- } else {
- retval.setError(
- BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.GetSQL.NoConnectionDefined"));
- }
- return retval;
+ return sqlStatement;
+ } catch (Exception e) {
+ throw new HopTransformException("Error generating SQL statement", e);
+ }
}
@Override
@@ -404,8 +385,7 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
DatabaseMeta databaseMeta =
getParentTransformMeta().getParentPipelineMeta().findDatabase(connection,
variables);
if (databaseMeta != null) {
- Database db = new Database(loggingObject, variables, databaseMeta);
- try {
+ try (Database db = new Database(loggingObject, variables, databaseMeta))
{
db.connect();
if (!Utils.isEmpty(realTableName)) {
@@ -427,8 +407,6 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
} catch (Exception e) {
throw new HopException(
BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.Exception.ErrorGettingFields"), e);
- } finally {
- db.disconnect();
}
} else {
throw new HopException(
@@ -437,29 +415,64 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
}
public static String[] getFieldFormatTypeDescriptions() {
- return fieldFormatTypeDescriptions;
+ return IEnumHasCodeAndDescription.getDescriptions(FieldFormatType.class);
}
- public static String getFieldFormatTypeCode(int type) {
- return fieldFormatTypeCodes[type];
+ public static String getFieldFormatTypeCode(FieldFormatType type) {
+ if (type == null) return "";
+ return type.getCode();
}
- public static String getFieldFormatTypeDescription(int type) {
- return fieldFormatTypeDescriptions[type];
+ public static String getFieldFormatTypeDescription(FieldFormatType type) {
+ if (type == null) return "";
+ return type.getDescription();
}
- public static int getFieldFormatType(String codeOrDescription) {
- for (int i = 0; i < fieldFormatTypeCodes.length; i++) {
- if (fieldFormatTypeCodes[i].equalsIgnoreCase(codeOrDescription)) {
- return i;
- }
+ public static FieldFormatType getFieldFormatType(String codeOrDescription) {
+ FieldFormatType type = IEnumHasCode.lookupCode(FieldFormatType.class,
codeOrDescription, null);
+ if (type != null) {
+ return type;
}
- for (int i = 0; i < fieldFormatTypeDescriptions.length; i++) {
- if (fieldFormatTypeDescriptions[i].equalsIgnoreCase(codeOrDescription)) {
- return i;
- }
+ return IEnumHasCodeAndDescription.lookupDescription(
+ FieldFormatType.class, codeOrDescription, FieldFormatType.OK);
+ }
+
+ @Getter
+ public enum FieldFormatType implements IEnumHasCodeAndDescription {
+ OK("OK", BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.OK.Description")),
+ DATE(
+ "DATE",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Date.Description")),
+ TIMESTAMP(
+ "TIMESTAMP",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description")),
+ TIMESTAMP_MS(
+ "TIMESTAMP_MS",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.TimestampMs.Description")),
+ TIMESTAMP_IS(
+ "TIMESTAMP_IS",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.TimestampIs.Description")),
+ TIMESTAMP_NS(
+ "TIMESTAMP_NS",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.TimestampNs.Description")),
+ NUMBER(
+ "NUMBER",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.Number.Description")),
+ STRING_ESCAPE(
+ "STRING_ESC",
+ BaseMessages.getString(
+ PKG,
"MySqlBulkLoaderMeta.FieldFormatType.StringEscape.Description")),
+ AUTO_FORMAT(
+ "AUTO_FORMAT",
+ BaseMessages.getString(PKG,
"MySqlBulkLoaderMeta.FieldFormatType.AutoFormat.Description")),
+ ;
+ private final String code;
+ private final String description;
+
+ FieldFormatType(String code, String description) {
+ this.code = code;
+ this.description = description;
}
- return FIELD_FORMAT_TYPE_OK;
}
@Getter
@@ -472,7 +485,42 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
@HopMetadataProperty(key = "field_name")
private String fieldStream;
- @HopMetadataProperty(key = "field_format_ok")
- private String fieldFormatType;
+ @HopMetadataProperty(key = "field_format_ok", storeWithCode = true)
+ private FieldFormatType fieldFormatType;
+
+ public void setFieldFormatTypeWithCode(String code) {
+ fieldFormatType = IEnumHasCode.lookupCode(FieldFormatType.class, code,
FieldFormatType.OK);
+ }
+
+ public void setFieldFormatTypeWithDescription(String description) {
+ fieldFormatType =
+ IEnumHasCodeAndDescription.lookupDescription(
+ FieldFormatType.class, description, FieldFormatType.OK);
+ }
+ }
+
+ /** Added for backwards compatibility with older XML "mapping" blocks. */
+ @Override
+ public void convertLegacyXml(Node node) throws HopException {
+ if (node == null) {
+ return;
+ }
+
+ if (fields == null) {
+ fields = new java.util.ArrayList<>();
+ }
+
+ int nrvalues = XmlHandler.countNodes(node, "mapping");
+ for (int i = 0; i < nrvalues; i++) {
+ Node vnode = XmlHandler.getSubNodeByNr(node, "mapping", i);
+ if (vnode == null) {
+ continue;
+ }
+ Field field = new Field();
+ field.setFieldStream(XmlHandler.getTagValue(vnode, "field_name"));
+ field.setFieldTable(XmlHandler.getTagValue(vnode, "stream_name"));
+ field.setFieldFormatTypeWithCode(XmlHandler.getTagValue(vnode,
"field_format_ok"));
+ fields.add(field);
+ }
}
}
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties
b/plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties
index 01f1a6fe33..eb20198792 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties
+++
b/plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties
@@ -97,9 +97,14 @@ MySqlBulkLoaderMeta.FieldFormatType.Date.Description=Format
as Date
MySqlBulkLoaderMeta.FieldFormatType.Number.Description=Format as Number
MySqlBulkLoaderMeta.FieldFormatType.OK.Description=Don''t change formatting
MySqlBulkLoaderMeta.FieldFormatType.StringEscape.Description=Escape enclosure
characters when found
-MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description=Format as a timestamp
+MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description=Format as a
timestamp (s)
+MySqlBulkLoaderMeta.FieldFormatType.TimestampMs.Description=Format as a
timestamp (ms)
+MySqlBulkLoaderMeta.FieldFormatType.TimestampIs.Description=Format as a
timestamp (micros)
+MySqlBulkLoaderMeta.FieldFormatType.TimestampNs.Description=Format as a
timestamp (ns)
MySqlBulkLoaderMeta.GetSQL.ErrorOccurred=An error occurred\:
MySqlBulkLoaderMeta.GetSQL.NoConnectionDefined=There is no connection defined
in this transform.
MySqlBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection=No table is defined on
this connection.
MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields=Not receiving any fields from
previous transforms. Check the previous transforms for errors & the connecting
hops.
-MySqlBulkLoaderDialog.LoadCharSet.Label=Character set (load command)
\ No newline at end of file
+MySqlBulkLoaderDialog.LoadCharSet.Label=Character set (load command)
+MySqlBulkLoaderMeta.FieldFormatType.Boolean.Description=Format as a boolean
(0/1)
+MySqlBulkLoaderMeta.FieldFormatType.AutoFormat.Description=Automatically format
\ No newline at end of file
