This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new c3abb154be Blocking transform is slow when "Pass all rows" is on,
fixes #9344 (#9348)
c3abb154be is described below
commit c3abb154bee933658807e00e04a9106efcce3a1c
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:59:19 2026 +0100
Blocking transform is slow when "Pass all rows" is on, fixes #9344 (#9348)
---
.../pipeline/transforms/blockingtransform.adoc | 6 +-
.../ROOT/pages/pipeline/transforms/sort.adoc | 4 +
.../core/compress/gzip/GzipCompressionLevel.java | 100 ++++++++++
.../gzip/messages/messages_en_US.properties | 26 +++
.../compress/gzip/GzipCompressionLevelTest.java | 105 ++++++++++
.../blockingtransform/BlockingTransform.java | 43 +++--
.../blockingtransform/BlockingTransformData.java | 7 +
.../blockingtransform/BlockingTransformDialog.java | 47 ++++-
.../blockingtransform/BlockingTransformMeta.java | 26 +++
.../messages/messages_en_US.properties | 2 +
.../BlockingTransformMetaTest.java | 32 ++++
.../blockingtransform/BlockingTransformTest.java | 212 +++++++++++++++++++++
.../hop/pipeline/transforms/sort/SortRows.java | 11 +-
.../pipeline/transforms/sort/SortRowsDialog.java | 41 +++-
.../hop/pipeline/transforms/sort/SortRowsMeta.java | 18 ++
.../sort/messages/messages_en_US.properties | 3 +
.../transforms/sort/SortRowsMetaInjectionTest.java | 2 +
.../pipeline/transforms/sort/SortRowsMetaTest.java | 33 ++++
.../hop/pipeline/transforms/sort/SortRowsTest.java | 147 ++++++++++++++
19 files changed, 845 insertions(+), 20 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
index 68d8817c49..f29d490f0b 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
@@ -40,5 +40,9 @@ Use the Blocking transform for triggering plugins, stored
procedures, Java scrip
|Spool directory|The directory in which the temporary files are stored if
needed; the default is the standard temporary directory for the system
|Spool-file prefix|Choose a recognizable prefix to identify the files when
they appear in the temp directory
|Cache size|The more rows you can store in memory, the faster the transform
works
-|Compress spool files?|Compresses temporary files when they are needed
+|Compress spool files?|Compresses temporary files when they are needed.
+You can also give a variable: when it is set, its value (Y or N) decides
whether the files are compressed.
+|Compression level|How hard to compress the spool files: 1 - Fastest, 2 - Very
fast, 3 - Fast, 4 - Balanced, faster, 5 - Balanced, 6 - Standard (the Java
default), 7 - Smaller files, 8 - Very small files, 9 - Smallest files.
+Spool files are read back only once, so a low level is usually best.
+A new transform uses 1 - Fastest. A transform saved before this option existed
keeps 6 - Standard, the level it always used.
|===
\ No newline at end of file
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
index 0e9ab79147..f01d182bfb 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
@@ -52,6 +52,10 @@ The native Spark engine does support this transform: it is
mapped onto a Spark s
|Sort size|The more rows you store in memory, the faster the sorting process
because fewer temporary files must be used and less I/O is generated.
|Free memory threshold (in %)|If the sort algorithm finds that it has less
available free memory than the indicated number, it will start to page data to
disk.
|Compress TMP Files|Compresses temporary files when they are needed to
complete the sort.
+You can also give a variable: when it is set, its value (Y or N) decides
whether the files are compressed.
+|Compression level|How hard to compress the temporary files: 1 - Fastest, 2 -
Very fast, 3 - Fast, 4 - Balanced, faster, 5 - Balanced, 6 - Standard (the Java
default), 7 - Smaller files, 8 - Very small files, 9 - Smallest files.
+Temporary files are read back only once, so a low level is usually best.
+A new transform uses 1 - Fastest. A transform saved before this option existed
keeps 6 - Standard, the level it always used.
|Only pass unique rows?|Enable if you want to pass unique rows only to the
output stream(s).
|Fields table|Specify the fields and direction (ascending/descending) to sort.
You can specify whether to perform a case sensitive sort (optional)
diff --git
a/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
b/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
new file mode 100644
index 0000000000..77ce538cdd
--- /dev/null
+++
b/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.compress.gzip;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+
+/**
+ * The GZIP compression levels 1 to 9: lower is faster, higher gives smaller
files. Used for the
+ * temporary files of transforms such as Sort rows and Blocking transform.
+ */
+public enum GzipCompressionLevel implements IEnumHasCodeAndDescription {
+ FASTEST(1),
+ VERY_FAST(2),
+ FAST(3),
+ BALANCED_FASTER(4),
+ BALANCED(5),
+ STANDARD(6),
+ SMALLER(7),
+ VERY_SMALL(8),
+ SMALLEST(9);
+
+ private static final Class<?> PKG = GzipCompressionLevel.class;
+
+ /**
+ * The level of a newly created transform: temporary files are read back
only once, so speed
+ * matters most.
+ */
+ public static final GzipCompressionLevel NEW_TRANSFORM_DEFAULT = FASTEST;
+
+ /**
+ * The level of a transform saved before the level could be chosen: the
level those transforms
+ * always used, so they keep working the same way.
+ */
+ public static final GzipCompressionLevel NOT_STORED_DEFAULT = STANDARD;
+
+ private final int level;
+
+ GzipCompressionLevel(int level) {
+ this.level = level;
+ }
+
+ /** The level passed to {@link java.util.zip.Deflater}, from 1 to 9. */
+ public int getLevel() {
+ return level;
+ }
+
+ /** The constant name: this is what transforms store in their metadata. */
+ @Override
+ public String getCode() {
+ return name();
+ }
+
+ /** The level number and its name, for example "1 - Fastest". */
+ @Override
+ public String getDescription() {
+ return level + " - " + BaseMessages.getString(PKG, "GzipCompressionLevel."
+ name());
+ }
+
+ /**
+ * @param level a stored level, or null when none was stored (metadata from
before the option)
+ * @return the level, or {@link #NOT_STORED_DEFAULT} when it is null
+ */
+ public static GzipCompressionLevel orNotStoredDefault(GzipCompressionLevel
level) {
+ return level == null ? NOT_STORED_DEFAULT : level;
+ }
+
+ /**
+ * Opens a GZIP stream that compresses at this level.
+ *
+ * @param out the stream to write the compressed data to
+ * @return the GZIP stream
+ * @throws IOException when the GZIP header cannot be written
+ */
+ public GZIPOutputStream newOutputStream(OutputStream out) throws IOException
{
+ return new GZIPOutputStream(out) {
+ {
+ def.setLevel(level);
+ }
+ };
+ }
+}
diff --git
a/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
b/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
new file mode 100644
index 0000000000..4ffc9db7be
--- /dev/null
+++
b/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+GzipCompressionLevel.FASTEST=Fastest
+GzipCompressionLevel.VERY_FAST=Very fast
+GzipCompressionLevel.FAST=Fast
+GzipCompressionLevel.BALANCED_FASTER=Balanced, faster
+GzipCompressionLevel.BALANCED=Balanced
+GzipCompressionLevel.STANDARD=Standard (the Java default)
+GzipCompressionLevel.SMALLER=Smaller files
+GzipCompressionLevel.VERY_SMALL=Very small files
+GzipCompressionLevel.SMALLEST=Smallest files
diff --git
a/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
b/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
new file mode 100644
index 0000000000..5fa8b6bd6b
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.compress.gzip;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class GzipCompressionLevelTest {
+
+ @Test
+ void levelsRunFromOneToNine() {
+ GzipCompressionLevel[] levels = GzipCompressionLevel.values();
+ assertEquals(9, levels.length);
+ for (int i = 0; i < levels.length; i++) {
+ assertEquals(i + 1, levels[i].getLevel());
+ }
+ }
+
+ @Test
+ void descriptionsShowTheNumberAndAName() {
+ assertEquals("1 - Fastest", GzipCompressionLevel.FASTEST.getDescription());
+ assertEquals("6 - Standard (the Java default)",
GzipCompressionLevel.STANDARD.getDescription());
+ assertEquals("9 - Smallest files",
GzipCompressionLevel.SMALLEST.getDescription());
+ for (GzipCompressionLevel level : GzipCompressionLevel.values()) {
+ assertEquals(
+ level,
+ IEnumHasCodeAndDescription.lookupDescription(
+ GzipCompressionLevel.class, level.getDescription(), null));
+ assertEquals(level.name(), level.getCode());
+ }
+ }
+
+ @Test
+ void noStoredLevelMeansTheLevelUsedBefore() {
+ assertEquals(GzipCompressionLevel.STANDARD,
GzipCompressionLevel.orNotStoredDefault(null));
+ assertEquals(6, GzipCompressionLevel.orNotStoredDefault(null).getLevel());
+ assertEquals(
+ GzipCompressionLevel.SMALLEST,
+
GzipCompressionLevel.orNotStoredDefault(GzipCompressionLevel.SMALLEST));
+ }
+
+ @ParameterizedTest
+ @EnumSource(GzipCompressionLevel.class)
+ void compressedDataReadsBack(GzipCompressionLevel level) throws Exception {
+ byte[] data = sampleData();
+ assertArrayEquals(data, decompress(compress(level, data)));
+ }
+
+ @Test
+ void theLevelIsApplied() throws Exception {
+ byte[] data = sampleData();
+ int fastest = compress(GzipCompressionLevel.FASTEST, data).length;
+ int smallest = compress(GzipCompressionLevel.SMALLEST, data).length;
+ assertTrue(smallest < fastest, "level 9: " + smallest + " bytes, level 1:
" + fastest);
+ }
+
+ private static byte[] sampleData() {
+ StringBuilder text = new StringBuilder();
+ for (int i = 0; i < 20_000; i++) {
+ text.append("order ").append(i).append(", customer ").append(i %
977).append('\n');
+ }
+ return text.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static byte[] compress(GzipCompressionLevel level, byte[] data)
throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (GZIPOutputStream out = level.newOutputStream(bytes)) {
+ out.write(data);
+ }
+ return bytes.toByteArray();
+ }
+
+ private static byte[] decompress(byte[] data) throws IOException {
+ try (GZIPInputStream in = new GZIPInputStream(new
ByteArrayInputStream(data))) {
+ return in.readAllBytes();
+ }
+ }
+}
diff --git
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
index d9320b37e8..d74f6ed592 100644
---
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
+++
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
@@ -30,6 +30,7 @@ import java.util.zip.GZIPOutputStream;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.io.CountingInputStream;
@@ -50,6 +51,12 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
private static final Class<?> PKG = BlockingTransformMeta.class;
+ /**
+ * Buffer between the row data and the (compressed) temp file, so values are
written and read in
+ * blocks rather than one at a time.
+ */
+ private static final int TEMP_FILE_BUFFER_SIZE = 65536;
+
private Object[] lastRow;
public BlockingTransform(
@@ -85,11 +92,13 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
data.files.add(fileObject); // Remember the files!
OutputStream outputStream = HopVfs.getOutputStream(fileObject, false);
CountingOutputStream counting = new CountingOutputStream(outputStream);
- if (meta.isCompressFiles()) {
- gzos = new GZIPOutputStream(new BufferedOutputStream(counting));
- dos = new DataOutputStream(gzos);
+ if (data.compressFiles) {
+ gzos =
+
GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel())
+ .newOutputStream(new BufferedOutputStream(counting));
+ dos = new DataOutputStream(new BufferedOutputStream(gzos,
TEMP_FILE_BUFFER_SIZE));
} else {
- dos = new DataOutputStream(counting);
+ dos = new DataOutputStream(new BufferedOutputStream(counting,
TEMP_FILE_BUFFER_SIZE));
gzos = null;
}
@@ -141,13 +150,16 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
data.countingInputStreams.add(countingFi);
data.fis.add(countingFi);
DataInputStream di;
- if (meta.isCompressFiles()) {
+ if (data.compressFiles) {
GZIPInputStream gzfi =
new GZIPInputStream(new
BufferedInputStream(data.fis.get(data.fis.size() - 1)));
- di = new DataInputStream(gzfi);
+ di = new DataInputStream(new BufferedInputStream(gzfi,
TEMP_FILE_BUFFER_SIZE));
data.gzis.add(gzfi);
} else {
- di = new DataInputStream(data.fis.get(data.fis.size() - 1));
+ di =
+ new DataInputStream(
+ new BufferedInputStream(
+ data.fis.get(data.fis.size() - 1),
TEMP_FILE_BUFFER_SIZE));
}
data.dis.add(di);
@@ -176,10 +188,15 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
}
if (data.files.isEmpty()) {
- if (!data.buffer.isEmpty()) {
- retval = data.buffer.get(0);
- data.buffer.remove(0);
+ // Walk the buffer in order instead of removing its first row, which
moves all the others.
+ // Each slot is cleared once passed on, so the memory is still freed as
the rows leave.
+ if (data.bufferIndex < data.buffer.size()) {
+ retval = data.buffer.get(data.bufferIndex);
+ data.buffer.set(data.bufferIndex, null);
+ data.bufferIndex++;
} else {
+ data.buffer.clear();
+ data.bufferIndex = 0;
retval = null;
}
} else {
@@ -194,7 +211,7 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
FileObject file = data.files.get(0);
DataInputStream di = data.dis.get(0);
InputStream fi = data.fis.get(0);
- GZIPInputStream gzfi = (meta.isCompressFiles()) ? data.gzis.get(0) :
null;
+ GZIPInputStream gzfi = (data.compressFiles) ? data.gzis.get(0) : null;
try {
data.rowbuffer.add(0, data.outputRowMeta.readData(di));
@@ -267,7 +284,9 @@ public class BlockingTransform extends
BaseTransform<BlockingTransformMeta, Bloc
public boolean init() {
if (super.init()) {
- // Add init code here.
+ // Compress the temporary files? A variable, when given, decides.
+ data.compressFiles =
+ getVariableBoolean(meta.getCompressFilesVariable(),
meta.isCompressFiles());
return true;
}
return false;
diff --git
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
index c6f797d81f..d84170cb32 100644
---
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
+++
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
@@ -32,12 +32,19 @@ import org.apache.hop.pipeline.transform.ITransformData;
public class BlockingTransformData extends BaseTransformData implements
ITransformData {
public List<FileObject> files;
public List<Object[]> buffer;
+
+ /** Position of the next row to pass on from {@link #buffer} when no rows
went to disk. */
+ public int bufferIndex;
+
public List<InputStream> fis;
public List<CountingInputStream> countingInputStreams;
public List<GZIPInputStream> gzis;
public List<DataInputStream> dis;
public List<Object[]> rowbuffer;
+ /** Whether the temporary files are compressed, resolved when the transform
starts. */
+ public boolean compressFiles;
+
public IRowMeta outputRowMeta;
public int[] fieldnrs; // the corresponding field numbers
diff --git
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
index 913516e883..2300e7d52e 100644
---
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
+++
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
@@ -18,15 +18,19 @@
package org.apache.hop.pipeline.transforms.blockingtransform;
import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.widget.CheckBoxVar;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.custom.ScrolledComposite;
import org.eclipse.swt.events.ModifyListener;
import org.eclipse.swt.events.SelectionAdapter;
@@ -60,7 +64,10 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
private Text wCacheSize;
private Label wlCompress;
- private Button wCompress;
+ private CheckBoxVar wCompress;
+
+ private Label wlCompressionLevel;
+ private CCombo wCompressionLevel;
public BlockingTransformDialog(
Shell parent,
@@ -203,7 +210,7 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
fdlCompress.right = new FormAttachment(middle, -margin);
fdlCompress.top = new FormAttachment(wCacheSize, margin);
wlCompress.setLayoutData(fdlCompress);
- wCompress = new Button(wContent, SWT.CHECK);
+ wCompress = new CheckBoxVar(variables, wContent, SWT.CHECK, "");
PropsUi.setLook(wCompress);
FormData fdCompress = new FormData();
fdCompress.left = new FormAttachment(middle, 0);
@@ -218,6 +225,31 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
}
});
+ // How hard to compress the temporary files
+ wlCompressionLevel = new Label(wContent, SWT.RIGHT);
+ wlCompressionLevel.setText(
+ BaseMessages.getString(PKG,
"BlockingTransformDialog.CompressionLevel.Label"));
+ wlCompressionLevel.setToolTipText(
+ BaseMessages.getString(PKG,
"BlockingTransformDialog.CompressionLevel.Tooltip"));
+ PropsUi.setLook(wlCompressionLevel);
+ FormData fdlCompressionLevel = new FormData();
+ fdlCompressionLevel.left = new FormAttachment(0, 0);
+ fdlCompressionLevel.right = new FormAttachment(middle, -margin);
+ fdlCompressionLevel.top = new FormAttachment(wCompress, margin);
+ wlCompressionLevel.setLayoutData(fdlCompressionLevel);
+ wCompressionLevel = new CCombo(wContent, SWT.SINGLE | SWT.READ_ONLY |
SWT.BORDER);
+ wCompressionLevel.setItems(
+
IEnumHasCodeAndDescription.getDescriptions(GzipCompressionLevel.class));
+ wCompressionLevel.setToolTipText(
+ BaseMessages.getString(PKG,
"BlockingTransformDialog.CompressionLevel.Tooltip"));
+ PropsUi.setLook(wCompressionLevel);
+ wCompressionLevel.addModifyListener(lsMod);
+ FormData fdCompressionLevel = new FormData();
+ fdCompressionLevel.left = new FormAttachment(middle, 0);
+ fdCompressionLevel.top = new FormAttachment(wCompress, margin);
+ fdCompressionLevel.right = new FormAttachment(100, 0);
+ wCompressionLevel.setLayoutData(fdCompressionLevel);
+
wContent.pack();
Rectangle bounds = wContent.getBounds();
scrolledComposite.setContent(wContent);
@@ -247,6 +279,9 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
}
wCacheSize.setText("" + input.getCacheSize());
wCompress.setSelection(input.isCompressFiles());
+ wCompress.setVariableName(input.getCompressFilesVariable());
+ wCompressionLevel.select(
+
GzipCompressionLevel.orNotStoredDefault(input.getCompressionLevel()).getIndex());
}
private void cancel() {
@@ -269,6 +304,12 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
logDetailed("Compression is set to " + wCompress.getSelection());
}
input.setCompressFiles(wCompress.getSelection());
+ input.setCompressFilesVariable(wCompress.getVariableName());
+ input.setCompressionLevel(
+ IEnumHasCodeAndDescription.lookupDescription(
+ GzipCompressionLevel.class,
+ wCompressionLevel.getText(),
+ GzipCompressionLevel.NOT_STORED_DEFAULT));
input.setPassAllRows(wPassAllRows.getSelection());
dispose();
@@ -285,5 +326,7 @@ public class BlockingTransformDialog extends
BaseTransformDialog {
wCacheSize.setEnabled(wPassAllRows.getSelection());
wlCompress.setEnabled(wPassAllRows.getSelection());
wCompress.setEnabled(wPassAllRows.getSelection());
+ wlCompressionLevel.setEnabled(wPassAllRows.getSelection());
+ wCompressionLevel.setEnabled(wPassAllRows.getSelection());
}
}
diff --git
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
index 6a2e34f101..2e55629186 100644
---
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
+++
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
@@ -19,9 +19,12 @@ package org.apache.hop.pipeline.transforms.blockingtransform;
import java.io.File;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.variables.IVariables;
@@ -63,6 +66,18 @@ public class BlockingTransformMeta
@HopMetadataProperty(key = "compress")
private boolean compressFiles;
+ /** A variable that switches compression on or off, overriding {@link
#compressFiles}. */
+ @Getter
+ @Setter
+ @HopMetadataProperty(key = "compress_variables")
+ private String compressFilesVariable;
+
+ /** How hard to compress the temporary files, from 1 (fastest) to 9
(smallest files). */
+ @Getter
+ @Setter
+ @HopMetadataProperty(key = "compression_level", storeWithCode = true)
+ private GzipCompressionLevel compressionLevel;
+
/** Pass all rows, or only the last one. Only the last row was the original
behaviour. */
@HopMetadataProperty(key = "pass_all_rows")
private boolean passAllRows;
@@ -76,6 +91,17 @@ public class BlockingTransformMeta
prefix = "block";
cacheSize = CACHE_SIZE;
compressFiles = true;
+ compressFilesVariable = null;
+ // No level here: a transform loaded without one keeps null and runs at
+ // GzipCompressionLevel.NOT_STORED_DEFAULT, the level it always used.
+ compressionLevel = null;
+ }
+
+ /** Called for a newly created transform only, not when a saved one is
loaded. */
+ @Override
+ public void setDefault() {
+ super.setDefault();
+ compressionLevel = GzipCompressionLevel.NEW_TRANSFORM_DEFAULT;
}
@Override
diff --git
a/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
b/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
index c8426dfb53..59fe235b48 100644
---
a/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
+++
b/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
@@ -25,6 +25,8 @@ BlockingTransform.Log.Openfiles=Opening tmp-file ...
BlockingTransform.Name=Blocking transform
BlockingTransformDialog.CacheSize.Label=Cache size (rows in memory)
BlockingTransformDialog.Compress.Label=Compress spool files?
+BlockingTransformDialog.CompressionLevel.Label=Compression level
+BlockingTransformDialog.CompressionLevel.Tooltip=How hard to compress the
spool files when compression is on. 1 is the fastest, 9 gives the smallest
files. Spool files are read back only once, so a low level is usually best.
BlockingTransformDialog.PassAllRows.Label=Pass all rows?
BlockingTransformDialog.Prefix.Label=Spool-file prefix
BlockingTransformDialog.Shell.Title=Blocking transform
diff --git
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
index 9f14f2448f..68614959d7 100644
---
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
+++
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
@@ -18,7 +18,9 @@
package org.apache.hop.pipeline.transforms.blockingtransform;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.Test;
@@ -33,6 +35,8 @@ class BlockingTransformMetaTest {
meta.setCacheSize(123456);
meta.setPassAllRows(false);
meta.setPrefix("blocking");
+ meta.setCompressFilesVariable("COMPRESS_SPOOL_FILES");
+ meta.setCompressionLevel(GzipCompressionLevel.SMALLEST);
String xml = meta.getXml();
@@ -49,5 +53,33 @@ class BlockingTransformMetaTest {
assertEquals(meta.getCacheSize(), meta2.getCacheSize());
assertEquals(meta.isPassAllRows(), meta2.isPassAllRows());
assertEquals(meta.getPrefix(), meta2.getPrefix());
+ assertEquals("COMPRESS_SPOOL_FILES", meta2.getCompressFilesVariable());
+ assertEquals(GzipCompressionLevel.SMALLEST, meta2.getCompressionLevel());
+ }
+
+ @Test
+ void testNewTransformUsesTheFastestLevel() {
+ // Hop GUI calls setDefault() on a transform it creates, never on one it
loads
+ BlockingTransformMeta meta = new BlockingTransformMeta();
+ meta.setDefault();
+ assertEquals(GzipCompressionLevel.FASTEST, meta.getCompressionLevel());
+ }
+
+ @Test
+ void testOlderXmlWithoutLevelKeepsLevelSix() throws Exception {
+ // A transform saved before the compression level option: no level, no
variable
+ String transformXml =
+ XmlHandler.openTag(TransformMeta.XML_TAG)
+ + "<compress>Y</compress><pass_all_rows>Y</pass_all_rows>"
+ + XmlHandler.closeTag(TransformMeta.XML_TAG);
+
+ BlockingTransformMeta meta = new BlockingTransformMeta();
+ meta.loadXml(XmlHandler.loadXmlString(transformXml,
TransformMeta.XML_TAG), null);
+
+ assertNull(meta.getCompressFilesVariable());
+ assertNull(meta.getCompressionLevel());
+ assertEquals(
+ GzipCompressionLevel.STANDARD,
+ GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel()));
}
}
diff --git
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
new file mode 100644
index 0000000000..4c34675dc4
--- /dev/null
+++
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.blockingtransform;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class BlockingTransformTest {
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<BlockingTransformMeta, BlockingTransformData>
helper;
+
+ private BlockingTransformData lastData;
+
+ @BeforeEach
+ void setUp() {
+ helper =
+ new TransformMockHelper<>(
+ "Blocking transform", BlockingTransformMeta.class,
BlockingTransformData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ @Test
+ void passesAllRowsInOrderFromMemory() throws Exception {
+ List<Object[]> output = run(true, 10_000, true, 2_500);
+
+ assertRowsInOrder(2_500, output);
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void passesAllRowsInOrderFromCompressedTempFiles() throws Exception {
+ // 10 full caches and a partial one: the last 50 rows are written to a
file of their own
+ List<Object[]> output = run(true, 100, true, 1_050);
+
+ assertRowsInOrder(1_050, output);
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void passesAllRowsInOrderFromUncompressedTempFiles() throws Exception {
+ List<Object[]> output = run(true, 100, false, 1_050);
+
+ assertRowsInOrder(1_050, output);
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void passesNothingWithoutInput() throws Exception {
+ assertTrue(run(true, 100, true, 0).isEmpty());
+ }
+
+ @Test
+ void passesOnlyTheLastRow() throws Exception {
+ List<Object[]> output = run(false, 100, true, 1_050);
+
+ assertEquals(1, output.size());
+ assertArrayEquals(row(1_049), output.get(0));
+ assertNoTempFilesLeft();
+ }
+
+ @ParameterizedTest
+ @EnumSource(GzipCompressionLevel.class)
+ void passesAllRowsInOrderAtEveryCompressionLevel(GzipCompressionLevel level)
throws Exception {
+ List<Object[]> output =
+ run(true, 100, true, 1_050, meta -> meta.setCompressionLevel(level),
transform -> {});
+
+ assertRowsInOrder(1_050, output);
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void aVariableSwitchesCompressionOn() throws Exception {
+ List<Object[]> output =
+ run(
+ true,
+ 100,
+ false,
+ 1_050,
+ meta -> meta.setCompressFilesVariable("COMPRESS_SPOOL_FILES"),
+ transform -> transform.setVariable("COMPRESS_SPOOL_FILES", "Y"));
+
+ assertTrue(lastData.compressFiles);
+ assertRowsInOrder(1_050, output);
+ assertNoTempFilesLeft();
+ }
+
+ private List<Object[]> run(boolean passAllRows, int cacheSize, boolean
compress, int rowCount)
+ throws Exception {
+ return run(passAllRows, cacheSize, compress, rowCount, meta -> {},
transform -> {});
+ }
+
+ private List<Object[]> run(
+ boolean passAllRows,
+ int cacheSize,
+ boolean compress,
+ int rowCount,
+ Consumer<BlockingTransformMeta> metaSetup,
+ Consumer<BlockingTransform> beforeInit)
+ throws Exception {
+ BlockingTransformMeta meta = new BlockingTransformMeta();
+ meta.setPassAllRows(passAllRows);
+ meta.setCacheSize(cacheSize);
+ meta.setCompressFiles(compress);
+ meta.setDirectory(tempDir.toString());
+ meta.setPrefix("block");
+ metaSetup.accept(meta);
+ lastData = new BlockingTransformData();
+
+ BlockingTransform transform =
+ new BlockingTransform(
+ helper.transformMeta, meta, lastData, 0, helper.pipelineMeta,
helper.pipeline);
+ beforeInit.accept(transform);
+ transform.init();
+
+ List<Object[]> input = new ArrayList<>();
+ for (int i = 0; i < rowCount; i++) {
+ input.add(row(i));
+ }
+ IRowSet rowSet = helper.getMockInputRowSet(input);
+ // BaseTransform also reads with getRowImmediate() (every 500 rows): same
rows
+ when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ when(rowSet.getRowMeta()).thenReturn(rowMeta);
+ transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+ List<Object[]> output = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ output.add(row);
+ }
+ });
+
+ while (transform.processRow()) {
+ // keep going until the transform is done
+ }
+ transform.dispose();
+ return output;
+ }
+
+ private static Object[] row(int i) {
+ return new Object[] {(long) i, "row " + i};
+ }
+
+ private static void assertRowsInOrder(int rowCount, List<Object[]> output) {
+ assertEquals(rowCount, output.size());
+ for (int i = 0; i < rowCount; i++) {
+ Object[] expected = row(i);
+ Object[] actual = output.get(i);
+ assertEquals(expected[0], actual[0], "row " + i);
+ assertEquals(expected[1], actual[1], "row " + i);
+ }
+ }
+
+ private void assertNoTempFilesLeft() throws Exception {
+ try (Stream<Path> left = Files.list(tempDir)) {
+ assertEquals(0, left.count(), "temp files left behind");
+ }
+ }
+}
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
index 035f776ec4..4f6ee06f17 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
@@ -35,6 +35,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.exception.HopValueException;
@@ -136,8 +137,11 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
OutputStream outputStream = HopVfs.getOutputStream(fileObject, false);
CountingOutputStream counting = new CountingOutputStream(outputStream);
if (data.compressFiles) {
- gzos = new GZIPOutputStream(new BufferedOutputStream(counting));
- dos = new DataOutputStream(gzos);
+ gzos =
+ GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel())
+ .newOutputStream(new BufferedOutputStream(counting));
+ // Buffer above the GZIP stream too, so values are compressed in
blocks, not one by one.
+ dos = new DataOutputStream(new BufferedOutputStream(gzos, 500000));
} else {
dos = new DataOutputStream(new BufferedOutputStream(counting, 500000));
gzos = null;
@@ -222,7 +226,8 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
}
private DataInputStream getDataInputStream(GZIPInputStream gzipInputStream) {
- DataInputStream result = new DataInputStream(gzipInputStream);
+ // Buffer above the GZIP stream, so values are decompressed in blocks, not
one by one.
+ DataInputStream result = new DataInputStream(new
BufferedInputStream(gzipInputStream, 50000));
data.gzis.add(gzipInputStream);
return result;
}
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
index e720b52790..577ee446c8 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
@@ -20,11 +20,13 @@ package org.apache.hop.pipeline.transforms.sort;
import java.util.ArrayList;
import java.util.List;
import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.ui.core.ConstUi;
@@ -40,6 +42,7 @@ import
org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.apache.hop.ui.pipeline.transform.ComponentSelectionListener;
import org.apache.hop.ui.pipeline.transform.ITableItemInsertListener;
import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.events.ModifyListener;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
@@ -69,6 +72,8 @@ public class SortRowsDialog extends BaseTransformDialog {
private CheckBoxVar wCompress;
+ private CCombo wCompressionLevel;
+
private Button wUniqueRows;
private TableView wFields;
@@ -209,14 +214,39 @@ public class SortRowsDialog extends BaseTransformDialog {
}
});
- // Using compression for temporary files?
+ // How hard to compress the temporary files
+ Label wlCompressionLevel = new Label(shell, SWT.RIGHT);
+ wlCompressionLevel.setText(
+ BaseMessages.getString(PKG, "SortRowsDialog.CompressionLevel.Label"));
+ wlCompressionLevel.setToolTipText(
+ BaseMessages.getString(PKG,
"SortRowsDialog.CompressionLevel.Tooltip"));
+ PropsUi.setLook(wlCompressionLevel);
+ FormData fdlCompressionLevel = new FormData();
+ fdlCompressionLevel.left = new FormAttachment(0, 0);
+ fdlCompressionLevel.right = new FormAttachment(middle, -margin);
+ fdlCompressionLevel.top = new FormAttachment(wCompress, margin);
+ wlCompressionLevel.setLayoutData(fdlCompressionLevel);
+ wCompressionLevel = new CCombo(shell, SWT.SINGLE | SWT.READ_ONLY |
SWT.BORDER);
+ wCompressionLevel.setItems(
+
IEnumHasCodeAndDescription.getDescriptions(GzipCompressionLevel.class));
+ wCompressionLevel.setToolTipText(
+ BaseMessages.getString(PKG,
"SortRowsDialog.CompressionLevel.Tooltip"));
+ PropsUi.setLook(wCompressionLevel);
+ wCompressionLevel.addModifyListener(lsMod);
+ FormData fdCompressionLevel = new FormData();
+ fdCompressionLevel.left = new FormAttachment(middle, 0);
+ fdCompressionLevel.top = new FormAttachment(wCompress, margin);
+ fdCompressionLevel.right = new FormAttachment(100, 0);
+ wCompressionLevel.setLayoutData(fdCompressionLevel);
+
+ // Only pass unique rows?
Label wlUniqueRows = new Label(shell, SWT.RIGHT);
wlUniqueRows.setText(BaseMessages.getString(PKG,
"SortRowsDialog.UniqueRows.Label"));
PropsUi.setLook(wlUniqueRows);
FormData fdlUniqueRows = new FormData();
fdlUniqueRows.left = new FormAttachment(0, 0);
fdlUniqueRows.right = new FormAttachment(middle, -margin);
- fdlUniqueRows.top = new FormAttachment(wCompress, margin);
+ fdlUniqueRows.top = new FormAttachment(wCompressionLevel, margin);
wlUniqueRows.setLayoutData(fdlUniqueRows);
wUniqueRows = new Button(shell, SWT.CHECK);
wUniqueRows.setToolTipText(BaseMessages.getString(PKG,
"SortRowsDialog.UniqueRows.Tooltip"));
@@ -361,6 +391,8 @@ public class SortRowsDialog extends BaseTransformDialog {
wFreeMemory.setText(Const.NVL(input.getFreeMemoryLimit(), ""));
wCompress.setSelection(input.isCompressFiles());
wCompress.setVariableName(input.getCompressFilesVariable());
+ wCompressionLevel.select(
+
GzipCompressionLevel.orNotStoredDefault(input.getCompressionLevel()).getIndex());
wUniqueRows.setSelection(input.isOnlyPassingUniqueRows());
Table table = wFields.table;
@@ -424,6 +456,11 @@ public class SortRowsDialog extends BaseTransformDialog {
log.logDetailed("Sort rows", "Compression is set to " +
wCompress.getSelection());
input.setCompressFiles(wCompress.getSelection());
input.setCompressFilesVariable(wCompress.getVariableName());
+ input.setCompressionLevel(
+ IEnumHasCodeAndDescription.lookupDescription(
+ GzipCompressionLevel.class,
+ wCompressionLevel.getText(),
+ GzipCompressionLevel.NOT_STORED_DEFAULT));
input.setOnlyPassingUniqueRows(wUniqueRows.getSelection());
int nrFields = wFields.nrNonEmpty();
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
index 43bad10458..53163a4a98 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
@@ -30,6 +30,7 @@ import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
@@ -96,6 +97,13 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
@HopMetadataProperty(key = "compress_variables", injectionKey =
"COMPRESS_VARIABLE")
private String compressFilesVariable;
+ /** How hard to compress the temporary files, from 1 (fastest) to 9
(smallest files). */
+ @HopMetadataProperty(
+ key = "compression_level",
+ storeWithCode = true,
+ injectionKey = "COMPRESSION_LEVEL")
+ private GzipCompressionLevel compressionLevel;
+
public SortRowsMeta() {
super();
sortFields = new ArrayList<>();
@@ -105,9 +113,19 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
freeMemoryLimit = null;
compressFiles = false;
compressFilesVariable = null;
+ // No level here: a transform loaded without one keeps null and runs at
+ // GzipCompressionLevel.NOT_STORED_DEFAULT, the level it always used.
+ compressionLevel = null;
onlyPassingUniqueRows = false;
}
+ /** Called for a newly created transform only, not when a saved one is
loaded. */
+ @Override
+ public void setDefault() {
+ super.setDefault();
+ compressionLevel = GzipCompressionLevel.NEW_TRANSFORM_DEFAULT;
+ }
+
/**
* Returns the collation strength based on the passed in locale. Package
protected for testing
* purposes
diff --git
a/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
b/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
index b9b5383359..8885ef4694 100644
---
a/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
+++
b/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
@@ -31,6 +31,7 @@ SortRows.Error.UnableToCloseFile=Unable to close/delete file
#{0} --> "{1}
SortRows.Injection.COLLATOR_ENABLED=Enable this option to use collator support.
SortRows.Injection.COLLATOR_STRENGTH=Specify the collator strength when
collator support is enabled (0-3).
SortRows.Injection.COMPRESS_TEMP_FILES=Enable this option to compress
temporary files.
+SortRows.Injection.COMPRESSION_LEVEL=How hard to compress the temporary files:
FASTEST, VERY_FAST, FAST, BALANCED_FASTER, BALANCED, STANDARD, SMALLER,
VERY_SMALL or SMALLEST.
SortRows.Injection.FIELDS=The fields to sort.
SortRows.Injection.FREE_MEMORY_TRESHOLD=The percentage of free memory to allow
until writing to temporary files.
SortRows.Injection.IGNORE_CASE=Enable this option to ignore case when sorting
the field.
@@ -50,6 +51,8 @@ SortRowsDialog.CaseInsensitive.Column=Case sensitive compare
SortRowsDialog.CollatorDisabled.Column=Sort based on current locale
SortRowsDialog.CollatorStrength.Column=Collator Strength
SortRowsDialog.Compress.Label=Compress TMP files
+SortRowsDialog.CompressionLevel.Label=Compression level
+SortRowsDialog.CompressionLevel.Tooltip=How hard to compress the temporary
files when compression is on. 1 is the fastest, 9 gives the smallest files.
Temporary files are read back only once, so a low level is usually best.
SortRowsDialog.DialogTitle=Sort rows
SortRowsDialog.Fieldname.Column=Fieldname
SortRowsDialog.Fields.Label=Fields:
diff --git
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
index 72ef2906e3..15a86d4785 100644
---
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
+++
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.sort;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.injection.BaseMetadataInjectionTestJunit5;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.junit.jupiter.api.BeforeEach;
@@ -53,6 +54,7 @@ class SortRowsMetaInjectionTest extends
BaseMetadataInjectionTestJunit5<SortRows
check("COLLATOR_ENABLED", () ->
meta.getSortFields().get(0).isCollatorEnabled());
check("COLLATOR_STRENGTH", () ->
meta.getSortFields().get(0).getCollatorStrength());
check("COMPRESS_VARIABLE", () -> meta.getCompressFilesVariable());
+ check("COMPRESSION_LEVEL", () -> meta.getCompressionLevel(),
GzipCompressionLevel.class);
// check("NAME", () -> meta.getFieldName()[0]);
// check("SORT_ASCENDING", () -> meta.getAscending()[0]);
// check("IGNORE_CASE", () -> meta.getCaseSensitive()[0]);
diff --git
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
index 3f4bec42ef..852f73bc78 100644
---
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
+++
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.sort;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.text.Collator;
@@ -29,10 +30,14 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
import
org.apache.hop.pipeline.transforms.loadsave.validator.BooleanLoadSaveValidator;
+import
org.apache.hop.pipeline.transforms.loadsave.validator.EnumLoadSaveValidator;
import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
import
org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator;
import
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
@@ -47,6 +52,31 @@ class SortRowsMetaTest {
/**
* @throws HopException
*/
+ @Test
+ void testNewTransformUsesTheFastestLevel() {
+ // Hop GUI calls setDefault() on a transform it creates, never on one it
loads
+ SortRowsMeta meta = new SortRowsMeta();
+ meta.setDefault();
+ assertEquals(GzipCompressionLevel.FASTEST, meta.getCompressionLevel());
+ }
+
+ @Test
+ void testOlderXmlWithoutLevelKeepsLevelSix() throws Exception {
+ // A transform saved before the compression level option
+ String transformXml =
+ XmlHandler.openTag(TransformMeta.XML_TAG)
+ + "<compress>Y</compress><sort_size>1000</sort_size>"
+ + XmlHandler.closeTag(TransformMeta.XML_TAG);
+
+ SortRowsMeta meta = new SortRowsMeta();
+ meta.loadXml(XmlHandler.loadXmlString(transformXml,
TransformMeta.XML_TAG), null);
+
+ assertNull(meta.getCompressionLevel());
+ assertEquals(
+ GzipCompressionLevel.STANDARD,
+ GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel()));
+ }
+
@Test
void testRoundTrips() throws Exception {
List<String> attributes =
@@ -57,6 +87,7 @@ class SortRowsMetaTest {
"FreeMemoryLimit",
"CompressFiles",
"CompressFilesVariable",
+ "CompressionLevel",
"OnlyPassingUniqueRows",
"SortFields");
@@ -74,6 +105,8 @@ class SortRowsMetaTest {
fieldLoadSaveValidatorAttributeMap.put("FreeMemoryLimit",
stringFieldLoadSaveValidator);
fieldLoadSaveValidatorAttributeMap.put("CompressFiles",
booleanFieldLoadSaveValidator);
fieldLoadSaveValidatorAttributeMap.put("CompressFilesVariable",
stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put(
+ "CompressionLevel", new
EnumLoadSaveValidator<>(GzipCompressionLevel.SMALLEST));
fieldLoadSaveValidatorAttributeMap.put("OnlyPassingUniqueRows",
booleanFieldLoadSaveValidator);
LoadSaveTester<SortRowsMeta> loadSaveTester =
diff --git
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
new file mode 100644
index 0000000000..30b91f7530
--- /dev/null
+++
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.sort;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class SortRowsTest {
+
+ private static final int ROWS = 1_050;
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<SortRowsMeta, SortRowsData> helper;
+
+ @BeforeEach
+ void setUp() {
+ helper = new TransformMockHelper<>("Sort rows", SortRowsMeta.class,
SortRowsData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ @Test
+ void sortsInMemory() throws Exception {
+ assertSorted(run("10000", true));
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void sortsThroughCompressedTempFiles() throws Exception {
+ // 10 full sort buffers and a partial one: 11 temp files to merge
+ assertSorted(run("100", true));
+ assertNoTempFilesLeft();
+ }
+
+ @Test
+ void sortsThroughUncompressedTempFiles() throws Exception {
+ assertSorted(run("100", false));
+ assertNoTempFilesLeft();
+ }
+
+ private List<Object[]> run(String sortSize, boolean compress) throws
Exception {
+ SortRowsMeta meta = new SortRowsMeta();
+ meta.setDefault();
+ meta.setSortSize(sortSize);
+ meta.setFreeMemoryLimit(null);
+ meta.setCompressFiles(compress);
+ meta.setDirectory(tempDir.toString());
+ meta.setPrefix("sort");
+ meta.getSortFields().clear();
+ meta.getSortFields().add(new SortRowsField("id", true, true, false, 0,
false));
+
+ SortRows transform =
+ new SortRows(
+ helper.transformMeta,
+ meta,
+ new SortRowsData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ transform.init();
+
+ // Every id from 0 to ROWS - 1 once, in a scrambled order
+ List<Object[]> input = new ArrayList<>();
+ for (int i = 0; i < ROWS; i++) {
+ long id = (i * 7919L) % ROWS;
+ input.add(new Object[] {id, "row " + id});
+ }
+ IRowSet rowSet = helper.getMockInputRowSet(input);
+ // BaseTransform also reads with getRowImmediate() (every 500 rows): same
rows
+ when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ when(rowSet.getRowMeta()).thenReturn(rowMeta);
+ transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+ List<Object[]> output = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ output.add(row);
+ }
+ });
+
+ while (transform.processRow()) {
+ // keep going until the transform is done
+ }
+ transform.dispose();
+ return output;
+ }
+
+ private static void assertSorted(List<Object[]> output) {
+ assertEquals(ROWS, output.size());
+ for (int i = 0; i < ROWS; i++) {
+ assertEquals((long) i, output.get(i)[0], "row " + i);
+ assertEquals("row " + i, output.get(i)[1], "row " + i);
+ }
+ }
+
+ private void assertNoTempFilesLeft() throws Exception {
+ try (Stream<Path> left = Files.list(tempDir)) {
+ assertEquals(0, left.count(), "temp files left behind");
+ }
+ }
+}