This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new c3abb154be Blocking transform is slow when "Pass all rows" is on, 
fixes #9344 (#9348)
c3abb154be is described below

commit c3abb154bee933658807e00e04a9106efcce3a1c
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:59:19 2026 +0100

    Blocking transform is slow when "Pass all rows" is on, fixes #9344 (#9348)
---
 .../pipeline/transforms/blockingtransform.adoc     |   6 +-
 .../ROOT/pages/pipeline/transforms/sort.adoc       |   4 +
 .../core/compress/gzip/GzipCompressionLevel.java   | 100 ++++++++++
 .../gzip/messages/messages_en_US.properties        |  26 +++
 .../compress/gzip/GzipCompressionLevelTest.java    | 105 ++++++++++
 .../blockingtransform/BlockingTransform.java       |  43 +++--
 .../blockingtransform/BlockingTransformData.java   |   7 +
 .../blockingtransform/BlockingTransformDialog.java |  47 ++++-
 .../blockingtransform/BlockingTransformMeta.java   |  26 +++
 .../messages/messages_en_US.properties             |   2 +
 .../BlockingTransformMetaTest.java                 |  32 ++++
 .../blockingtransform/BlockingTransformTest.java   | 212 +++++++++++++++++++++
 .../hop/pipeline/transforms/sort/SortRows.java     |  11 +-
 .../pipeline/transforms/sort/SortRowsDialog.java   |  41 +++-
 .../hop/pipeline/transforms/sort/SortRowsMeta.java |  18 ++
 .../sort/messages/messages_en_US.properties        |   3 +
 .../transforms/sort/SortRowsMetaInjectionTest.java |   2 +
 .../pipeline/transforms/sort/SortRowsMetaTest.java |  33 ++++
 .../hop/pipeline/transforms/sort/SortRowsTest.java | 147 ++++++++++++++
 19 files changed, 845 insertions(+), 20 deletions(-)

diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
index 68d8817c49..f29d490f0b 100644
--- 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
+++ 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/blockingtransform.adoc
@@ -40,5 +40,9 @@ Use the Blocking transform for triggering plugins, stored 
procedures, Java scrip
 |Spool directory|The directory in which the temporary files are stored if 
needed; the default is the standard temporary directory for the system
 |Spool-file prefix|Choose a recognizable prefix to identify the files when 
they appear in the temp directory
 |Cache size|The more rows you can store in memory, the faster the transform 
works
-|Compress spool files?|Compresses temporary files when they are needed
+|Compress spool files?|Compresses temporary files when they are needed.
+You can also give a variable: when it is set, its value (Y or N) decides 
whether the files are compressed.
+|Compression level|How hard to compress the spool files: 1 - Fastest, 2 - Very 
fast, 3 - Fast, 4 - Balanced, faster, 5 - Balanced, 6 - Standard (the Java 
default), 7 - Smaller files, 8 - Very small files, 9 - Smallest files.
+Spool files are read back only once, so a low level is usually best.
+A new transform uses 1 - Fastest. A transform saved before this option existed 
keeps 6 - Standard, the level it always used.
 |===
\ No newline at end of file
diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
index 0e9ab79147..f01d182bfb 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sort.adoc
@@ -52,6 +52,10 @@ The native Spark engine does support this transform: it is 
mapped onto a Spark s
 |Sort size|The more rows you store in memory, the faster the sorting process 
because fewer temporary files must be used and less I/O is generated.
 |Free memory threshold (in %)|If the sort algorithm finds that it has less 
available free memory than the indicated number, it will start to page data to 
disk.
 |Compress TMP Files|Compresses temporary files when they are needed to 
complete the sort.
+You can also give a variable: when it is set, its value (Y or N) decides 
whether the files are compressed.
+|Compression level|How hard to compress the temporary files: 1 - Fastest, 2 - 
Very fast, 3 - Fast, 4 - Balanced, faster, 5 - Balanced, 6 - Standard (the Java 
default), 7 - Smaller files, 8 - Very small files, 9 - Smallest files.
+Temporary files are read back only once, so a low level is usually best.
+A new transform uses 1 - Fastest. A transform saved before this option existed 
keeps 6 - Standard, the level it always used.
 |Only pass unique rows?|Enable if you want to pass unique rows only to the 
output stream(s).
 |Fields table|Specify the fields and direction (ascending/descending) to sort.
 You can specify whether to perform a case sensitive sort (optional)
diff --git 
a/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
 
b/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
new file mode 100644
index 0000000000..77ce538cdd
--- /dev/null
+++ 
b/engine/src/main/java/org/apache/hop/core/compress/gzip/GzipCompressionLevel.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.compress.gzip;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+
+/**
+ * The GZIP compression levels 1 to 9: lower is faster, higher gives smaller 
files. Used for the
+ * temporary files of transforms such as Sort rows and Blocking transform.
+ */
+public enum GzipCompressionLevel implements IEnumHasCodeAndDescription {
+  FASTEST(1),
+  VERY_FAST(2),
+  FAST(3),
+  BALANCED_FASTER(4),
+  BALANCED(5),
+  STANDARD(6),
+  SMALLER(7),
+  VERY_SMALL(8),
+  SMALLEST(9);
+
+  private static final Class<?> PKG = GzipCompressionLevel.class;
+
+  /**
+   * The level of a newly created transform: temporary files are read back 
only once, so speed
+   * matters most.
+   */
+  public static final GzipCompressionLevel NEW_TRANSFORM_DEFAULT = FASTEST;
+
+  /**
+   * The level of a transform saved before the level could be chosen: the 
level those transforms
+   * always used, so they keep working the same way.
+   */
+  public static final GzipCompressionLevel NOT_STORED_DEFAULT = STANDARD;
+
+  private final int level;
+
+  GzipCompressionLevel(int level) {
+    this.level = level;
+  }
+
+  /** The level passed to {@link java.util.zip.Deflater}, from 1 to 9. */
+  public int getLevel() {
+    return level;
+  }
+
+  /** The constant name: this is what transforms store in their metadata. */
+  @Override
+  public String getCode() {
+    return name();
+  }
+
+  /** The level number and its name, for example "1 - Fastest". */
+  @Override
+  public String getDescription() {
+    return level + " - " + BaseMessages.getString(PKG, "GzipCompressionLevel." 
+ name());
+  }
+
+  /**
+   * @param level a stored level, or null when none was stored (metadata from 
before the option)
+   * @return the level, or {@link #NOT_STORED_DEFAULT} when it is null
+   */
+  public static GzipCompressionLevel orNotStoredDefault(GzipCompressionLevel 
level) {
+    return level == null ? NOT_STORED_DEFAULT : level;
+  }
+
+  /**
+   * Opens a GZIP stream that compresses at this level.
+   *
+   * @param out the stream to write the compressed data to
+   * @return the GZIP stream
+   * @throws IOException when the GZIP header cannot be written
+   */
+  public GZIPOutputStream newOutputStream(OutputStream out) throws IOException 
{
+    return new GZIPOutputStream(out) {
+      {
+        def.setLevel(level);
+      }
+    };
+  }
+}
diff --git 
a/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
 
b/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
new file mode 100644
index 0000000000..4ffc9db7be
--- /dev/null
+++ 
b/engine/src/main/resources/org/apache/hop/core/compress/gzip/messages/messages_en_US.properties
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+GzipCompressionLevel.FASTEST=Fastest
+GzipCompressionLevel.VERY_FAST=Very fast
+GzipCompressionLevel.FAST=Fast
+GzipCompressionLevel.BALANCED_FASTER=Balanced, faster
+GzipCompressionLevel.BALANCED=Balanced
+GzipCompressionLevel.STANDARD=Standard (the Java default)
+GzipCompressionLevel.SMALLER=Smaller files
+GzipCompressionLevel.VERY_SMALL=Very small files
+GzipCompressionLevel.SMALLEST=Smallest files
diff --git 
a/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
 
b/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
new file mode 100644
index 0000000000..5fa8b6bd6b
--- /dev/null
+++ 
b/engine/src/test/java/org/apache/hop/core/compress/gzip/GzipCompressionLevelTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.compress.gzip;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class GzipCompressionLevelTest {
+
+  @Test
+  void levelsRunFromOneToNine() {
+    GzipCompressionLevel[] levels = GzipCompressionLevel.values();
+    assertEquals(9, levels.length);
+    for (int i = 0; i < levels.length; i++) {
+      assertEquals(i + 1, levels[i].getLevel());
+    }
+  }
+
+  @Test
+  void descriptionsShowTheNumberAndAName() {
+    assertEquals("1 - Fastest", GzipCompressionLevel.FASTEST.getDescription());
+    assertEquals("6 - Standard (the Java default)", 
GzipCompressionLevel.STANDARD.getDescription());
+    assertEquals("9 - Smallest files", 
GzipCompressionLevel.SMALLEST.getDescription());
+    for (GzipCompressionLevel level : GzipCompressionLevel.values()) {
+      assertEquals(
+          level,
+          IEnumHasCodeAndDescription.lookupDescription(
+              GzipCompressionLevel.class, level.getDescription(), null));
+      assertEquals(level.name(), level.getCode());
+    }
+  }
+
+  @Test
+  void noStoredLevelMeansTheLevelUsedBefore() {
+    assertEquals(GzipCompressionLevel.STANDARD, 
GzipCompressionLevel.orNotStoredDefault(null));
+    assertEquals(6, GzipCompressionLevel.orNotStoredDefault(null).getLevel());
+    assertEquals(
+        GzipCompressionLevel.SMALLEST,
+        
GzipCompressionLevel.orNotStoredDefault(GzipCompressionLevel.SMALLEST));
+  }
+
+  @ParameterizedTest
+  @EnumSource(GzipCompressionLevel.class)
+  void compressedDataReadsBack(GzipCompressionLevel level) throws Exception {
+    byte[] data = sampleData();
+    assertArrayEquals(data, decompress(compress(level, data)));
+  }
+
+  @Test
+  void theLevelIsApplied() throws Exception {
+    byte[] data = sampleData();
+    int fastest = compress(GzipCompressionLevel.FASTEST, data).length;
+    int smallest = compress(GzipCompressionLevel.SMALLEST, data).length;
+    assertTrue(smallest < fastest, "level 9: " + smallest + " bytes, level 1: 
" + fastest);
+  }
+
+  private static byte[] sampleData() {
+    StringBuilder text = new StringBuilder();
+    for (int i = 0; i < 20_000; i++) {
+      text.append("order ").append(i).append(", customer ").append(i % 
977).append('\n');
+    }
+    return text.toString().getBytes(StandardCharsets.UTF_8);
+  }
+
+  private static byte[] compress(GzipCompressionLevel level, byte[] data) 
throws IOException {
+    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+    try (GZIPOutputStream out = level.newOutputStream(bytes)) {
+      out.write(data);
+    }
+    return bytes.toByteArray();
+  }
+
+  private static byte[] decompress(byte[] data) throws IOException {
+    try (GZIPInputStream in = new GZIPInputStream(new 
ByteArrayInputStream(data))) {
+      return in.readAllBytes();
+    }
+  }
+}
diff --git 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
index d9320b37e8..d74f6ed592 100644
--- 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
+++ 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransform.java
@@ -30,6 +30,7 @@ import java.util.zip.GZIPOutputStream;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.exception.HopFileException;
 import org.apache.hop.core.io.CountingInputStream;
@@ -50,6 +51,12 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
 
   private static final Class<?> PKG = BlockingTransformMeta.class;
 
+  /**
+   * Buffer between the row data and the (compressed) temp file, so values are 
written and read in
+   * blocks rather than one at a time.
+   */
+  private static final int TEMP_FILE_BUFFER_SIZE = 65536;
+
   private Object[] lastRow;
 
   public BlockingTransform(
@@ -85,11 +92,13 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
         data.files.add(fileObject); // Remember the files!
         OutputStream outputStream = HopVfs.getOutputStream(fileObject, false);
         CountingOutputStream counting = new CountingOutputStream(outputStream);
-        if (meta.isCompressFiles()) {
-          gzos = new GZIPOutputStream(new BufferedOutputStream(counting));
-          dos = new DataOutputStream(gzos);
+        if (data.compressFiles) {
+          gzos =
+              
GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel())
+                  .newOutputStream(new BufferedOutputStream(counting));
+          dos = new DataOutputStream(new BufferedOutputStream(gzos, 
TEMP_FILE_BUFFER_SIZE));
         } else {
-          dos = new DataOutputStream(counting);
+          dos = new DataOutputStream(new BufferedOutputStream(counting, 
TEMP_FILE_BUFFER_SIZE));
           gzos = null;
         }
 
@@ -141,13 +150,16 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
         data.countingInputStreams.add(countingFi);
         data.fis.add(countingFi);
         DataInputStream di;
-        if (meta.isCompressFiles()) {
+        if (data.compressFiles) {
           GZIPInputStream gzfi =
               new GZIPInputStream(new 
BufferedInputStream(data.fis.get(data.fis.size() - 1)));
-          di = new DataInputStream(gzfi);
+          di = new DataInputStream(new BufferedInputStream(gzfi, 
TEMP_FILE_BUFFER_SIZE));
           data.gzis.add(gzfi);
         } else {
-          di = new DataInputStream(data.fis.get(data.fis.size() - 1));
+          di =
+              new DataInputStream(
+                  new BufferedInputStream(
+                      data.fis.get(data.fis.size() - 1), 
TEMP_FILE_BUFFER_SIZE));
         }
         data.dis.add(di);
 
@@ -176,10 +188,15 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
     }
 
     if (data.files.isEmpty()) {
-      if (!data.buffer.isEmpty()) {
-        retval = data.buffer.get(0);
-        data.buffer.remove(0);
+      // Walk the buffer in order instead of removing its first row, which 
moves all the others.
+      // Each slot is cleared once passed on, so the memory is still freed as 
the rows leave.
+      if (data.bufferIndex < data.buffer.size()) {
+        retval = data.buffer.get(data.bufferIndex);
+        data.buffer.set(data.bufferIndex, null);
+        data.bufferIndex++;
       } else {
+        data.buffer.clear();
+        data.bufferIndex = 0;
         retval = null;
       }
     } else {
@@ -194,7 +211,7 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
         FileObject file = data.files.get(0);
         DataInputStream di = data.dis.get(0);
         InputStream fi = data.fis.get(0);
-        GZIPInputStream gzfi = (meta.isCompressFiles()) ? data.gzis.get(0) : 
null;
+        GZIPInputStream gzfi = (data.compressFiles) ? data.gzis.get(0) : null;
 
         try {
           data.rowbuffer.add(0, data.outputRowMeta.readData(di));
@@ -267,7 +284,9 @@ public class BlockingTransform extends 
BaseTransform<BlockingTransformMeta, Bloc
   public boolean init() {
 
     if (super.init()) {
-      // Add init code here.
+      // Compress the temporary files? A variable, when given, decides.
+      data.compressFiles =
+          getVariableBoolean(meta.getCompressFilesVariable(), 
meta.isCompressFiles());
       return true;
     }
     return false;
diff --git 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
index c6f797d81f..d84170cb32 100644
--- 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
+++ 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformData.java
@@ -32,12 +32,19 @@ import org.apache.hop.pipeline.transform.ITransformData;
 public class BlockingTransformData extends BaseTransformData implements 
ITransformData {
   public List<FileObject> files;
   public List<Object[]> buffer;
+
+  /** Position of the next row to pass on from {@link #buffer} when no rows 
went to disk. */
+  public int bufferIndex;
+
   public List<InputStream> fis;
   public List<CountingInputStream> countingInputStreams;
   public List<GZIPInputStream> gzis;
   public List<DataInputStream> dis;
   public List<Object[]> rowbuffer;
 
+  /** Whether the temporary files are compressed, resolved when the transform 
starts. */
+  public boolean compressFiles;
+
   public IRowMeta outputRowMeta;
 
   public int[] fieldnrs; // the corresponding field numbers
diff --git 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
index 913516e883..2300e7d52e 100644
--- 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
+++ 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformDialog.java
@@ -18,15 +18,19 @@
 package org.apache.hop.pipeline.transforms.blockingtransform;
 
 import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
 import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.ui.core.PropsUi;
 import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.widget.CheckBoxVar;
 import org.apache.hop.ui.core.widget.TextVar;
 import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
 import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CCombo;
 import org.eclipse.swt.custom.ScrolledComposite;
 import org.eclipse.swt.events.ModifyListener;
 import org.eclipse.swt.events.SelectionAdapter;
@@ -60,7 +64,10 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
   private Text wCacheSize;
 
   private Label wlCompress;
-  private Button wCompress;
+  private CheckBoxVar wCompress;
+
+  private Label wlCompressionLevel;
+  private CCombo wCompressionLevel;
 
   public BlockingTransformDialog(
       Shell parent,
@@ -203,7 +210,7 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
     fdlCompress.right = new FormAttachment(middle, -margin);
     fdlCompress.top = new FormAttachment(wCacheSize, margin);
     wlCompress.setLayoutData(fdlCompress);
-    wCompress = new Button(wContent, SWT.CHECK);
+    wCompress = new CheckBoxVar(variables, wContent, SWT.CHECK, "");
     PropsUi.setLook(wCompress);
     FormData fdCompress = new FormData();
     fdCompress.left = new FormAttachment(middle, 0);
@@ -218,6 +225,31 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
           }
         });
 
+    // How hard to compress the temporary files
+    wlCompressionLevel = new Label(wContent, SWT.RIGHT);
+    wlCompressionLevel.setText(
+        BaseMessages.getString(PKG, 
"BlockingTransformDialog.CompressionLevel.Label"));
+    wlCompressionLevel.setToolTipText(
+        BaseMessages.getString(PKG, 
"BlockingTransformDialog.CompressionLevel.Tooltip"));
+    PropsUi.setLook(wlCompressionLevel);
+    FormData fdlCompressionLevel = new FormData();
+    fdlCompressionLevel.left = new FormAttachment(0, 0);
+    fdlCompressionLevel.right = new FormAttachment(middle, -margin);
+    fdlCompressionLevel.top = new FormAttachment(wCompress, margin);
+    wlCompressionLevel.setLayoutData(fdlCompressionLevel);
+    wCompressionLevel = new CCombo(wContent, SWT.SINGLE | SWT.READ_ONLY | 
SWT.BORDER);
+    wCompressionLevel.setItems(
+        
IEnumHasCodeAndDescription.getDescriptions(GzipCompressionLevel.class));
+    wCompressionLevel.setToolTipText(
+        BaseMessages.getString(PKG, 
"BlockingTransformDialog.CompressionLevel.Tooltip"));
+    PropsUi.setLook(wCompressionLevel);
+    wCompressionLevel.addModifyListener(lsMod);
+    FormData fdCompressionLevel = new FormData();
+    fdCompressionLevel.left = new FormAttachment(middle, 0);
+    fdCompressionLevel.top = new FormAttachment(wCompress, margin);
+    fdCompressionLevel.right = new FormAttachment(100, 0);
+    wCompressionLevel.setLayoutData(fdCompressionLevel);
+
     wContent.pack();
     Rectangle bounds = wContent.getBounds();
     scrolledComposite.setContent(wContent);
@@ -247,6 +279,9 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
     }
     wCacheSize.setText("" + input.getCacheSize());
     wCompress.setSelection(input.isCompressFiles());
+    wCompress.setVariableName(input.getCompressFilesVariable());
+    wCompressionLevel.select(
+        
GzipCompressionLevel.orNotStoredDefault(input.getCompressionLevel()).getIndex());
   }
 
   private void cancel() {
@@ -269,6 +304,12 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
       logDetailed("Compression is set to " + wCompress.getSelection());
     }
     input.setCompressFiles(wCompress.getSelection());
+    input.setCompressFilesVariable(wCompress.getVariableName());
+    input.setCompressionLevel(
+        IEnumHasCodeAndDescription.lookupDescription(
+            GzipCompressionLevel.class,
+            wCompressionLevel.getText(),
+            GzipCompressionLevel.NOT_STORED_DEFAULT));
     input.setPassAllRows(wPassAllRows.getSelection());
 
     dispose();
@@ -285,5 +326,7 @@ public class BlockingTransformDialog extends 
BaseTransformDialog {
     wCacheSize.setEnabled(wPassAllRows.getSelection());
     wlCompress.setEnabled(wPassAllRows.getSelection());
     wCompress.setEnabled(wPassAllRows.getSelection());
+    wlCompressionLevel.setEnabled(wPassAllRows.getSelection());
+    wCompressionLevel.setEnabled(wPassAllRows.getSelection());
   }
 }
diff --git 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
index 6a2e34f101..2e55629186 100644
--- 
a/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
+++ 
b/plugins/transforms/blockingtransform/src/main/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMeta.java
@@ -19,9 +19,12 @@ package org.apache.hop.pipeline.transforms.blockingtransform;
 
 import java.io.File;
 import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.hop.core.CheckResult;
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopTransformException;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.variables.IVariables;
@@ -63,6 +66,18 @@ public class BlockingTransformMeta
   @HopMetadataProperty(key = "compress")
   private boolean compressFiles;
 
+  /** A variable that switches compression on or off, overriding {@link 
#compressFiles}. */
+  @Getter
+  @Setter
+  @HopMetadataProperty(key = "compress_variables")
+  private String compressFilesVariable;
+
+  /** How hard to compress the temporary files, from 1 (fastest) to 9 
(smallest files). */
+  @Getter
+  @Setter
+  @HopMetadataProperty(key = "compression_level", storeWithCode = true)
+  private GzipCompressionLevel compressionLevel;
+
   /** Pass all rows, or only the last one. Only the last row was the original 
behaviour. */
   @HopMetadataProperty(key = "pass_all_rows")
   private boolean passAllRows;
@@ -76,6 +91,17 @@ public class BlockingTransformMeta
     prefix = "block";
     cacheSize = CACHE_SIZE;
     compressFiles = true;
+    compressFilesVariable = null;
+    // No level here: a transform loaded without one keeps null and runs at
+    // GzipCompressionLevel.NOT_STORED_DEFAULT, the level it always used.
+    compressionLevel = null;
+  }
+
+  /** Called for a newly created transform only, not when a saved one is 
loaded. */
+  @Override
+  public void setDefault() {
+    super.setDefault();
+    compressionLevel = GzipCompressionLevel.NEW_TRANSFORM_DEFAULT;
   }
 
   @Override
diff --git 
a/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
 
b/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
index c8426dfb53..59fe235b48 100644
--- 
a/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
+++ 
b/plugins/transforms/blockingtransform/src/main/resources/org/apache/hop/pipeline/transforms/blockingtransform/messages/messages_en_US.properties
@@ -25,6 +25,8 @@ BlockingTransform.Log.Openfiles=Opening tmp-file ...
 BlockingTransform.Name=Blocking transform
 BlockingTransformDialog.CacheSize.Label=Cache size (rows in memory) 
 BlockingTransformDialog.Compress.Label=Compress spool files? 
+BlockingTransformDialog.CompressionLevel.Label=Compression level
+BlockingTransformDialog.CompressionLevel.Tooltip=How hard to compress the 
spool files when compression is on. 1 is the fastest, 9 gives the smallest 
files. Spool files are read back only once, so a low level is usually best.
 BlockingTransformDialog.PassAllRows.Label=Pass all rows?
 BlockingTransformDialog.Prefix.Label=Spool-file prefix 
 BlockingTransformDialog.Shell.Title=Blocking transform
diff --git 
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
 
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
index 9f14f2448f..68614959d7 100644
--- 
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
+++ 
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformMetaTest.java
@@ -18,7 +18,9 @@
 package org.apache.hop.pipeline.transforms.blockingtransform;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.xml.XmlHandler;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.junit.jupiter.api.Test;
@@ -33,6 +35,8 @@ class BlockingTransformMetaTest {
     meta.setCacheSize(123456);
     meta.setPassAllRows(false);
     meta.setPrefix("blocking");
+    meta.setCompressFilesVariable("COMPRESS_SPOOL_FILES");
+    meta.setCompressionLevel(GzipCompressionLevel.SMALLEST);
 
     String xml = meta.getXml();
 
@@ -49,5 +53,33 @@ class BlockingTransformMetaTest {
     assertEquals(meta.getCacheSize(), meta2.getCacheSize());
     assertEquals(meta.isPassAllRows(), meta2.isPassAllRows());
     assertEquals(meta.getPrefix(), meta2.getPrefix());
+    assertEquals("COMPRESS_SPOOL_FILES", meta2.getCompressFilesVariable());
+    assertEquals(GzipCompressionLevel.SMALLEST, meta2.getCompressionLevel());
+  }
+
+  @Test
+  void testNewTransformUsesTheFastestLevel() {
+    // Hop GUI calls setDefault() on a transform it creates, never on one it 
loads
+    BlockingTransformMeta meta = new BlockingTransformMeta();
+    meta.setDefault();
+    assertEquals(GzipCompressionLevel.FASTEST, meta.getCompressionLevel());
+  }
+
+  @Test
+  void testOlderXmlWithoutLevelKeepsLevelSix() throws Exception {
+    // A transform saved before the compression level option: no level, no 
variable
+    String transformXml =
+        XmlHandler.openTag(TransformMeta.XML_TAG)
+            + "<compress>Y</compress><pass_all_rows>Y</pass_all_rows>"
+            + XmlHandler.closeTag(TransformMeta.XML_TAG);
+
+    BlockingTransformMeta meta = new BlockingTransformMeta();
+    meta.loadXml(XmlHandler.loadXmlString(transformXml, 
TransformMeta.XML_TAG), null);
+
+    assertNull(meta.getCompressFilesVariable());
+    assertNull(meta.getCompressionLevel());
+    assertEquals(
+        GzipCompressionLevel.STANDARD,
+        GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel()));
   }
 }
diff --git 
a/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
 
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
new file mode 100644
index 0000000000..4c34675dc4
--- /dev/null
+++ 
b/plugins/transforms/blockingtransform/src/test/java/org/apache/hop/pipeline/transforms/blockingtransform/BlockingTransformTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.blockingtransform;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class BlockingTransformTest {
+
+  @TempDir Path tempDir;
+
+  private TransformMockHelper<BlockingTransformMeta, BlockingTransformData> 
helper;
+
+  private BlockingTransformData lastData;
+
+  @BeforeEach
+  void setUp() {
+    helper =
+        new TransformMockHelper<>(
+            "Blocking transform", BlockingTransformMeta.class, 
BlockingTransformData.class);
+    when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(helper.iLogChannel);
+    when(helper.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    helper.cleanUp();
+  }
+
+  @Test
+  void passesAllRowsInOrderFromMemory() throws Exception {
+    List<Object[]> output = run(true, 10_000, true, 2_500);
+
+    assertRowsInOrder(2_500, output);
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void passesAllRowsInOrderFromCompressedTempFiles() throws Exception {
+    // 10 full caches and a partial one: the last 50 rows are written to a 
file of their own
+    List<Object[]> output = run(true, 100, true, 1_050);
+
+    assertRowsInOrder(1_050, output);
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void passesAllRowsInOrderFromUncompressedTempFiles() throws Exception {
+    List<Object[]> output = run(true, 100, false, 1_050);
+
+    assertRowsInOrder(1_050, output);
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void passesNothingWithoutInput() throws Exception {
+    assertTrue(run(true, 100, true, 0).isEmpty());
+  }
+
+  @Test
+  void passesOnlyTheLastRow() throws Exception {
+    List<Object[]> output = run(false, 100, true, 1_050);
+
+    assertEquals(1, output.size());
+    assertArrayEquals(row(1_049), output.get(0));
+    assertNoTempFilesLeft();
+  }
+
+  @ParameterizedTest
+  @EnumSource(GzipCompressionLevel.class)
+  void passesAllRowsInOrderAtEveryCompressionLevel(GzipCompressionLevel level) 
throws Exception {
+    List<Object[]> output =
+        run(true, 100, true, 1_050, meta -> meta.setCompressionLevel(level), 
transform -> {});
+
+    assertRowsInOrder(1_050, output);
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void aVariableSwitchesCompressionOn() throws Exception {
+    List<Object[]> output =
+        run(
+            true,
+            100,
+            false,
+            1_050,
+            meta -> meta.setCompressFilesVariable("COMPRESS_SPOOL_FILES"),
+            transform -> transform.setVariable("COMPRESS_SPOOL_FILES", "Y"));
+
+    assertTrue(lastData.compressFiles);
+    assertRowsInOrder(1_050, output);
+    assertNoTempFilesLeft();
+  }
+
+  private List<Object[]> run(boolean passAllRows, int cacheSize, boolean 
compress, int rowCount)
+      throws Exception {
+    return run(passAllRows, cacheSize, compress, rowCount, meta -> {}, 
transform -> {});
+  }
+
+  private List<Object[]> run(
+      boolean passAllRows,
+      int cacheSize,
+      boolean compress,
+      int rowCount,
+      Consumer<BlockingTransformMeta> metaSetup,
+      Consumer<BlockingTransform> beforeInit)
+      throws Exception {
+    BlockingTransformMeta meta = new BlockingTransformMeta();
+    meta.setPassAllRows(passAllRows);
+    meta.setCacheSize(cacheSize);
+    meta.setCompressFiles(compress);
+    meta.setDirectory(tempDir.toString());
+    meta.setPrefix("block");
+    metaSetup.accept(meta);
+    lastData = new BlockingTransformData();
+
+    BlockingTransform transform =
+        new BlockingTransform(
+            helper.transformMeta, meta, lastData, 0, helper.pipelineMeta, 
helper.pipeline);
+    beforeInit.accept(transform);
+    transform.init();
+
+    List<Object[]> input = new ArrayList<>();
+    for (int i = 0; i < rowCount; i++) {
+      input.add(row(i));
+    }
+    IRowSet rowSet = helper.getMockInputRowSet(input);
+    // BaseTransform also reads with getRowImmediate() (every 500 rows): same 
rows
+    when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("id"));
+    rowMeta.addValueMeta(new ValueMetaString("name"));
+    when(rowSet.getRowMeta()).thenReturn(rowMeta);
+    transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+    List<Object[]> output = new ArrayList<>();
+    transform.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+            output.add(row);
+          }
+        });
+
+    while (transform.processRow()) {
+      // keep going until the transform is done
+    }
+    transform.dispose();
+    return output;
+  }
+
+  private static Object[] row(int i) {
+    return new Object[] {(long) i, "row " + i};
+  }
+
+  private static void assertRowsInOrder(int rowCount, List<Object[]> output) {
+    assertEquals(rowCount, output.size());
+    for (int i = 0; i < rowCount; i++) {
+      Object[] expected = row(i);
+      Object[] actual = output.get(i);
+      assertEquals(expected[0], actual[0], "row " + i);
+      assertEquals(expected[1], actual[1], "row " + i);
+    }
+  }
+
+  private void assertNoTempFilesLeft() throws Exception {
+    try (Stream<Path> left = Files.list(tempDir)) {
+      assertEquals(0, left.count(), "temp files left behind");
+    }
+  }
+}
diff --git 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
index 035f776ec4..4f6ee06f17 100644
--- 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
+++ 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
@@ -35,6 +35,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.exception.HopFileException;
 import org.apache.hop.core.exception.HopValueException;
@@ -136,8 +137,11 @@ public class SortRows extends BaseTransform<SortRowsMeta, 
SortRowsData> {
       OutputStream outputStream = HopVfs.getOutputStream(fileObject, false);
       CountingOutputStream counting = new CountingOutputStream(outputStream);
       if (data.compressFiles) {
-        gzos = new GZIPOutputStream(new BufferedOutputStream(counting));
-        dos = new DataOutputStream(gzos);
+        gzos =
+            GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel())
+                .newOutputStream(new BufferedOutputStream(counting));
+        // Buffer above the GZIP stream too, so values are compressed in 
blocks, not one by one.
+        dos = new DataOutputStream(new BufferedOutputStream(gzos, 500000));
       } else {
         dos = new DataOutputStream(new BufferedOutputStream(counting, 500000));
         gzos = null;
@@ -222,7 +226,8 @@ public class SortRows extends BaseTransform<SortRowsMeta, 
SortRowsData> {
   }
 
   private DataInputStream getDataInputStream(GZIPInputStream gzipInputStream) {
-    DataInputStream result = new DataInputStream(gzipInputStream);
+    // Buffer above the GZIP stream, so values are decompressed in blocks, not 
one by one.
+    DataInputStream result = new DataInputStream(new 
BufferedInputStream(gzipInputStream, 50000));
     data.gzis.add(gzipInputStream);
     return result;
   }
diff --git 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
index e720b52790..577ee446c8 100644
--- 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
+++ 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
@@ -20,11 +20,13 @@ package org.apache.hop.pipeline.transforms.sort;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.hop.core.Const;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
 import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.ui.core.ConstUi;
@@ -40,6 +42,7 @@ import 
org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
 import org.apache.hop.ui.pipeline.transform.ComponentSelectionListener;
 import org.apache.hop.ui.pipeline.transform.ITableItemInsertListener;
 import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CCombo;
 import org.eclipse.swt.events.ModifyListener;
 import org.eclipse.swt.events.SelectionAdapter;
 import org.eclipse.swt.events.SelectionEvent;
@@ -69,6 +72,8 @@ public class SortRowsDialog extends BaseTransformDialog {
 
   private CheckBoxVar wCompress;
 
+  private CCombo wCompressionLevel;
+
   private Button wUniqueRows;
 
   private TableView wFields;
@@ -209,14 +214,39 @@ public class SortRowsDialog extends BaseTransformDialog {
           }
         });
 
-    // Using compression for temporary files?
+    // How hard to compress the temporary files
+    Label wlCompressionLevel = new Label(shell, SWT.RIGHT);
+    wlCompressionLevel.setText(
+        BaseMessages.getString(PKG, "SortRowsDialog.CompressionLevel.Label"));
+    wlCompressionLevel.setToolTipText(
+        BaseMessages.getString(PKG, 
"SortRowsDialog.CompressionLevel.Tooltip"));
+    PropsUi.setLook(wlCompressionLevel);
+    FormData fdlCompressionLevel = new FormData();
+    fdlCompressionLevel.left = new FormAttachment(0, 0);
+    fdlCompressionLevel.right = new FormAttachment(middle, -margin);
+    fdlCompressionLevel.top = new FormAttachment(wCompress, margin);
+    wlCompressionLevel.setLayoutData(fdlCompressionLevel);
+    wCompressionLevel = new CCombo(shell, SWT.SINGLE | SWT.READ_ONLY | 
SWT.BORDER);
+    wCompressionLevel.setItems(
+        
IEnumHasCodeAndDescription.getDescriptions(GzipCompressionLevel.class));
+    wCompressionLevel.setToolTipText(
+        BaseMessages.getString(PKG, 
"SortRowsDialog.CompressionLevel.Tooltip"));
+    PropsUi.setLook(wCompressionLevel);
+    wCompressionLevel.addModifyListener(lsMod);
+    FormData fdCompressionLevel = new FormData();
+    fdCompressionLevel.left = new FormAttachment(middle, 0);
+    fdCompressionLevel.top = new FormAttachment(wCompress, margin);
+    fdCompressionLevel.right = new FormAttachment(100, 0);
+    wCompressionLevel.setLayoutData(fdCompressionLevel);
+
+    // Only pass unique rows?
     Label wlUniqueRows = new Label(shell, SWT.RIGHT);
     wlUniqueRows.setText(BaseMessages.getString(PKG, 
"SortRowsDialog.UniqueRows.Label"));
     PropsUi.setLook(wlUniqueRows);
     FormData fdlUniqueRows = new FormData();
     fdlUniqueRows.left = new FormAttachment(0, 0);
     fdlUniqueRows.right = new FormAttachment(middle, -margin);
-    fdlUniqueRows.top = new FormAttachment(wCompress, margin);
+    fdlUniqueRows.top = new FormAttachment(wCompressionLevel, margin);
     wlUniqueRows.setLayoutData(fdlUniqueRows);
     wUniqueRows = new Button(shell, SWT.CHECK);
     wUniqueRows.setToolTipText(BaseMessages.getString(PKG, 
"SortRowsDialog.UniqueRows.Tooltip"));
@@ -361,6 +391,8 @@ public class SortRowsDialog extends BaseTransformDialog {
     wFreeMemory.setText(Const.NVL(input.getFreeMemoryLimit(), ""));
     wCompress.setSelection(input.isCompressFiles());
     wCompress.setVariableName(input.getCompressFilesVariable());
+    wCompressionLevel.select(
+        
GzipCompressionLevel.orNotStoredDefault(input.getCompressionLevel()).getIndex());
     wUniqueRows.setSelection(input.isOnlyPassingUniqueRows());
 
     Table table = wFields.table;
@@ -424,6 +456,11 @@ public class SortRowsDialog extends BaseTransformDialog {
     log.logDetailed("Sort rows", "Compression is set to " + 
wCompress.getSelection());
     input.setCompressFiles(wCompress.getSelection());
     input.setCompressFilesVariable(wCompress.getVariableName());
+    input.setCompressionLevel(
+        IEnumHasCodeAndDescription.lookupDescription(
+            GzipCompressionLevel.class,
+            wCompressionLevel.getText(),
+            GzipCompressionLevel.NOT_STORED_DEFAULT));
     input.setOnlyPassingUniqueRows(wUniqueRows.getSelection());
 
     int nrFields = wFields.nrNonEmpty();
diff --git 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
index 43bad10458..53163a4a98 100644
--- 
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
+++ 
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
@@ -30,6 +30,7 @@ import org.apache.hop.core.CheckResult;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopTransformException;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
@@ -96,6 +97,13 @@ public class SortRowsMeta extends 
BaseTransformMeta<SortRows, SortRowsData>
   @HopMetadataProperty(key = "compress_variables", injectionKey = 
"COMPRESS_VARIABLE")
   private String compressFilesVariable;
 
+  /** How hard to compress the temporary files, from 1 (fastest) to 9 
(smallest files). */
+  @HopMetadataProperty(
+      key = "compression_level",
+      storeWithCode = true,
+      injectionKey = "COMPRESSION_LEVEL")
+  private GzipCompressionLevel compressionLevel;
+
   public SortRowsMeta() {
     super();
     sortFields = new ArrayList<>();
@@ -105,9 +113,19 @@ public class SortRowsMeta extends 
BaseTransformMeta<SortRows, SortRowsData>
     freeMemoryLimit = null;
     compressFiles = false;
     compressFilesVariable = null;
+    // No level here: a transform loaded without one keeps null and runs at
+    // GzipCompressionLevel.NOT_STORED_DEFAULT, the level it always used.
+    compressionLevel = null;
     onlyPassingUniqueRows = false;
   }
 
+  /** Called for a newly created transform only, not when a saved one is 
loaded. */
+  @Override
+  public void setDefault() {
+    super.setDefault();
+    compressionLevel = GzipCompressionLevel.NEW_TRANSFORM_DEFAULT;
+  }
+
   /**
    * Returns the collation strength based on the passed in locale. Package 
protected for testing
    * purposes
diff --git 
a/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
 
b/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
index b9b5383359..8885ef4694 100644
--- 
a/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
+++ 
b/plugins/transforms/sort/src/main/resources/org/apache/hop/pipeline/transforms/sort/messages/messages_en_US.properties
@@ -31,6 +31,7 @@ SortRows.Error.UnableToCloseFile=Unable to close/delete file 
#{0} --> "{1}
 SortRows.Injection.COLLATOR_ENABLED=Enable this option to use collator support.
 SortRows.Injection.COLLATOR_STRENGTH=Specify the collator strength when 
collator support is enabled (0-3).
 SortRows.Injection.COMPRESS_TEMP_FILES=Enable this option to compress 
temporary files.
+SortRows.Injection.COMPRESSION_LEVEL=How hard to compress the temporary files: 
FASTEST, VERY_FAST, FAST, BALANCED_FASTER, BALANCED, STANDARD, SMALLER, 
VERY_SMALL or SMALLEST.
 SortRows.Injection.FIELDS=The fields to sort.
 SortRows.Injection.FREE_MEMORY_TRESHOLD=The percentage of free memory to allow 
until writing to temporary files.
 SortRows.Injection.IGNORE_CASE=Enable this option to ignore case when sorting 
the field.
@@ -50,6 +51,8 @@ SortRowsDialog.CaseInsensitive.Column=Case sensitive compare
 SortRowsDialog.CollatorDisabled.Column=Sort based on current locale
 SortRowsDialog.CollatorStrength.Column=Collator Strength
 SortRowsDialog.Compress.Label=Compress TMP files
+SortRowsDialog.CompressionLevel.Label=Compression level
+SortRowsDialog.CompressionLevel.Tooltip=How hard to compress the temporary 
files when compression is on. 1 is the fastest, 9 gives the smallest files. 
Temporary files are read back only once, so a low level is usually best.
 SortRowsDialog.DialogTitle=Sort rows
 SortRowsDialog.Fieldname.Column=Fieldname
 SortRowsDialog.Fields.Label=Fields:
diff --git 
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
index 72ef2906e3..15a86d4785 100644
--- 
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
+++ 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.sort;
 
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.injection.BaseMetadataInjectionTestJunit5;
 import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
 import org.junit.jupiter.api.BeforeEach;
@@ -53,6 +54,7 @@ class SortRowsMetaInjectionTest extends 
BaseMetadataInjectionTestJunit5<SortRows
     check("COLLATOR_ENABLED", () -> 
meta.getSortFields().get(0).isCollatorEnabled());
     check("COLLATOR_STRENGTH", () -> 
meta.getSortFields().get(0).getCollatorStrength());
     check("COMPRESS_VARIABLE", () -> meta.getCompressFilesVariable());
+    check("COMPRESSION_LEVEL", () -> meta.getCompressionLevel(), 
GzipCompressionLevel.class);
     //    check("NAME", () -> meta.getFieldName()[0]);
     //    check("SORT_ASCENDING", () -> meta.getAscending()[0]);
     //    check("IGNORE_CASE", () -> meta.getCaseSensitive()[0]);
diff --git 
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
index 3f4bec42ef..852f73bc78 100644
--- 
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
+++ 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.sort;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.text.Collator;
@@ -29,10 +30,14 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.hop.core.compress.gzip.GzipCompressionLevel;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.xml.XmlHandler;
 import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
 import 
org.apache.hop.pipeline.transforms.loadsave.validator.BooleanLoadSaveValidator;
+import 
org.apache.hop.pipeline.transforms.loadsave.validator.EnumLoadSaveValidator;
 import 
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
 import 
org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator;
 import 
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
@@ -47,6 +52,31 @@ class SortRowsMetaTest {
   /**
    * @throws HopException
    */
+  @Test
+  void testNewTransformUsesTheFastestLevel() {
+    // Hop GUI calls setDefault() on a transform it creates, never on one it 
loads
+    SortRowsMeta meta = new SortRowsMeta();
+    meta.setDefault();
+    assertEquals(GzipCompressionLevel.FASTEST, meta.getCompressionLevel());
+  }
+
+  @Test
+  void testOlderXmlWithoutLevelKeepsLevelSix() throws Exception {
+    // A transform saved before the compression level option
+    String transformXml =
+        XmlHandler.openTag(TransformMeta.XML_TAG)
+            + "<compress>Y</compress><sort_size>1000</sort_size>"
+            + XmlHandler.closeTag(TransformMeta.XML_TAG);
+
+    SortRowsMeta meta = new SortRowsMeta();
+    meta.loadXml(XmlHandler.loadXmlString(transformXml, 
TransformMeta.XML_TAG), null);
+
+    assertNull(meta.getCompressionLevel());
+    assertEquals(
+        GzipCompressionLevel.STANDARD,
+        GzipCompressionLevel.orNotStoredDefault(meta.getCompressionLevel()));
+  }
+
   @Test
   void testRoundTrips() throws Exception {
     List<String> attributes =
@@ -57,6 +87,7 @@ class SortRowsMetaTest {
             "FreeMemoryLimit",
             "CompressFiles",
             "CompressFilesVariable",
+            "CompressionLevel",
             "OnlyPassingUniqueRows",
             "SortFields");
 
@@ -74,6 +105,8 @@ class SortRowsMetaTest {
     fieldLoadSaveValidatorAttributeMap.put("FreeMemoryLimit", 
stringFieldLoadSaveValidator);
     fieldLoadSaveValidatorAttributeMap.put("CompressFiles", 
booleanFieldLoadSaveValidator);
     fieldLoadSaveValidatorAttributeMap.put("CompressFilesVariable", 
stringFieldLoadSaveValidator);
+    fieldLoadSaveValidatorAttributeMap.put(
+        "CompressionLevel", new 
EnumLoadSaveValidator<>(GzipCompressionLevel.SMALLEST));
     fieldLoadSaveValidatorAttributeMap.put("OnlyPassingUniqueRows", 
booleanFieldLoadSaveValidator);
 
     LoadSaveTester<SortRowsMeta> loadSaveTester =
diff --git 
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
new file mode 100644
index 0000000000..30b91f7530
--- /dev/null
+++ 
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.sort;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class SortRowsTest {
+
+  private static final int ROWS = 1_050;
+
+  @TempDir Path tempDir;
+
+  private TransformMockHelper<SortRowsMeta, SortRowsData> helper;
+
+  @BeforeEach
+  void setUp() {
+    helper = new TransformMockHelper<>("Sort rows", SortRowsMeta.class, 
SortRowsData.class);
+    when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(helper.iLogChannel);
+    when(helper.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    helper.cleanUp();
+  }
+
+  @Test
+  void sortsInMemory() throws Exception {
+    assertSorted(run("10000", true));
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void sortsThroughCompressedTempFiles() throws Exception {
+    // 10 full sort buffers and a partial one: 11 temp files to merge
+    assertSorted(run("100", true));
+    assertNoTempFilesLeft();
+  }
+
+  @Test
+  void sortsThroughUncompressedTempFiles() throws Exception {
+    assertSorted(run("100", false));
+    assertNoTempFilesLeft();
+  }
+
+  private List<Object[]> run(String sortSize, boolean compress) throws 
Exception {
+    SortRowsMeta meta = new SortRowsMeta();
+    meta.setDefault();
+    meta.setSortSize(sortSize);
+    meta.setFreeMemoryLimit(null);
+    meta.setCompressFiles(compress);
+    meta.setDirectory(tempDir.toString());
+    meta.setPrefix("sort");
+    meta.getSortFields().clear();
+    meta.getSortFields().add(new SortRowsField("id", true, true, false, 0, 
false));
+
+    SortRows transform =
+        new SortRows(
+            helper.transformMeta,
+            meta,
+            new SortRowsData(),
+            0,
+            helper.pipelineMeta,
+            helper.pipeline);
+    transform.init();
+
+    // Every id from 0 to ROWS - 1 once, in a scrambled order
+    List<Object[]> input = new ArrayList<>();
+    for (int i = 0; i < ROWS; i++) {
+      long id = (i * 7919L) % ROWS;
+      input.add(new Object[] {id, "row " + id});
+    }
+    IRowSet rowSet = helper.getMockInputRowSet(input);
+    // BaseTransform also reads with getRowImmediate() (every 500 rows): same 
rows
+    when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("id"));
+    rowMeta.addValueMeta(new ValueMetaString("name"));
+    when(rowSet.getRowMeta()).thenReturn(rowMeta);
+    transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+    List<Object[]> output = new ArrayList<>();
+    transform.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+            output.add(row);
+          }
+        });
+
+    while (transform.processRow()) {
+      // keep going until the transform is done
+    }
+    transform.dispose();
+    return output;
+  }
+
+  private static void assertSorted(List<Object[]> output) {
+    assertEquals(ROWS, output.size());
+    for (int i = 0; i < ROWS; i++) {
+      assertEquals((long) i, output.get(i)[0], "row " + i);
+      assertEquals("row " + i, output.get(i)[1], "row " + i);
+    }
+  }
+
+  private void assertNoTempFilesLeft() throws Exception {
+    try (Stream<Path> left = Files.list(tempDir)) {
+      assertEquals(0, left.count(), "temp files left behind");
+    }
+  }
+}

Reply via email to