This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 11ae50a2f6 Serialize to file and De-serialize from file call the GZIP 
stream for every value, fixes #9346 (#9347)
11ae50a2f6 is described below

commit 11ae50a2f6e73086bb668b8a285990dda4de04b3
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:52:18 2026 +0100

    Serialize to file and De-serialize from file call the GZIP stream for every 
value, fixes #9346 (#9347)
---
 .../hop/pipeline/transforms/cube/CubeStreams.java  |  46 ++++
 .../pipeline/transforms/cubeinput/CubeInput.java   |   5 +-
 .../transforms/cubeinput/CubeInputMeta.java        |   3 +-
 .../pipeline/transforms/cubeoutput/CubeOutput.java |   4 +-
 .../cubeoutput/CubeFileCompatibilityTest.java      | 239 +++++++++++++++++++++
 5 files changed, 292 insertions(+), 5 deletions(-)

diff --git 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
new file mode 100644
index 0000000000..9dec892f4d
--- /dev/null
+++ 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.cube;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * The data streams on top of a cube file's GZIP stream. Rows are written and 
read a few bytes at a
+ * time; without a buffer in between, each of those is its own deflate or 
inflate call.
+ */
+public final class CubeStreams {
+
+  /** The size of the buffer between the data stream and the GZIP stream. */
+  public static final int BUFFER_SIZE = 64 * 1024;
+
+  private CubeStreams() {}
+
+  /** Closing the returned stream flushes the buffer and finishes the GZIP 
stream. */
+  public static DataOutputStream dataOutput(GZIPOutputStream gzip) {
+    return new DataOutputStream(new BufferedOutputStream(gzip, BUFFER_SIZE));
+  }
+
+  public static DataInputStream dataInput(GZIPInputStream gzip) {
+    return new DataInputStream(new BufferedInputStream(gzip, BUFFER_SIZE));
+  }
+}
diff --git 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
index 7d3ee62b0b..6bf283dbbe 100644
--- 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
+++ 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
@@ -39,6 +39,7 @@ import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.BaseTransform;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
 
 public class CubeInput extends BaseTransform<CubeInputMeta, CubeInputData> {
 
@@ -133,7 +134,7 @@ public class CubeInput extends BaseTransform<CubeInputMeta, 
CubeInputData> {
       input = HopVfs.getInputStream(filename, variables);
       input = new CountingInputStream(input);
       GZIPInputStream gzip = new GZIPInputStream(input);
-      dataInput = new DataInputStream(gzip);
+      dataInput = CubeStreams.dataInput(gzip);
       IRowMeta layout = new RowMeta(dataInput);
       if (data.meta != null && !sameLayout(data.meta, layout)) {
         throw new HopException(
@@ -234,7 +235,7 @@ public class CubeInput extends BaseTransform<CubeInputMeta, 
CubeInputData> {
     }
     try (InputStream is = HopVfs.getInputStream(filename, variables);
         GZIPInputStream fis = new GZIPInputStream(is);
-        DataInputStream dis = new DataInputStream(fis)) {
+        DataInputStream dis = CubeStreams.dataInput(fis)) {
       data.meta = new RowMeta(dis);
       data.referenceFilename = filename;
     } catch (HopException e) {
diff --git 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
index b22cffb285..f422f526c3 100644
--- 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
+++ 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
@@ -49,6 +49,7 @@ import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.BaseTransformMeta;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
 import org.apache.hop.resource.IResourceNaming;
 import org.apache.hop.resource.ResourceDefinition;
 
@@ -234,7 +235,7 @@ public class CubeInputMeta extends 
BaseTransformMeta<CubeInput, CubeInputData> {
         CubeFilename.resolve(variables, file.getName(), 
usesTransformNrInFilename(), 0);
     try (InputStream is = HopVfs.getInputStream(filename, variables);
         GZIPInputStream fis = new GZIPInputStream(is);
-        DataInputStream dis = new DataInputStream(fis)) {
+        DataInputStream dis = CubeStreams.dataInput(fis)) {
       IRowMeta add = new RowMeta(dis);
       for (int i = 0; i < add.size(); i++) {
         add.getValueMeta(i).setOrigin(name);
diff --git 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
index e2b687b410..8614a334f4 100644
--- 
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
+++ 
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
@@ -17,7 +17,6 @@
 
 package org.apache.hop.pipeline.transforms.cubeoutput;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.zip.GZIPOutputStream;
 import org.apache.commons.vfs2.FileObject;
@@ -35,6 +34,7 @@ import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.BaseTransform;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
 
 /**
  * Outputs a stream/series of rows to a file, effectively building a sort of 
(compressed) microcube.
@@ -229,7 +229,7 @@ public class CubeOutput extends 
BaseTransform<CubeOutputMeta, CubeOutputData> {
       data.fos = HopVfs.getOutputStream(filename, false, variables);
       data.countingOutputStream = new CountingOutputStream(data.fos);
       data.zip = new GZIPOutputStream(data.countingOutputStream);
-      data.dos = new DataOutputStream(data.zip);
+      data.dos = CubeStreams.dataOutput(data.zip);
     } catch (Exception e) {
       throw new HopFileException(e);
     }
diff --git 
a/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
 
b/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
new file mode 100644
index 0000000000..f862903150
--- /dev/null
+++ 
b/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.cubeoutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopEofException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.LoggingObject;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBigNumber;
+import org.apache.hop.core.row.value.ValueMetaBinary;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.cubeinput.CubeInputMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Cube files written without a buffer between the data stream and the GZIP 
stream (as before) and
+ * files written by Serialize to file read in both directions, with the same 
layout and rows. 20,000
+ * rows is more than one buffer.
+ */
+class CubeFileCompatibilityTest {
+
+  private static final int NR_ROWS = 20_000;
+
+  @TempDir Path tempDir;
+
+  @BeforeAll
+  static void initHop() throws Exception {
+    HopEnvironment.init();
+    PluginRegistry registry = PluginRegistry.getInstance();
+    for (ITransformMeta meta :
+        List.of(new CubeInputMeta(), new CubeOutputMeta(), new InjectorMeta(), 
new DummyMeta())) {
+      if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+        registry.registerPluginClass(
+            meta.getClass().getName(), TransformPluginType.class, 
Transform.class);
+      }
+    }
+  }
+
+  @Test
+  void fileWrittenUnbufferedIsReadByDeSerializeFromFile() throws Exception {
+    Path file = tempDir.resolve("old.cube");
+    IRowMeta layout = layout();
+    List<Object[]> rows = rows();
+    try (OutputStream os = Files.newOutputStream(file);
+        DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))) 
{
+      layout.writeMeta(dos);
+      for (Object[] row : rows) {
+        layout.writeData(dos, row);
+      }
+    }
+
+    CubeInputMeta inputMeta = new CubeInputMeta();
+    inputMeta.setDefault();
+    inputMeta.getFile().setName(file.toString());
+    PipelineMeta pipelineMeta = new PipelineMeta();
+    pipelineMeta.setName("read cube");
+    TransformMeta input = addTransform(pipelineMeta, "cube input", inputMeta);
+    TransformMeta output = addTransform(pipelineMeta, "output", new 
DummyMeta());
+    pipelineMeta.addPipelineHop(new PipelineHopMeta(input, output));
+
+    LocalPipelineEngine pipeline =
+        new LocalPipelineEngine(pipelineMeta, new Variables(), new 
LoggingObject("test"));
+    pipeline.prepareExecution();
+    List<Object[]> read = new ArrayList<>();
+    List<IRowMeta> readLayouts = new ArrayList<>();
+    pipeline
+        .getTransform("cube input", 0)
+        .addRowListener(
+            new RowAdapter() {
+              @Override
+              public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
+                  throws HopTransformException {
+                if (readLayouts.isEmpty()) {
+                  readLayouts.add(rowMeta);
+                }
+                read.add(Arrays.copyOf(row, rowMeta.size()));
+              }
+            });
+    pipeline.startThreads();
+    pipeline.waitUntilFinished();
+
+    assertEquals(0, pipeline.getErrors());
+    assertEquals(List.of(layout.getFieldNames()), 
List.of(readLayouts.get(0).getFieldNames()));
+    assertRows(rows, read);
+  }
+
+  @Test
+  void fileWrittenBySerializeToFileReadsUnbuffered() throws Exception {
+    Path file = tempDir.resolve("new.cube");
+    IRowMeta layout = layout();
+    List<Object[]> rows = rows();
+
+    CubeOutputMeta outputMeta = new CubeOutputMeta();
+    outputMeta.setDefault();
+    outputMeta.setFilename(file.toString());
+    PipelineMeta pipelineMeta = new PipelineMeta();
+    pipelineMeta.setName("write cube");
+    TransformMeta input = addTransform(pipelineMeta, "input", new 
InjectorMeta());
+    TransformMeta output = addTransform(pipelineMeta, "cube output", 
outputMeta);
+    pipelineMeta.addPipelineHop(new PipelineHopMeta(input, output));
+
+    LocalPipelineEngine pipeline =
+        new LocalPipelineEngine(pipelineMeta, new Variables(), new 
LoggingObject("test"));
+    pipeline.setVariable(Const.HOP_METRIC_DATA_VOLUME, "Y");
+    pipeline.prepareExecution();
+    RowProducer producer = pipeline.addRowProducer("input", 0);
+    pipeline.startThreads();
+    for (Object[] row : rows) {
+      producer.putRow(layout, row);
+    }
+    producer.finished();
+    pipeline.waitUntilFinished();
+    assertEquals(0, pipeline.getErrors());
+
+    List<Object[]> read = new ArrayList<>();
+    IRowMeta readLayout;
+    try (InputStream is = Files.newInputStream(file);
+        DataInputStream dis = new DataInputStream(new GZIPInputStream(is))) {
+      readLayout = new RowMeta(dis);
+      while (true) {
+        try {
+          read.add(readLayout.readData(dis));
+        } catch (HopEofException eof) {
+          break;
+        }
+      }
+    }
+    assertEquals(List.of(layout.getFieldNames()), 
List.of(readLayout.getFieldNames()));
+    assertRows(rows, read);
+
+    // The counter below the GZIP stream still sees every compressed byte.
+    assertEquals(
+        Files.size(file), (long) pipeline.getTransform("cube output", 
0).getDataVolumeOut());
+  }
+
+  private static IRowMeta layout() {
+    IRowMeta layout = new RowMeta();
+    layout.addValueMeta(new ValueMetaString("name"));
+    layout.addValueMeta(new ValueMetaInteger("id"));
+    layout.addValueMeta(new ValueMetaNumber("price"));
+    layout.addValueMeta(new ValueMetaBigNumber("amount"));
+    layout.addValueMeta(new ValueMetaDate("ordered"));
+    layout.addValueMeta(new ValueMetaBoolean("paid"));
+    layout.addValueMeta(new ValueMetaBinary("payload"));
+    return layout;
+  }
+
+  /** Every fifth row has only nulls apart from its id. */
+  private static List<Object[]> rows() {
+    List<Object[]> rows = new ArrayList<>();
+    for (long i = 0; i < NR_ROWS; i++) {
+      if (i % 5 == 0) {
+        rows.add(new Object[] {null, i, null, null, null, null, null});
+      } else {
+        rows.add(
+            new Object[] {
+              "name-" + i,
+              i,
+              i * 1.25,
+              BigDecimal.valueOf(i, 3),
+              new Date(1_700_000_000_000L + i),
+              i % 2 == 0,
+              new byte[] {(byte) i, (byte) (i >> 8)}
+            });
+      }
+    }
+    return rows;
+  }
+
+  private static void assertRows(List<Object[]> expected, List<Object[]> 
actual) {
+    assertEquals(expected.size(), actual.size());
+    for (int i = 0; i < expected.size(); i++) {
+      assertTrue(
+          Arrays.deepEquals(expected.get(i), actual.get(i)),
+          "row " + i + ": " + Arrays.deepToString(actual.get(i)));
+    }
+  }
+
+  private static TransformMeta addTransform(
+      PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+    String pluginId = 
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+    TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+    pipelineMeta.addTransform(transformMeta);
+    return transformMeta;
+  }
+}

Reply via email to