This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 11ae50a2f6 Serialize to file and De-serialize from file call the GZIP
stream for every value, fixes #9346 (#9347)
11ae50a2f6 is described below
commit 11ae50a2f6e73086bb668b8a285990dda4de04b3
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:52:18 2026 +0100
Serialize to file and De-serialize from file call the GZIP stream for every
value, fixes #9346 (#9347)
---
.../hop/pipeline/transforms/cube/CubeStreams.java | 46 ++++
.../pipeline/transforms/cubeinput/CubeInput.java | 5 +-
.../transforms/cubeinput/CubeInputMeta.java | 3 +-
.../pipeline/transforms/cubeoutput/CubeOutput.java | 4 +-
.../cubeoutput/CubeFileCompatibilityTest.java | 239 +++++++++++++++++++++
5 files changed, 292 insertions(+), 5 deletions(-)
diff --git
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
new file mode 100644
index 0000000000..9dec892f4d
--- /dev/null
+++
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cube/CubeStreams.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.cube;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * The data streams on top of a cube file's GZIP stream. Rows are written and
read a few bytes at a
+ * time; without a buffer in between, each of those is its own deflate or
inflate call.
+ */
+public final class CubeStreams {
+
+ /** The size of the buffer between the data stream and the GZIP stream. */
+ public static final int BUFFER_SIZE = 64 * 1024;
+
+ private CubeStreams() {}
+
+ /** Closing the returned stream flushes the buffer and finishes the GZIP
stream. */
+ public static DataOutputStream dataOutput(GZIPOutputStream gzip) {
+ return new DataOutputStream(new BufferedOutputStream(gzip, BUFFER_SIZE));
+ }
+
+ public static DataInputStream dataInput(GZIPInputStream gzip) {
+ return new DataInputStream(new BufferedInputStream(gzip, BUFFER_SIZE));
+ }
+}
diff --git
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
index 7d3ee62b0b..6bf283dbbe 100644
---
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
+++
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java
@@ -39,6 +39,7 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
public class CubeInput extends BaseTransform<CubeInputMeta, CubeInputData> {
@@ -133,7 +134,7 @@ public class CubeInput extends BaseTransform<CubeInputMeta,
CubeInputData> {
input = HopVfs.getInputStream(filename, variables);
input = new CountingInputStream(input);
GZIPInputStream gzip = new GZIPInputStream(input);
- dataInput = new DataInputStream(gzip);
+ dataInput = CubeStreams.dataInput(gzip);
IRowMeta layout = new RowMeta(dataInput);
if (data.meta != null && !sameLayout(data.meta, layout)) {
throw new HopException(
@@ -234,7 +235,7 @@ public class CubeInput extends BaseTransform<CubeInputMeta,
CubeInputData> {
}
try (InputStream is = HopVfs.getInputStream(filename, variables);
GZIPInputStream fis = new GZIPInputStream(is);
- DataInputStream dis = new DataInputStream(fis)) {
+ DataInputStream dis = CubeStreams.dataInput(fis)) {
data.meta = new RowMeta(dis);
data.referenceFilename = filename;
} catch (HopException e) {
diff --git
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
index b22cffb285..f422f526c3 100644
---
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
+++
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInputMeta.java
@@ -49,6 +49,7 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
import org.apache.hop.resource.IResourceNaming;
import org.apache.hop.resource.ResourceDefinition;
@@ -234,7 +235,7 @@ public class CubeInputMeta extends
BaseTransformMeta<CubeInput, CubeInputData> {
CubeFilename.resolve(variables, file.getName(),
usesTransformNrInFilename(), 0);
try (InputStream is = HopVfs.getInputStream(filename, variables);
GZIPInputStream fis = new GZIPInputStream(is);
- DataInputStream dis = new DataInputStream(fis)) {
+ DataInputStream dis = CubeStreams.dataInput(fis)) {
IRowMeta add = new RowMeta(dis);
for (int i = 0; i < add.size(); i++) {
add.getValueMeta(i).setOrigin(name);
diff --git
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
index e2b687b410..8614a334f4 100644
---
a/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
+++
b/plugins/transforms/cube/src/main/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeOutput.java
@@ -17,7 +17,6 @@
package org.apache.hop.pipeline.transforms.cubeoutput;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.vfs2.FileObject;
@@ -35,6 +34,7 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.cube.CubeFilename;
+import org.apache.hop.pipeline.transforms.cube.CubeStreams;
/**
* Outputs a stream/series of rows to a file, effectively building a sort of
(compressed) microcube.
@@ -229,7 +229,7 @@ public class CubeOutput extends
BaseTransform<CubeOutputMeta, CubeOutputData> {
data.fos = HopVfs.getOutputStream(filename, false, variables);
data.countingOutputStream = new CountingOutputStream(data.fos);
data.zip = new GZIPOutputStream(data.countingOutputStream);
- data.dos = new DataOutputStream(data.zip);
+ data.dos = CubeStreams.dataOutput(data.zip);
} catch (Exception e) {
throw new HopFileException(e);
}
diff --git
a/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
b/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
new file mode 100644
index 0000000000..f862903150
--- /dev/null
+++
b/plugins/transforms/cube/src/test/java/org/apache/hop/pipeline/transforms/cubeoutput/CubeFileCompatibilityTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.cubeoutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopEofException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.LoggingObject;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBigNumber;
+import org.apache.hop.core.row.value.ValueMetaBinary;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.cubeinput.CubeInputMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Cube files written without a buffer between the data stream and the GZIP
stream (as before) and
+ * files written by Serialize to file read in both directions, with the same
layout and rows. 20,000
+ * rows is more than one buffer.
+ */
+class CubeFileCompatibilityTest {
+
+ private static final int NR_ROWS = 20_000;
+
+ @TempDir Path tempDir;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta :
+ List.of(new CubeInputMeta(), new CubeOutputMeta(), new InjectorMeta(),
new DummyMeta())) {
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ @Test
+ void fileWrittenUnbufferedIsReadByDeSerializeFromFile() throws Exception {
+ Path file = tempDir.resolve("old.cube");
+ IRowMeta layout = layout();
+ List<Object[]> rows = rows();
+ try (OutputStream os = Files.newOutputStream(file);
+ DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os)))
{
+ layout.writeMeta(dos);
+ for (Object[] row : rows) {
+ layout.writeData(dos, row);
+ }
+ }
+
+ CubeInputMeta inputMeta = new CubeInputMeta();
+ inputMeta.setDefault();
+ inputMeta.getFile().setName(file.toString());
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("read cube");
+ TransformMeta input = addTransform(pipelineMeta, "cube input", inputMeta);
+ TransformMeta output = addTransform(pipelineMeta, "output", new
DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, output));
+
+ LocalPipelineEngine pipeline =
+ new LocalPipelineEngine(pipelineMeta, new Variables(), new
LoggingObject("test"));
+ pipeline.prepareExecution();
+ List<Object[]> read = new ArrayList<>();
+ List<IRowMeta> readLayouts = new ArrayList<>();
+ pipeline
+ .getTransform("cube input", 0)
+ .addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
+ throws HopTransformException {
+ if (readLayouts.isEmpty()) {
+ readLayouts.add(rowMeta);
+ }
+ read.add(Arrays.copyOf(row, rowMeta.size()));
+ }
+ });
+ pipeline.startThreads();
+ pipeline.waitUntilFinished();
+
+ assertEquals(0, pipeline.getErrors());
+ assertEquals(List.of(layout.getFieldNames()),
List.of(readLayouts.get(0).getFieldNames()));
+ assertRows(rows, read);
+ }
+
+ @Test
+ void fileWrittenBySerializeToFileReadsUnbuffered() throws Exception {
+ Path file = tempDir.resolve("new.cube");
+ IRowMeta layout = layout();
+ List<Object[]> rows = rows();
+
+ CubeOutputMeta outputMeta = new CubeOutputMeta();
+ outputMeta.setDefault();
+ outputMeta.setFilename(file.toString());
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("write cube");
+ TransformMeta input = addTransform(pipelineMeta, "input", new
InjectorMeta());
+ TransformMeta output = addTransform(pipelineMeta, "cube output",
outputMeta);
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, output));
+
+ LocalPipelineEngine pipeline =
+ new LocalPipelineEngine(pipelineMeta, new Variables(), new
LoggingObject("test"));
+ pipeline.setVariable(Const.HOP_METRIC_DATA_VOLUME, "Y");
+ pipeline.prepareExecution();
+ RowProducer producer = pipeline.addRowProducer("input", 0);
+ pipeline.startThreads();
+ for (Object[] row : rows) {
+ producer.putRow(layout, row);
+ }
+ producer.finished();
+ pipeline.waitUntilFinished();
+ assertEquals(0, pipeline.getErrors());
+
+ List<Object[]> read = new ArrayList<>();
+ IRowMeta readLayout;
+ try (InputStream is = Files.newInputStream(file);
+ DataInputStream dis = new DataInputStream(new GZIPInputStream(is))) {
+ readLayout = new RowMeta(dis);
+ while (true) {
+ try {
+ read.add(readLayout.readData(dis));
+ } catch (HopEofException eof) {
+ break;
+ }
+ }
+ }
+ assertEquals(List.of(layout.getFieldNames()),
List.of(readLayout.getFieldNames()));
+ assertRows(rows, read);
+
+ // The counter below the GZIP stream still sees every compressed byte.
+ assertEquals(
+ Files.size(file), (long) pipeline.getTransform("cube output",
0).getDataVolumeOut());
+ }
+
+ private static IRowMeta layout() {
+ IRowMeta layout = new RowMeta();
+ layout.addValueMeta(new ValueMetaString("name"));
+ layout.addValueMeta(new ValueMetaInteger("id"));
+ layout.addValueMeta(new ValueMetaNumber("price"));
+ layout.addValueMeta(new ValueMetaBigNumber("amount"));
+ layout.addValueMeta(new ValueMetaDate("ordered"));
+ layout.addValueMeta(new ValueMetaBoolean("paid"));
+ layout.addValueMeta(new ValueMetaBinary("payload"));
+ return layout;
+ }
+
+ /** Every fifth row has only nulls apart from its id. */
+ private static List<Object[]> rows() {
+ List<Object[]> rows = new ArrayList<>();
+ for (long i = 0; i < NR_ROWS; i++) {
+ if (i % 5 == 0) {
+ rows.add(new Object[] {null, i, null, null, null, null, null});
+ } else {
+ rows.add(
+ new Object[] {
+ "name-" + i,
+ i,
+ i * 1.25,
+ BigDecimal.valueOf(i, 3),
+ new Date(1_700_000_000_000L + i),
+ i % 2 == 0,
+ new byte[] {(byte) i, (byte) (i >> 8)}
+ });
+ }
+ }
+ return rows;
+ }
+
+ private static void assertRows(List<Object[]> expected, List<Object[]>
actual) {
+ assertEquals(expected.size(), actual.size());
+ for (int i = 0; i < expected.size(); i++) {
+ assertTrue(
+ Arrays.deepEquals(expected.get(i), actual.get(i)),
+ "row " + i + ": " + Arrays.deepToString(actual.get(i)));
+ }
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+}