This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e6dda0598a PgVector Upsert: schema DDL changes commit mode on a shared
connection, fixes #9249 (#9252)
e6dda0598a is described below
commit e6dda0598ad3e883501b384b981cec6536fb04b7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 4 17:29:02 2026 +0100
PgVector Upsert: schema DDL changes commit mode on a shared connection,
fixes #9249 (#9252)
---
.../pgvector/transforms/upsert/PgVectorUpsert.java | 14 +++-
.../upsert/PgVectorUpsertPostgresIT.java | 93 +++++++++++++++++++++-
.../transforms/upsert/PgVectorUpsertTest.java | 21 +++++
3 files changed, 123 insertions(+), 5 deletions(-)
diff --git
a/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
b/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
index 593f3fd249..6f6aebc654 100644
---
a/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
+++
b/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
@@ -317,12 +317,20 @@ public class PgVectorUpsert extends
BaseTransform<PgVectorUpsertMeta, PgVectorUp
* <p>The DDL runs with autocommit on. Inside the data transaction a table
or index created here
* would be undone by the rollback of a single failed row, and an index
would hold its locks for
* the length of the load.
+ *
+ * <p>Not on a connection shared by a transactional pipeline: switching it
to autocommit commits
+ * the open transaction, with the rows the other transforms wrote so far,
and a later rollback of
+ * the pipeline can no longer undo them. There the DDL runs in the
pipeline's transaction (it is
+ * transactional in PostgreSQL), and the commit mode is the pipeline's to
keep.
*/
void prepareSchemaAndCommitMode(String schemaName, String tableName) throws
HopException {
+ boolean sharedConnection =
!Utils.isEmpty(data.database.getConnectionGroup());
// The HNSW index and the mapped-column ALTERs are independent options, so
they have to run on
// an existing table too, not only when this transform creates one.
if (meta.isCreateTableIfMissing() || meta.isCreateHnswIndex() ||
hasColumnMappings()) {
- data.database.setCommit(0);
+ if (!sharedConnection) {
+ data.database.setCommit(0);
+ }
VectorDistanceMetric indexMetric =
meta.getIndexMetric() != null ? meta.getIndexMetric() :
VectorDistanceMetric.COSINE;
PgVectorDatabase.ensureSchema(
@@ -333,7 +341,9 @@ public class PgVectorUpsert extends
BaseTransform<PgVectorUpsertMeta, PgVectorUp
indexMetric,
Const.toInt(resolve(meta.getEmbeddingDimensions()), 768));
}
- data.database.setCommit(data.commitSize);
+ if (!sharedConnection) {
+ data.database.setCommit(data.commitSize);
+ }
}
private void openDatabase() throws HopException {
diff --git
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
index aa60ca1f71..477b89eb82 100644
---
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
+++
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
@@ -26,6 +26,11 @@ import java.util.Collections;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.database.Database;
import org.apache.hop.core.database.DatabaseMeta;
@@ -151,6 +156,21 @@ class PgVectorUpsertPostgresIT {
private static PgVectorUpsert upsert(
String name, String table, int commitSize, Queue<Object[]> input,
List<Object[]> errors)
throws Exception {
+ return upsert(name, table, commitSize, input::poll, errors, false);
+ }
+
+ /**
+ * An upsert into table, with an error hop when errors isn't null, that
creates the table and an
+ * HNSW index when createSchema is set.
+ */
+ private static PgVectorUpsert upsert(
+ String name,
+ String table,
+ int commitSize,
+ Supplier<Object[]> input,
+ List<Object[]> errors,
+ boolean createSchema)
+ throws Exception {
IRowMeta inputRowMeta = inputRowMeta();
PipelineMeta pipelineMeta = new PipelineMeta();
// Not named metadataProvider: inside the transform that name is
BaseTransform's own field.
@@ -163,8 +183,8 @@ class PgVectorUpsertPostgresIT {
meta.setSchemaName("public");
meta.setTableName(table);
meta.setEmbeddingDimensions("2");
- meta.setCreateTableIfMissing(false);
- meta.setCreateHnswIndex(false);
+ meta.setCreateTableIfMissing(createSchema);
+ meta.setCreateHnswIndex(createSchema);
meta.setCommitSize(Integer.toString(commitSize));
TransformMeta transformMeta = new TransformMeta(name, meta);
pipelineMeta.addTransform(transformMeta);
@@ -189,7 +209,7 @@ class PgVectorUpsertPostgresIT {
@Override
public Object[] getRow() {
- Object[] row = input.poll();
+ Object[] row = input.get();
return row == null ? null : row.clone();
}
@@ -283,4 +303,71 @@ class PgVectorUpsertPostgresIT {
assertEquals(ROWS - ROWS / 13, count("SELECT count(*) FROM vec_tx_err"));
assertEquals(ROWS, count("SELECT count(*) FROM vec_tx_plain"));
}
+
+ /**
+ * In a transactional pipeline the schema DDL runs in the pipeline's
transaction. It used to
+ * switch the shared connection to autocommit, which commits the open
transaction: the rows the
+ * other transforms had written so far were committed halfway through the
run, so a later rollback
+ * of the pipeline could no longer undo them.
+ */
+ @Test
+ void theSchemaDdlDoesNotCommitTheTransactionOfAPipeline() throws Exception {
+ createTable("vec_tx_writer");
+ execute("DROP TABLE IF EXISTS vec_tx_ddl");
+ CountDownLatch writerWrote = new CountDownLatch(1);
+ CountDownLatch schemaCreated = new CountDownLatch(1);
+ AtomicLong committedHalfway = new AtomicLong(-1);
+
+ // The writer batches 20 rows into the transaction (commit size 10), then
waits for the schema.
+ AtomicInteger written = new AtomicInteger();
+ Supplier<Object[]> writerInput =
+ () -> {
+ int chunk = written.incrementAndGet();
+ if (chunk <= 20) {
+ return new Object[] {"doc", (long) chunk, "w", "[1,2]"};
+ }
+ if (chunk == 21) {
+ writerWrote.countDown();
+ await(schemaCreated);
+ try {
+ committedHalfway.set(count("SELECT count(*) FROM
vec_tx_writer"));
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return null;
+ };
+
+ // Its first row creates the table and index; asking for the second means
that is done.
+ AtomicInteger asked = new AtomicInteger();
+ Supplier<Object[]> schemaInput =
+ () -> {
+ if (asked.incrementAndGet() == 1) {
+ await(writerWrote);
+ return new Object[] {"doc", 1L, "s", "[1,2]"};
+ }
+ schemaCreated.countDown();
+ return null;
+ };
+
+ SharedConnectionRun run = new SharedConnectionRun().timeoutSeconds(60);
+ run.join(upsert("writer", "vec_tx_writer", 10, writerInput, null, false));
+ run.join(upsert("with schema", "vec_tx_ddl", 10, schemaInput, null, true));
+ assertTrue(run.run(), "the transaction was rolled back");
+
+ assertEquals(0, committedHalfway.get(), "the rows of the writer were
committed halfway");
+ assertEquals(20, count("SELECT count(*) FROM vec_tx_writer"));
+ assertEquals(1, count("SELECT count(*) FROM vec_tx_ddl"));
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ if (!latch.await(30, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("timed out");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
}
diff --git
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
index ec9849cb0d..7d8a4a7857 100644
---
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
+++
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
@@ -306,6 +306,27 @@ class PgVectorUpsertTest {
inOrder.verify(database).setCommit(500);
}
+ @Test
+ void leavesTheCommitModeOfASharedConnectionAlone() throws Exception {
+ // Switching a connection shared by a transactional pipeline to autocommit
commits the rows the
+ // other transforms wrote so far.
+ PgVectorUpsertMeta meta = newMeta();
+ meta.setCreateTableIfMissing(true);
+ meta.setCreateHnswIndex(true);
+ meta.setEmbeddingDimensions("768");
+
+ PgVectorUpsertData data = newData(meta, mock(PreparedStatement.class));
+ data.commitSize = 500;
+ when(database.getConnectionGroup()).thenReturn("pipeline-transaction");
+ PgVectorUpsert transform =
+ new PgVectorUpsert(
+ helper.transformMeta, meta, data, 0, helper.pipelineMeta,
helper.pipeline);
+ transform.prepareSchemaAndCommitMode("public", "chunks");
+
+ verify(database, atLeastOnce()).execStatement(anyString());
+ verify(database, never()).setCommit(anyInt());
+ }
+
@Test
void refusesToDeleteADocumentOnceTheTrackedSetIsFull() throws Exception {
// Evicting an entry would let a document be deleted twice, the second
time taking the chunks