This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9a85e0430a Fix OpenLineage IT test (#8102)
9a85e0430a is described below
commit 9a85e0430a651c0bd75a57235f9caf5e806aca95
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Aug 26 12:43:06 2026 +0200
Fix OpenLineage IT test (#8102)
Caused by shared postgresql server with the dbt integration tests.
---
.../integration-tests-openlineage.yaml | 12 +++-
.../openlineage-warehouse-startup.sql | 67 ----------------------
.../main-0002-openlineage-warehouse.hwf | 25 +++++++-
.../openlineage/main-0003-openlineage-writers.hwf | 25 +++++++-
4 files changed, 57 insertions(+), 72 deletions(-)
diff --git a/docker/integration-tests/integration-tests-openlineage.yaml
b/docker/integration-tests/integration-tests-openlineage.yaml
index b89228dd4f..303a5d1a33 100644
--- a/docker/integration-tests/integration-tests-openlineage.yaml
+++ b/docker/integration-tests/integration-tests-openlineage.yaml
@@ -45,6 +45,15 @@ services:
# "Warehouse" Postgres that the test pipelines read from and write to. Its
host:port
# ("postgres":5432) is what the warehouse dataset namespace is keyed on
# (postgres://postgres:5432), so it must match run-tests.sh's POSTGRES_HOST
default.
+ #
+ # The schema is created by the workflows themselves (a "Create warehouse
tables" SQL action),
+ # not by a /docker-entrypoint-initdb.d script. Every project compose file in
this directory
+ # shares one compose project name, so "postgres" is the same container and
the same data volume
+ # for all of them, and nothing is torn down between projects. An init script
therefore runs only
+ # for whichever project happens to start Postgres first - after that the
data dir is no longer
+ # empty and the script is skipped, leaving these tests to read whatever the
previous project
+ # left behind (integration-tests/dbt drops and recreates
public.orders_source with its own
+ # columns).
postgres:
image: postgres:14
environment:
@@ -57,9 +66,6 @@ services:
timeout: 5s
retries: 5
start_period: 10s
- volumes:
- # Seeds public.orders_source and creates public.orders_target on first
init.
- -
./openlineage-warehouse-startup.sql:/docker-entrypoint-initdb.d/01-warehouse.sql:ro
marquez-db:
image: postgres:14
diff --git a/docker/integration-tests/openlineage-warehouse-startup.sql
b/docker/integration-tests/openlineage-warehouse-startup.sql
deleted file mode 100644
index 5cf7fed1ad..0000000000
--- a/docker/integration-tests/openlineage-warehouse-startup.sql
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-**/
-
--- Seed the "warehouse" Postgres used by the OpenLineage warehouse-lineage
integration test.
--- The source table is populated; the target table is created empty for Table
Output to fill.
--- Runs automatically on first cluster init (mounted into
/docker-entrypoint-initdb.d).
-
-CREATE TABLE IF NOT EXISTS public.orders_source (
- id INTEGER,
- amount NUMERIC(10, 2)
-);
-
-INSERT INTO public.orders_source (id, amount) VALUES
- (1, 10.00),
- (2, 20.00),
- (3, 30.00);
-
-CREATE TABLE IF NOT EXISTS public.orders_target (
- id INTEGER,
- amount NUMERIC(10, 2)
-);
-
--- Targets for the per-writer column-lineage tests (Insert/Update, Combination
Lookup,
--- Dimension Lookup, PostgreSQL bulk loader). Each is filled by reading
orders_source.
-CREATE TABLE IF NOT EXISTS public.orders_upsert (
- id INTEGER,
- amount NUMERIC(10, 2)
-);
-
-CREATE TABLE IF NOT EXISTS public.orders_bulk (
- id INTEGER,
- amount NUMERIC(10, 2)
-);
-
--- Combination Lookup junk dimension: technical key + the business-key columns.
-CREATE TABLE IF NOT EXISTS public.orders_combi (
- combi_tk INTEGER,
- id INTEGER,
- amount NUMERIC(10, 2)
-);
-
--- Dimension Lookup slowly-changing dimension: technical key, version,
validity dates,
--- the natural key and the dimension attribute.
-CREATE TABLE IF NOT EXISTS public.orders_dim (
- dim_tk INTEGER,
- version INTEGER,
- date_from TIMESTAMP,
- date_to TIMESTAMP,
- id INTEGER,
- amount NUMERIC(10, 2)
-);
diff --git a/integration-tests/openlineage/main-0002-openlineage-warehouse.hwf
b/integration-tests/openlineage/main-0002-openlineage-warehouse.hwf
index 8c6142128e..ed56218552 100644
--- a/integration-tests/openlineage/main-0002-openlineage-warehouse.hwf
+++ b/integration-tests/openlineage/main-0002-openlineage-warehouse.hwf
@@ -49,6 +49,22 @@ limitations under the License.
<yloc>48</yloc>
<attributes_hac/>
</action>
+ <action>
+ <name>Create warehouse tables</name>
+ <description>Seeds public.orders_source and (re)creates the targets this
workflow writes. The warehouse Postgres is shared with the other
integration-test projects and is not torn down between them, so the tables are
dropped and recreated here rather than seeded once on container init - another
project may have left a table of the same name with different columns
behind.</description>
+ <type>SQL</type>
+ <attributes/>
+ <sql>DROP TABLE IF EXISTS public.orders_daily ; DROP TABLE IF
EXISTS public.orders_target ; DROP TABLE IF EXISTS
public.orders_source ; CREATE TABLE public.orders_source (id INTEGER,
amount NUMERIC(10, 2)) ; INSERT INTO public.orders_source (id, amount)
VALUES (1, 10.00), (2, 20.00), (3, 30.00) ; CREATE TABLE
public.orders_target (id INTEGER, amount NUMERIC(10, 2)) ; </sql>
+ <useVariableSubstitution>T</useVariableSubstitution>
+ <sqlfromfile>F</sqlfromfile>
+ <sqlfilename/>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>warehouse</connection>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
<action>
<name>0003-openlineage-warehouse.hpl</name>
<description/>
@@ -215,11 +231,18 @@ limitations under the License.
<hops>
<hop>
<from>Start</from>
- <to>0003-openlineage-warehouse.hpl</to>
+ <to>Create warehouse tables</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
+ <hop>
+ <from>Create warehouse tables</from>
+ <to>0003-openlineage-warehouse.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
<hop>
<from>0003-openlineage-warehouse.hpl</from>
<to>0005-openlineage-warehouse-execsql.hpl</to>
diff --git a/integration-tests/openlineage/main-0003-openlineage-writers.hwf
b/integration-tests/openlineage/main-0003-openlineage-writers.hwf
index 7408ee464c..b759977886 100644
--- a/integration-tests/openlineage/main-0003-openlineage-writers.hwf
+++ b/integration-tests/openlineage/main-0003-openlineage-writers.hwf
@@ -49,6 +49,22 @@ limitations under the License.
<yloc>48</yloc>
<attributes_hac/>
</action>
+ <action>
+ <name>Create warehouse tables</name>
+ <description>Seeds public.orders_source and (re)creates the four writer
targets. The warehouse Postgres is shared with the other integration-test
projects and is not torn down between them, so the tables are dropped and
recreated here rather than seeded once on container init - another project may
have left a table of the same name with different columns behind.</description>
+ <type>SQL</type>
+ <attributes/>
+ <sql>DROP TABLE IF EXISTS public.orders_upsert ; DROP TABLE IF
EXISTS public.orders_combi ; DROP TABLE IF EXISTS
public.orders_dim ; DROP TABLE IF EXISTS
public.orders_bulk ; DROP TABLE IF EXISTS
public.orders_source ; CREATE TABLE public.orders_source (id INTEGER,
amount NUMERIC(10, 2)) ; INSERT INTO public.orders_source (id, amount)
VALUES (1, 10.00), (2, 20.00), (3, 30.00) ; CREATE TABLE
public.orders_upsert (id INTEGER, am [...]
+ <useVariableSubstitution>T</useVariableSubstitution>
+ <sqlfromfile>F</sqlfromfile>
+ <sqlfilename/>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>warehouse</connection>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
<action>
<name>0007-openlineage-insertupdate.hpl</name>
<description/>
@@ -309,11 +325,18 @@ limitations under the License.
<hops>
<hop>
<from>Start</from>
- <to>0007-openlineage-insertupdate.hpl</to>
+ <to>Create warehouse tables</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
+ <hop>
+ <from>Create warehouse tables</from>
+ <to>0007-openlineage-insertupdate.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
<hop>
<from>0007-openlineage-insertupdate.hpl</from>
<to>0008-openlineage-combinationlookup.hpl</to>
