This is an automated email from the ASF dual-hosted git repository.
ostinru pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new f200e802 Reduce amount of tests we run in CI (#139)
f200e802 is described below
commit f200e80283ee2afdaa451f1289f9089df80a07bd
Author: Nikolay Antonov <[email protected]>
AuthorDate: Fri Jul 31 12:48:59 2026 +0500
Reduce amount of tests we run in CI (#139)
In #136 it is planned to add 20+ Github Action Steps. And we agreed (#113)
that we should keep number of tests we run under control. This PR reduces
amount of test steps we run:
1. Remove `sanity` tests group (few tests here)
2. Remove `jdbc` test group (no tests in the group)
3. Run 'features' and 'gpdb' tests together - test groups shares 80% of the
tests:
| Group | Tests count |
|--------|--------|
| `features` total | 329 |
| `gpdb` total | 321 |
| `features` & `gpdb` | 264 |
| `features` only | 65 |
| `gpdb` only | 57 |
---
.github/workflows/pxf-ci.yml | 10 +--
.../automation/features/hdfs/HdfsAnalyzeTest.java | 6 +-
.../features/hdfs/HdfsReadableTextTest.java | 2 +-
.../pxf/automation/features/hive/HiveTest.java | 4 +-
.../automation/features/profiles/ProfilesTest.java | 2 +-
ci/docker/pxf-cbdb-dev/common/script/run_tests.sh | 76 +++++-----------------
6 files changed, 29 insertions(+), 71 deletions(-)
diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml
index 9ea85aad..2c440801 100644
--- a/.github/workflows/pxf-ci.yml
+++ b/.github/workflows/pxf-ci.yml
@@ -319,7 +319,6 @@ jobs:
- cli
- external-table
- fdw
- - sanity
- smoke
- hdfs
- hcatalog
@@ -327,12 +326,10 @@ jobs:
- hive
- hbase
- profile
- - jdbc
- proxy
- unused
- features
- - gpdb
- - gpdb_fdw
+ - features_fdw
- load
steps:
- name: Free disk space
@@ -515,7 +512,6 @@ jobs:
- external-table
- fdw
- server
- - sanity
- smoke
- hdfs
- hcatalog
@@ -523,12 +519,10 @@ jobs:
- hive
- hbase
- profile
- - jdbc
- proxy
- unused
- features
- - gpdb
- - gpdb_fdw
+ - features_fdw
- load
steps:
- name: Free disk space
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsAnalyzeTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsAnalyzeTest.java
index cb284517..692298a4 100755
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsAnalyzeTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsAnalyzeTest.java
@@ -1,5 +1,6 @@
package org.apache.cloudberry.pxf.automation.features.hdfs;
+import annotations.SkipForFDW;
import annotations.WorksWithFDW;
import org.apache.cloudberry.pxf.automation.components.cluster.PhdCluster;
import
org.apache.cloudberry.pxf.automation.datapreparer.CustomSequencePreparer;
@@ -219,6 +220,7 @@ public class HdfsAnalyzeTest extends BaseFeature {
* @throws Exception if test failed to run
*/
@Test(groups = { "features" })
+ @SkipForFDW // relies on explicit fragmenter/accessor/resolver without a
profile, unsupported by FDW
public void analyzeOnSequenceSmall() throws Exception {
String path = hdfs.getWorkingDirectory()
@@ -268,6 +270,7 @@ public class HdfsAnalyzeTest extends BaseFeature {
* @throws Exception if test failed to run
*/
@Test(groups = { "features" })
+ @SkipForFDW // relies on explicit fragmenter/accessor/resolver without a
profile, unsupported by FDW
public void negativeAnalyzeFailOnFragmenter() throws Exception {
String csvPath = hdfs.getWorkingDirectory() +
"/analyze_nofragmenter.csv";
@@ -323,7 +326,6 @@ public class HdfsAnalyzeTest extends BaseFeature {
* @throws Exception if test failed to run
*/
@Test(groups = { "features" }, enabled = false)
-
public void negativeAnalyzeFailOnEstimate() throws Exception {
String csvPath = hdfs.getWorkingDirectory() + "/no_such_file.csv";
@@ -375,6 +377,7 @@ public class HdfsAnalyzeTest extends BaseFeature {
* @throws Exception test failed to run
*/
@Test(groups = { "features" })
+ @SkipForFDW // relies on explicit fragmenter/accessor/resolver without a
profile, unsupported by FDW
public void negativeAnalyzeFailOnResolver() throws Exception {
String csvPath = hdfs.getWorkingDirectory() +
"/analyze_noresolver.csv";
@@ -454,6 +457,7 @@ public class HdfsAnalyzeTest extends BaseFeature {
* @throws Exception if test failed to run
*/
@Test(groups = { "features" })
+ @SkipForFDW // relies on explicit fragmenter/accessor/resolver without a
profile, unsupported by FDW
public void negativeAnalyzeFailAfter10000Rows() throws Exception {
String dataPath = hdfs.getWorkingDirectory()
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsReadableTextTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsReadableTextTest.java
index 9d6bbc5f..9d7ec52f 100755
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsReadableTextTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hdfs/HdfsReadableTextTest.java
@@ -130,7 +130,7 @@ public class HdfsReadableTextTest extends BaseFeature {
* format.
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"features", "sanity", "gpdb", "security"})
+ @Test(enabled = false, groups = {"features", "gpdb", "security"})
public void readDelimitedTextUsingTextFormat() throws Exception {
// set plugins and delimiter
exTable.setFragmenter("org.apache.cloudberry.pxf.plugins.hdfs.HdfsDataFragmenter");
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hive/HiveTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hive/HiveTest.java
index 283cde66..5b86be67 100755
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hive/HiveTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hive/HiveTest.java
@@ -155,7 +155,7 @@ public class HiveTest extends HiveBaseTest {
*
* @throws Exception if test fails to run
*/
- @Test(groups = {"hive", "sanity", "features", "gpdb", "security"})
+ @Test(groups = {"hive", "features", "gpdb", "security"})
public void sanity() throws Exception {
createExternalTable(PXF_HIVE_SMALL_DATA_TABLE,
PXF_HIVE_SMALLDATA_COLS, hiveSmallDataTable);
@@ -951,7 +951,7 @@ public class HiveTest extends HiveBaseTest {
*
* @throws Exception if test fails to run
*/
- @Test(groups = {"features", "hcatalog", "sanity"})
+ @Test(groups = {"features", "hcatalog"})
public void describeHiveTable() throws Exception {
prepareNonDefaultSchemaData();
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/profiles/ProfilesTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/profiles/ProfilesTest.java
index b9be99a8..da2eee9f 100755
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/profiles/ProfilesTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/profiles/ProfilesTest.java
@@ -80,7 +80,7 @@ public class ProfilesTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = {"sanity"})
+ @Test(groups = {"unused"})
protected void sanity() throws Exception {
gpdb.createTableAndVerify(exTable);
diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
index 2bca3ce8..bca400d0 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
@@ -236,10 +236,6 @@ base_test(){
export PATH="${GPHOME}/bin:${PATH}"
ensure_testuser_pg_hba
- make GROUP="sanity" || true
- save_test_reports "sanity"
- echo "[run_tests] GROUP=sanity finished"
-
make GROUP="smoke" || true
save_test_reports "smoke"
echo "[run_tests] GROUP=smoke finished"
@@ -273,10 +269,6 @@ base_test(){
save_test_reports "profile"
echo "[run_tests] GROUP=profile finished"
- make GROUP="jdbc" || true
- save_test_reports "jdbc"
- echo "[run_tests] GROUP=jdbc finished"
-
make GROUP="proxy" || true
save_test_reports "proxy"
echo "[run_tests] GROUP=proxy finished"
@@ -343,6 +335,7 @@ ensure_testplugin_jar() {
}
feature_test(){
+ local use_fdw="${1:-false}"
# Ensure PXF CLI is available for gpupgrade tests and sanity checks
export PXF_HOME=${PXF_HOME:-/usr/local/pxf}
export PATH="${PXF_HOME}/bin:${PATH}"
@@ -363,49 +356,18 @@ feature_test(){
configure_pxf_default_hdfs_server
- export PROTOCOL=
- make GROUP="features" || true
- save_test_reports "features"
- echo "[run_tests] GROUP=features finished"
-}
-
-gpdb_test() {
- local use_fdw="$1"
export PROTOCOL=HDFS
- export PXF_HOME=${PXF_HOME:-/usr/local/pxf}
- export PATH="${PXF_HOME}/bin:${PATH}"
- ensure_gpupgrade_helpers
- ensure_testplugin_jar
-
- # Make sure core services are alive before preparing configs
- health_check_with_retry || true
-
- export PGHOST=127.0.0.1
- export PATH="${GPHOME}/bin:${PATH}"
- ensure_testuser_pg_hba
- # Clean stale state from previous runs so gpdb suite starts fresh
- cleanup_hdfs_test_data
- hdfs dfs -rm -r -f /tmp/pxf_automation_data >/dev/null 2>&1 || true
- cleanup_hive_state
- cleanup_hbase_state
-
- # Ensure PXF points to local HDFS/Hive/HBase configs
- configure_pxf_default_hdfs_server
-
- local extra_args=""
+ local extra_args="USE_FDW=false"
+ local report_name="features"
if [[ "$use_fdw" == "true" ]]; then
extra_args="USE_FDW=true"
- else
- extra_args="USE_FDW=false"
+ report_name="features_fdw"
fi
- echo "[run_tests] Starting GROUP=gpdb $extra_args"
- make GROUP="gpdb" $extra_args || true
- if [[ "$use_fdw" == "true" ]]; then
- save_test_reports "gpdb_fdw"
- else
- save_test_reports "gpdb"
- fi
- echo "[run_tests] GROUP=gpdb $extra_args finished"
+ # "features" and "gpdb" tags largely overlap; union them so each test
+ # runs once per USE_FDW mode instead of once per tag
+ make GROUP="features,gpdb" $extra_args || true
+ save_test_reports "$report_name"
+ echo "[run_tests] GROUP=features,gpdb $extra_args finished"
}
bench_prepare_env() {
@@ -504,7 +466,7 @@ generate_test_summary() {
local group=$(basename "$group_dir")
# Skip if it's not a test group directory
- [[ "$group" =~
^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|features|load|performance|fdw|gpdb_fdw)$
]] || continue
+ [[ "$group" =~
^(smoke|hcatalog|hcfs|hdfs|hive|hbase|profile|proxy|unused|features|features_fdw|load|performance|fdw)$
]] || continue
echo "Processing $group test reports from $group_dir"
@@ -640,9 +602,6 @@ run_single_group() {
cd "${REPO_ROOT}/fdw"
make test
;;
- gpdb_fdw)
- gpdb_test "true"
- ;;
server)
cd "${REPO_ROOT}/server"
./gradlew test
@@ -663,10 +622,10 @@ run_single_group() {
save_test_reports "hbase"
;;
features)
- feature_test
+ feature_test "false"
;;
- gpdb)
- gpdb_test "false"
+ features_fdw)
+ feature_test "true"
;;
load)
bench_prepare_env
@@ -682,14 +641,14 @@ run_single_group() {
make GROUP="proxy"
save_test_reports "proxy"
;;
- sanity|smoke|hdfs|hcatalog|hcfs|profile|jdbc|unused)
+ smoke|hdfs|hcatalog|hcfs|profile|unused)
export PROTOCOL=
make GROUP="$group"
save_test_reports "$group"
;;
*)
echo "Unknown test group: $group"
- echo "Available groups: cli, external-table, fdw, server, sanity, smoke,
hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, features,
gpdb, gpdb_fdw, load, performance, bench"
+ echo "Available groups: cli, external-table, fdw, server, smoke, hdfs,
hcatalog, hcfs, hive, hbase, profile, proxy, unused, features, features_fdw,
load, performance, bench"
exit 1
;;
esac
@@ -713,8 +672,9 @@ main() {
# Run base tests (includes smoke, hdfs, hcatalog, hcfs, hive, etc.)
base_test
- # Run feature tests (includes features, gpdb)
- feature_test
+ # Run feature tests (union of features + gpdb tags, once per USE_FDW mode)
+ feature_test "false"
+ feature_test "true"
# Run bench tests (includes load, performance)
bench_test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
