This is an automated email from the ASF dual-hosted git repository. djwang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 2b2959c32dec367ed5a0b7ded39d8ed0d8e1984f Author: huluhuifeng <[email protected]> AuthorDate: Wed Dec 31 07:25:55 2025 +0800 Fix: Add load/performance/extension test groups and align env/test expectations - add bench_test and pxf_extension_test in run_tests.sh, plus matrix entries for bench and pxf_extension in CI - bump surefire heap to 4G to avoid OOM - update gpupgrade expected outputs to new PXF_HOME paths and JSON formatter error text - make ProtocolUtils/HiveBaseTest/JdbcHiveTest/OrcWriteTest/ParquetWriteTest more robust to env defaults (protocol, creds, hive JDBC URL) - keep MultiServerTest running under HDFS with a safe working directory fallback - set distribution key and INSERT pattern for performance test data load --- .../pxf/automation/components/hdfs/Hdfs.java | 62 +++++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java b/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java index ac59a60b..1b422487 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java @@ -570,16 +570,64 @@ public class Hdfs extends BaseSystemObject implements IFSFunctionality { if (parent != null) { fs.mkdirs(parent); } - FSDataOutputStream out = fs.create(datapath, true, - bufferSize, replicationSize, blockSize); - DataOutputStream dos = out; - if (codec != null) { - dos = new DataOutputStream(codec.createOutputStream(out)); + final int maxAttempts = 3; + try { + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + FSDataOutputStream out = null; + DataOutputStream dos = null; + try { + out = fs.create(datapath, true, bufferSize, replicationSize, blockSize); + dos = out; + if (codec != null) { + dos = new DataOutputStream(codec.createOutputStream(out)); + } + writeTableToStream(dos, dataTable, delimiter, encoding, newLine); + return; + } catch (Exception e) { + if (attempt >= maxAttempts || !isRetryableWriteException(e)) { + throw e; + } + + // Best-effort cleanup before retry (handles partially created files) + try { + if (dos != null) { + dos.close(); + } + } catch (Exception ignored) { + } + try { + if (out != null) { + out.close(); + } + } catch (Exception ignored) { + } + try { + fs.delete(datapath, false); + } catch (Exception ignored) { + } + + ReportUtils.report(report, getClass(), + String.format("HDFS write failed (attempt %d/%d), retrying: %s", attempt, maxAttempts, e.getMessage())); + Thread.sleep(2000L * attempt); + } + } + } finally { + ReportUtils.stopLevel(report); } + } - writeTableToStream(dos, dataTable, delimiter, encoding, newLine); - ReportUtils.stopLevel(report); + private boolean isRetryableWriteException(Exception e) { + if (e == null) { + return false; + } + String message = e.getMessage(); + if (message == null) { + return false; + } + // Common transient failure on single-node HDFS when the only DataNode is briefly unavailable/blacklisted + return message.contains("could only be written to 0 of the 1 minReplication nodes") + || message.contains("node(s) are excluded in this operation"); } public void appendTableToFile(String pathToFile, Table dataTable, String delimiter) throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
