This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 29f27f8 Add cleanup method for HadoopSegmentCreationJob (#7371)
29f27f8 is described below
commit 29f27f88abc7759e90eed08e3a3801f3722321f9
Author: Jialiang Li <[email protected]>
AuthorDate: Tue Aug 31 10:11:03 2021 -0700
Add cleanup method for HadoopSegmentCreationJob (#7371)
Co-authored-by: Jack Li(Analytics Engineering) <[email protected]>
---
.../apache/pinot/hadoop/job/HadoopSegmentCreationJob.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
index 474d9a4..06b02f3 100644
---
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
+++
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
@@ -134,9 +134,7 @@ public class HadoopSegmentCreationJob extends
SegmentCreationJob {
moveSegmentsToOutputDir();
- // Delete the staging directory
- _logger.info("Deleting the staging directory: {}", _stagingDir);
- _outputDirFileSystem.delete(new Path(_stagingDir), true);
+ cleanup(job);
}
protected void validateTableConfig(TableConfig tableConfig) {
@@ -177,4 +175,14 @@ public class HadoopSegmentCreationJob extends
SegmentCreationJob {
Path segmentTarDir = new Path(new Path(_stagingDir, "output"),
JobConfigConstants.SEGMENT_TAR_DIR);
movePath(_outputDirFileSystem, segmentTarDir.toString(), _outputDir, true);
}
+
+ /**
+ * Cleans up after the job completes.
+ */
+ protected void cleanup(Job job)
+ throws Exception {
+ // Delete the staging directory
+ _logger.info("Deleting the staging directory: {}", _stagingDir);
+ _outputDirFileSystem.delete(new Path(_stagingDir), true);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
