This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0a951fa7969 Add Storage Quota Check to Batch Segments Upload (#17512)
0a951fa7969 is described below
commit 0a951fa7969b79229a9cf78f6afc807833f6b2ff
Author: Arunkumar Saravanan <[email protected]>
AuthorDate: Thu Jan 29 23:46:27 2026 +0530
Add Storage Quota Check to Batch Segments Upload (#17512)
---
.../PinotSegmentUploadDownloadRestletResource.java | 26 +++++++++++++++++++---
1 file changed, 23 insertions(+), 3 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
index 9f2771cb3f9..bb9c2151410 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
@@ -627,8 +627,16 @@ public class PinotSegmentUploadDownloadRestletResource {
SegmentValidationUtils.validateTimeInterval(segmentMetadata,
tableConfig);
}
// TODO: Include the un-tarred segment size when using the METADATA
push rest API. Currently we can only use the
- // tarred segment size as an approximation. Additionally, add the
storage quota check for batch upload mode.
+ // tarred segment size as an approximation.
long segmentSizeInBytes = getSegmentSizeFromFile(sourceDownloadURIStr);
+ if (segmentSizeInBytes > 0) {
+ // Only check storage quota when segment size is available
+ SegmentValidationUtils.checkStorageQuota(segmentName,
segmentSizeInBytes, segmentSizeInBytes, tableConfig,
+ _storageQuotaChecker);
+ } else {
+ LOGGER.warn("Skipping storage quota check for segment: {} of table:
{} as segment size is unavailable",
+ segmentName, tableNameWithType);
+ }
// Encrypt segment
String crypterNameInTableConfig =
tableConfig.getValidationConfig().getCrypterClassName();
@@ -667,8 +675,20 @@ public class PinotSegmentUploadDownloadRestletResource {
} catch (Exception e) {
_controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR,
segmentUploadMetadataList.size());
- throw new ControllerApplicationException(LOGGER,
- "Exception while processing segments to upload: " + e.getMessage(),
Response.Status.INTERNAL_SERVER_ERROR, e);
+ _controllerMetrics.addMeteredTableValue(tableName,
ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR,
+ segmentUploadMetadataList.size());
+ if (e instanceof WebApplicationException) {
+ if (((WebApplicationException) e).getResponse().getStatus()
+ == Response.Status.FORBIDDEN.getStatusCode()) {
+ LOGGER.error("Segment upload forbidden for segments: {} of table:
{}",
+ segmentNames, tableNameWithType);
+ }
+ throw (WebApplicationException) e;
+ } else {
+ throw new ControllerApplicationException(LOGGER,
+ "Exception while processing segments to upload: " +
e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR,
+ e);
+ }
} finally {
cleanupTempFiles(tempFiles);
multiPart.cleanup();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
