This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 19be14f9e67c42b3787176ecab9d2ba32beff938
Author: qianhao.zhou <z.qian...@gmail.com>
AuthorDate: Thu Jan 5 18:48:38 2023 +0800
KYLIN-5449 fix storage quota issue
Co-authored-by: qhzhou <qianhao.z...@kyligence.io>
---
.../org/apache/kylin/rest/service/ProjectService.java | 5 ++---
.../kylin/job/runners/QuotaStorageCheckRunner.java | 4 +---
.../metadata/cube/storage/GarbageStorageCollector.java | 5 +++++
.../cube/storage/ProjectStorageInfoCollector.java | 16 +++++++---------
.../metadata/cube/storage/StorageInfoCollector.java | 2 ++
.../metadata/cube/storage/StorageQuotaCollector.java | 5 +++++
.../metadata/cube/storage/TotalStorageCollector.java | 5 +++++
.../cube/storage/ProjectStorageInfoCollectorTest.java | 12 +++++-------
8 files changed, 32 insertions(+), 22 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 393d7a0653..1af9955759 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -330,9 +330,8 @@ public class ProjectService extends BasicService {
public StorageVolumeInfoResponse getStorageVolumeInfoResponse(String
project) {
val response = new StorageVolumeInfoResponse();
- val storageInfoEnumList =
Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE,
StorageInfoEnum.STORAGE_QUOTA,
- StorageInfoEnum.TOTAL_STORAGE);
- val collector = new ProjectStorageInfoCollector(storageInfoEnumList);
+ val collector = new
ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE,
StorageInfoEnum.STORAGE_QUOTA,
+ StorageInfoEnum.TOTAL_STORAGE));
val storageVolumeInfo = collector.getStorageVolumeInfo(getConfig(),
project);
response.setGarbageStorageSize(storageVolumeInfo.getGarbageStorageSize());
response.setStorageQuotaSize(storageVolumeInfo.getStorageQuotaSize());
diff --git
a/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java
b/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java
index c7f3140671..a69ea16c87 100644
---
a/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java
+++
b/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java
@@ -38,9 +38,7 @@ public class QuotaStorageCheckRunner extends
AbstractDefaultSchedulerRunner {
public QuotaStorageCheckRunner(NDefaultScheduler nDefaultScheduler) {
super(nDefaultScheduler);
-
- val storageInfoEnumList =
Lists.newArrayList(StorageInfoEnum.STORAGE_QUOTA,
StorageInfoEnum.TOTAL_STORAGE);
- collector = new ProjectStorageInfoCollector(storageInfoEnumList);
+ collector = new
ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.STORAGE_QUOTA,
StorageInfoEnum.TOTAL_STORAGE));
}
@Override
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
index b1e0856197..12f0a21212 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
@@ -61,6 +61,11 @@ public class GarbageStorageCollector implements
StorageInfoCollector {
storageVolumeInfo.setGarbageStorageSize(storageSize);
}
+ @Override
+ public StorageInfoEnum getType() {
+ return StorageInfoEnum.GARBAGE_STORAGE;
+ }
+
private List<NDataModel> getModels(String project) {
val dataflowManager =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
return dataflowManager.listUnderliningDataModels();
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
index ea864bf15f..1148d304e3 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
@@ -24,17 +24,15 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
public class ProjectStorageInfoCollector {
private List<StorageInfoCollector> collectors = Lists.newArrayList();
- private static final ImmutableMap<Class<?>, StorageInfoEnum> collectorType
= ImmutableMap
- .<Class<?>, StorageInfoEnum>
builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE)
- .put(TotalStorageCollector.class, StorageInfoEnum.TOTAL_STORAGE)
- .put(StorageQuotaCollector.class,
StorageInfoEnum.STORAGE_QUOTA).build();
+ private static GarbageStorageCollector garbageStorageCollector = new
GarbageStorageCollector();
+ private static TotalStorageCollector totalStorageCollector = new
TotalStorageCollector();
+ private static StorageQuotaCollector storageQuotaCollector = new
StorageQuotaCollector();
public ProjectStorageInfoCollector(List<StorageInfoEnum> storageInfoList) {
if (CollectionUtils.isNotEmpty(storageInfoList)) {
@@ -47,7 +45,7 @@ public class ProjectStorageInfoCollector {
try {
collector.collect(config, project, storageVolumeInfo);
} catch (Exception e) {
-
storageVolumeInfo.getThrowableMap().put(collectorType.get(collector.getClass()),
e);
+ storageVolumeInfo.getThrowableMap().put(collector.getType(),
e);
}
}
}
@@ -55,13 +53,13 @@ public class ProjectStorageInfoCollector {
private void addCollectors(StorageInfoEnum storageInfoEnum) {
switch (storageInfoEnum) {
case GARBAGE_STORAGE:
- collectors.add(new GarbageStorageCollector());
+ collectors.add(garbageStorageCollector);
break;
case TOTAL_STORAGE:
- collectors.add(new TotalStorageCollector());
+ collectors.add(totalStorageCollector);
break;
case STORAGE_QUOTA:
- collectors.add(new StorageQuotaCollector());
+ collectors.add(storageQuotaCollector);
break;
default:
break;
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
index 73c63f13ac..6cd330804d 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
@@ -32,4 +32,6 @@ public interface StorageInfoCollector {
}
void doCollect(KylinConfig config, String project, StorageVolumeInfo
storageVolumeInfo) throws IOException;
+
+ StorageInfoEnum getType();
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
index 8b04f05164..306d77f9af 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
@@ -32,4 +32,9 @@ public class StorageQuotaCollector implements
StorageInfoCollector {
storageVolumeInfo.setStorageQuotaSize(quotaSize);
}
+ @Override
+ public StorageInfoEnum getType() {
+ return StorageInfoEnum.STORAGE_QUOTA;
+ }
+
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
index 74c30ab86a..92d38355ad 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
@@ -50,4 +50,9 @@ public class TotalStorageCollector implements
StorageInfoCollector {
storageVolumeInfo.setTotalStorageSize(totalStorageSize);
}
+ @Override
+ public StorageInfoEnum getType() {
+ return StorageInfoEnum.TOTAL_STORAGE;
+ }
+
}
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
index eb79e97fdb..95be8e38be 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.cube.storage;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -87,9 +88,8 @@ public class ProjectStorageInfoCollectorTest extends
NLocalFileMetadataTestCase
getTestConfig().setProperty("kylin.metadata.semi-automatic-mode",
"true");
initTestData();
- val storageInfoEnumList =
Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE,
StorageInfoEnum.STORAGE_QUOTA,
- StorageInfoEnum.TOTAL_STORAGE);
- val collector = new ProjectStorageInfoCollector(storageInfoEnumList);
+ val collector = new
ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE,
StorageInfoEnum.STORAGE_QUOTA,
+ StorageInfoEnum.TOTAL_STORAGE));
val volumeInfo = collector.getStorageVolumeInfo(getTestConfig(),
DEFAULT_PROJECT);
Assert.assertEquals(10240L * 1024 * 1024 * 1024,
volumeInfo.getStorageQuotaSize());
@@ -421,8 +421,7 @@ public class ProjectStorageInfoCollectorTest extends
NLocalFileMetadataTestCase
@Test
public void testGetStorageVolumeInfoEmpty() {
- List<StorageInfoEnum> storageInfoEnumList = Lists.newArrayList();
- val collector = new ProjectStorageInfoCollector(storageInfoEnumList);
+ val collector = new
ProjectStorageInfoCollector(Collections.emptyList());
val storageVolumeInfo =
collector.getStorageVolumeInfo(getTestConfig(), DEFAULT_PROJECT);
Assert.assertEquals(-1L, storageVolumeInfo.getStorageQuotaSize());
@@ -433,9 +432,8 @@ public class ProjectStorageInfoCollectorTest extends
NLocalFileMetadataTestCase
@Test
public void testGetStorageVolumeException() throws NoSuchFieldException,
IllegalAccessException, IOException {
- List<StorageInfoEnum> storageInfoEnumList = Lists.newArrayList();
TotalStorageCollector totalStorageCollector =
Mockito.spy(TotalStorageCollector.class);
- ProjectStorageInfoCollector collector = new
ProjectStorageInfoCollector(storageInfoEnumList);
+ ProjectStorageInfoCollector collector = new
ProjectStorageInfoCollector(Collections.emptyList());
Field field = collector.getClass().getDeclaredField("collectors");
Unsafe.changeAccessibleObject(field, true);
List<StorageInfoCollector> collectors = (List<StorageInfoCollector>)
field.get(collector);
