This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 19be14f9e67c42b3787176ecab9d2ba32beff938 Author: qianhao.zhou <z.qian...@gmail.com> AuthorDate: Thu Jan 5 18:48:38 2023 +0800 KYLIN-5449 fix storage quota issue Co-authored-by: qhzhou <qianhao.z...@kyligence.io> --- .../org/apache/kylin/rest/service/ProjectService.java | 5 ++--- .../kylin/job/runners/QuotaStorageCheckRunner.java | 4 +--- .../metadata/cube/storage/GarbageStorageCollector.java | 5 +++++ .../cube/storage/ProjectStorageInfoCollector.java | 16 +++++++--------- .../metadata/cube/storage/StorageInfoCollector.java | 2 ++ .../metadata/cube/storage/StorageQuotaCollector.java | 5 +++++ .../metadata/cube/storage/TotalStorageCollector.java | 5 +++++ .../cube/storage/ProjectStorageInfoCollectorTest.java | 12 +++++------- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java index 393d7a0653..1af9955759 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java @@ -330,9 +330,8 @@ public class ProjectService extends BasicService { public StorageVolumeInfoResponse getStorageVolumeInfoResponse(String project) { val response = new StorageVolumeInfoResponse(); - val storageInfoEnumList = Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE, StorageInfoEnum.STORAGE_QUOTA, - StorageInfoEnum.TOTAL_STORAGE); - val collector = new ProjectStorageInfoCollector(storageInfoEnumList); + val collector = new ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE, StorageInfoEnum.STORAGE_QUOTA, + StorageInfoEnum.TOTAL_STORAGE)); val storageVolumeInfo = collector.getStorageVolumeInfo(getConfig(), project); response.setGarbageStorageSize(storageVolumeInfo.getGarbageStorageSize()); response.setStorageQuotaSize(storageVolumeInfo.getStorageQuotaSize()); diff --git a/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java b/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java index c7f3140671..a69ea16c87 100644 --- a/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java +++ b/src/core-job/src/main/java/org/apache/kylin/job/runners/QuotaStorageCheckRunner.java @@ -38,9 +38,7 @@ public class QuotaStorageCheckRunner extends AbstractDefaultSchedulerRunner { public QuotaStorageCheckRunner(NDefaultScheduler nDefaultScheduler) { super(nDefaultScheduler); - - val storageInfoEnumList = Lists.newArrayList(StorageInfoEnum.STORAGE_QUOTA, StorageInfoEnum.TOTAL_STORAGE); - collector = new ProjectStorageInfoCollector(storageInfoEnumList); + collector = new ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.STORAGE_QUOTA, StorageInfoEnum.TOTAL_STORAGE)); } @Override diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java index b1e0856197..12f0a21212 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java @@ -61,6 +61,11 @@ public class GarbageStorageCollector implements StorageInfoCollector { storageVolumeInfo.setGarbageStorageSize(storageSize); } + @Override + public StorageInfoEnum getType() { + return StorageInfoEnum.GARBAGE_STORAGE; + } + private List<NDataModel> getModels(String project) { val dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project); return dataflowManager.listUnderliningDataModels(); diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java index ea864bf15f..1148d304e3 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java @@ -24,17 +24,15 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; public class ProjectStorageInfoCollector { private List<StorageInfoCollector> collectors = Lists.newArrayList(); - private static final ImmutableMap<Class<?>, StorageInfoEnum> collectorType = ImmutableMap - .<Class<?>, StorageInfoEnum> builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE) - .put(TotalStorageCollector.class, StorageInfoEnum.TOTAL_STORAGE) - .put(StorageQuotaCollector.class, StorageInfoEnum.STORAGE_QUOTA).build(); + private static GarbageStorageCollector garbageStorageCollector = new GarbageStorageCollector(); + private static TotalStorageCollector totalStorageCollector = new TotalStorageCollector(); + private static StorageQuotaCollector storageQuotaCollector = new StorageQuotaCollector(); public ProjectStorageInfoCollector(List<StorageInfoEnum> storageInfoList) { if (CollectionUtils.isNotEmpty(storageInfoList)) { @@ -47,7 +45,7 @@ public class ProjectStorageInfoCollector { try { collector.collect(config, project, storageVolumeInfo); } catch (Exception e) { - storageVolumeInfo.getThrowableMap().put(collectorType.get(collector.getClass()), e); + storageVolumeInfo.getThrowableMap().put(collector.getType(), e); } } } @@ -55,13 +53,13 @@ public class ProjectStorageInfoCollector { private void addCollectors(StorageInfoEnum storageInfoEnum) { switch (storageInfoEnum) { case GARBAGE_STORAGE: - collectors.add(new GarbageStorageCollector()); + collectors.add(garbageStorageCollector); break; case TOTAL_STORAGE: - collectors.add(new TotalStorageCollector()); + collectors.add(totalStorageCollector); break; case STORAGE_QUOTA: - collectors.add(new StorageQuotaCollector()); + collectors.add(storageQuotaCollector); break; default: break; diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java index 73c63f13ac..6cd330804d 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java @@ -32,4 +32,6 @@ public interface StorageInfoCollector { } void doCollect(KylinConfig config, String project, StorageVolumeInfo storageVolumeInfo) throws IOException; + + StorageInfoEnum getType(); } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java index 8b04f05164..306d77f9af 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java @@ -32,4 +32,9 @@ public class StorageQuotaCollector implements StorageInfoCollector { storageVolumeInfo.setStorageQuotaSize(quotaSize); } + @Override + public StorageInfoEnum getType() { + return StorageInfoEnum.STORAGE_QUOTA; + } + } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java index 74c30ab86a..92d38355ad 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java @@ -50,4 +50,9 @@ public class TotalStorageCollector implements StorageInfoCollector { storageVolumeInfo.setTotalStorageSize(totalStorageSize); } + @Override + public StorageInfoEnum getType() { + return StorageInfoEnum.TOTAL_STORAGE; + } + } diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java index eb79e97fdb..95be8e38be 100644 --- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java +++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java @@ -20,6 +20,7 @@ package org.apache.kylin.metadata.cube.storage; import java.io.IOException; import java.lang.reflect.Field; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,9 +88,8 @@ public class ProjectStorageInfoCollectorTest extends NLocalFileMetadataTestCase getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "true"); initTestData(); - val storageInfoEnumList = Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE, StorageInfoEnum.STORAGE_QUOTA, - StorageInfoEnum.TOTAL_STORAGE); - val collector = new ProjectStorageInfoCollector(storageInfoEnumList); + val collector = new ProjectStorageInfoCollector(Lists.newArrayList(StorageInfoEnum.GARBAGE_STORAGE, StorageInfoEnum.STORAGE_QUOTA, + StorageInfoEnum.TOTAL_STORAGE)); val volumeInfo = collector.getStorageVolumeInfo(getTestConfig(), DEFAULT_PROJECT); Assert.assertEquals(10240L * 1024 * 1024 * 1024, volumeInfo.getStorageQuotaSize()); @@ -421,8 +421,7 @@ public class ProjectStorageInfoCollectorTest extends NLocalFileMetadataTestCase @Test public void testGetStorageVolumeInfoEmpty() { - List<StorageInfoEnum> storageInfoEnumList = Lists.newArrayList(); - val collector = new ProjectStorageInfoCollector(storageInfoEnumList); + val collector = new ProjectStorageInfoCollector(Collections.emptyList()); val storageVolumeInfo = collector.getStorageVolumeInfo(getTestConfig(), DEFAULT_PROJECT); Assert.assertEquals(-1L, storageVolumeInfo.getStorageQuotaSize()); @@ -433,9 +432,8 @@ public class ProjectStorageInfoCollectorTest extends NLocalFileMetadataTestCase @Test public void testGetStorageVolumeException() throws NoSuchFieldException, IllegalAccessException, IOException { - List<StorageInfoEnum> storageInfoEnumList = Lists.newArrayList(); TotalStorageCollector totalStorageCollector = Mockito.spy(TotalStorageCollector.class); - ProjectStorageInfoCollector collector = new ProjectStorageInfoCollector(storageInfoEnumList); + ProjectStorageInfoCollector collector = new ProjectStorageInfoCollector(Collections.emptyList()); Field field = collector.getClass().getDeclaredField("collectors"); Unsafe.changeAccessibleObject(field, true); List<StorageInfoCollector> collectors = (List<StorageInfoCollector>) field.get(collector);