This is an automated email from the ASF dual-hosted git repository. nehapawar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push: new 0d6b5da FixedSegmentSelector for tier configs (#8389) 0d6b5da is described below commit 0d6b5da7c6872229501faf6cfc27481f35f52584 Author: Neha Pawar <neha.pawa...@gmail.com> AuthorDate: Mon Mar 28 10:18:51 2022 -0700 FixedSegmentSelector for tier configs (#8389) * FixedSegmentSelector for tier configs * Review comments * fix test * jdk 8 breakage * Review comments: --- .../common/tier/FixedTierSegmentSelector.java | 80 ++++++++++++++++++++ .../org/apache/pinot/common/tier/TierFactory.java | 8 ++ .../pinot/common/utils/config/TierConfigUtils.java | 25 ++++--- .../pinot/common/tier/TierConfigUtilsTest.java | 63 +++++++++++++--- .../pinot/common/tier/TierSegmentSelectorTest.java | 54 ++++++++++++++ .../common/utils/config/TableConfigSerDeTest.java | 18 +++-- ...NonReplicaGroupTieredSegmentAssignmentTest.java | 6 +- ...NonReplicaGroupTieredSegmentAssignmentTest.java | 6 +- .../TableRebalancerClusterStatelessTest.java | 24 ++++-- .../segment/local/utils/TableConfigUtils.java | 2 +- .../segment/local/utils/TableConfigUtilsTest.java | 85 +++++++++++++--------- .../apache/pinot/spi/config/table/TierConfig.java | 12 +++ 12 files changed, 310 insertions(+), 73 deletions(-) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/tier/FixedTierSegmentSelector.java b/pinot-common/src/main/java/org/apache/pinot/common/tier/FixedTierSegmentSelector.java new file mode 100644 index 0000000..c9db747 --- /dev/null +++ b/pinot-common/src/main/java/org/apache/pinot/common/tier/FixedTierSegmentSelector.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.tier; + +import com.google.common.base.Preconditions; +import java.util.Set; +import org.apache.commons.collections.CollectionUtils; +import org.apache.helix.HelixManager; +import org.apache.pinot.common.metadata.ZKMetadataProvider; +import org.apache.pinot.common.metadata.segment.SegmentZKMetadata; +import org.apache.pinot.spi.config.table.TableType; +import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime; +import org.apache.pinot.spi.utils.builder.TableNameBuilder; + + +/** + * A {@link TierSegmentSelector} strategy which selects segments for a tier based on a fixed list + */ +public class FixedTierSegmentSelector implements TierSegmentSelector { + private final Set<String> _segmentsToSelect; + private final HelixManager _helixManager; + + public FixedTierSegmentSelector(HelixManager helixManager, Set<String> segmentsToSelect) { + _segmentsToSelect = segmentsToSelect; + _helixManager = helixManager; + } + + @Override + public String getType() { + return TierFactory.FIXED_SEGMENT_SELECTOR_TYPE; + } + + /** + * Checks if a segment is eligible for the tier based on fixed list + * @param tableNameWithType Name of the table + * @param segmentName Name of the segment + * @return true if eligible + */ + @Override + public boolean selectSegment(String tableNameWithType, String segmentName) { + if (CollectionUtils.isNotEmpty(_segmentsToSelect) && _segmentsToSelect.contains(segmentName)) { + if (TableType.OFFLINE == TableNameBuilder.getTableTypeFromTableName(tableNameWithType)) { + return true; + } + + SegmentZKMetadata segmentZKMetadata = + ZKMetadataProvider.getSegmentZKMetadata(_helixManager.getHelixPropertyStore(), tableNameWithType, + segmentName); + Preconditions.checkNotNull(segmentZKMetadata, "Could not find zk metadata for segment: {} of table: {}", + segmentName, tableNameWithType); + return !segmentZKMetadata.getStatus().equals(Realtime.Status.IN_PROGRESS); + } + return false; + } + + public Set<String> getSegmentsToSelect() { + return _segmentsToSelect; + } + + @Override + public String toString() { + return "FixedTierSegmentSelector{" + "_segmentsToSelect=" + _segmentsToSelect + '}'; + } +} diff --git a/pinot-common/src/main/java/org/apache/pinot/common/tier/TierFactory.java b/pinot-common/src/main/java/org/apache/pinot/common/tier/TierFactory.java index f33a784..b50b055 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/tier/TierFactory.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/tier/TierFactory.java @@ -18,6 +18,9 @@ */ package org.apache.pinot.common.tier; +import com.google.common.collect.Sets; +import java.util.Collections; +import org.apache.commons.collections.CollectionUtils; import org.apache.helix.HelixManager; import org.apache.pinot.spi.config.table.TierConfig; @@ -28,6 +31,7 @@ import org.apache.pinot.spi.config.table.TierConfig; public final class TierFactory { public static final String TIME_SEGMENT_SELECTOR_TYPE = "time"; + public static final String FIXED_SEGMENT_SELECTOR_TYPE = "fixed"; public static final String PINOT_SERVER_STORAGE_TYPE = "pinot_server"; private TierFactory() { @@ -43,6 +47,10 @@ public final class TierFactory { String segmentSelectorType = tierConfig.getSegmentSelectorType(); if (segmentSelectorType.equalsIgnoreCase(TierFactory.TIME_SEGMENT_SELECTOR_TYPE)) { segmentSelector = new TimeBasedTierSegmentSelector(helixManager, tierConfig.getSegmentAge()); + } else if (segmentSelectorType.equalsIgnoreCase(TierFactory.FIXED_SEGMENT_SELECTOR_TYPE)) { + segmentSelector = new FixedTierSegmentSelector(helixManager, + CollectionUtils.isEmpty(tierConfig.getSegmentList()) ? Collections.emptySet() + : Sets.newHashSet(tierConfig.getSegmentList())); } else { throw new IllegalStateException("Unsupported segmentSelectorType: " + segmentSelectorType); } diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TierConfigUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TierConfigUtils.java index 818807c..e8a864e 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TierConfigUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TierConfigUtils.java @@ -18,16 +18,15 @@ */ package org.apache.pinot.common.utils.config; -import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import org.apache.commons.collections.CollectionUtils; import org.apache.helix.HelixManager; +import org.apache.pinot.common.tier.FixedTierSegmentSelector; import org.apache.pinot.common.tier.Tier; import org.apache.pinot.common.tier.TierFactory; import org.apache.pinot.common.tier.TierSegmentSelector; -import org.apache.pinot.common.tier.TierStorage; import org.apache.pinot.common.tier.TimeBasedTierSegmentSelector; import org.apache.pinot.spi.config.table.TableConfig; import org.apache.pinot.spi.config.table.TierConfig; @@ -76,20 +75,24 @@ public final class TierConfigUtils { } /** - * Comparator for sorting the {@link Tier}. - * As of now, we have only 1 type of {@link TierSegmentSelector} and 1 type of {@link TierStorage}. - * Tier with an older age bucket in {@link TimeBasedTierSegmentSelector} should appear before a younger age bucket, - * in sort order - * TODO: As we add more types, this logic needs to be upgraded + * Comparator for sorting the {@link Tier}. In the sort order + * 1) {@link FixedTierSegmentSelector} are always before others + * 2) For {@link TimeBasedTierSegmentSelector}, tiers with an older age bucket appear before a younger age bucket, */ public static Comparator<Tier> getTierComparator() { return (o1, o2) -> { TierSegmentSelector s1 = o1.getSegmentSelector(); TierSegmentSelector s2 = o2.getSegmentSelector(); - Preconditions.checkState(TierFactory.TIME_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s1.getType()), - "Unsupported segmentSelectorType class %s", s1.getClass()); - Preconditions.checkState(TierFactory.TIME_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s2.getType()), - "Unsupported segmentSelectorType class %s", s2.getClass()); + if (TierFactory.FIXED_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s1.getType()) + && TierFactory.FIXED_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s2.getType())) { + return 0; + } + if (TierFactory.FIXED_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s1.getType())) { + return -1; + } + if (TierFactory.FIXED_SEGMENT_SELECTOR_TYPE.equalsIgnoreCase(s2.getType())) { + return 1; + } Long period1 = ((TimeBasedTierSegmentSelector) s1).getSegmentAgeMillis(); Long period2 = ((TimeBasedTierSegmentSelector) s2).getSegmentAgeMillis(); return period2.compareTo(period1); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/tier/TierConfigUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/tier/TierConfigUtilsTest.java index 1afd525..3205ee0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/tier/TierConfigUtilsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/tier/TierConfigUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.pinot.common.tier; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.Collections; import java.util.Comparator; import org.apache.pinot.common.utils.config.TierConfigUtils; @@ -49,7 +50,18 @@ public class TierConfigUtilsTest { Assert.assertFalse(TierConfigUtils.shouldRelocateToTiers(tableConfig)); tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").setTierConfigList( - Lists.newArrayList(new TierConfig("myTier", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", + Lists.newArrayList(new TierConfig("myTier", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tag_OFFLINE", null, null))).build(); + Assert.assertTrue(TierConfigUtils.shouldRelocateToTiers(tableConfig)); + + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").setTierConfigList( + Lists.newArrayList( + new TierConfig("myTier", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, "10d", Lists.newArrayList("seg0", "seg1"), + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tag_OFFLINE", null, null))).build(); + Assert.assertTrue(TierConfigUtils.shouldRelocateToTiers(tableConfig)); + + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").setTierConfigList( + Lists.newArrayList(new TierConfig("myTier", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tag_OFFLINE", null, null))).build(); Assert.assertTrue(TierConfigUtils.shouldRelocateToTiers(tableConfig)); @@ -65,9 +77,15 @@ public class TierConfigUtilsTest { Assert.assertFalse(TierConfigUtils.shouldRelocateToTiers(tableConfig)); tableConfig = new TableConfigBuilder(TableType.REALTIME).setTableName("myTable").setTierConfigList( - Lists.newArrayList(new TierConfig("myTier", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", + Lists.newArrayList(new TierConfig("myTier", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tag_OFFLINE", null, null))).build(); Assert.assertTrue(TierConfigUtils.shouldRelocateToTiers(tableConfig)); + + tableConfig = new TableConfigBuilder(TableType.REALTIME).setTableName("myTable").setTierConfigList( + Lists.newArrayList( + new TierConfig("myTier", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, null, Lists.newArrayList("seg0", "seg1"), + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tag_OFFLINE", null, null))).build(); + Assert.assertTrue(TierConfigUtils.shouldRelocateToTiers(tableConfig)); } /** @@ -75,9 +93,8 @@ public class TierConfigUtilsTest { */ @Test public void testGetTier() { - TierConfig tierConfig = - new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", TierFactory.PINOT_SERVER_STORAGE_TYPE, - "tier1_tag_OFFLINE", null, null); + TierConfig tierConfig = new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null); Tier tier = TierFactory.getTier(tierConfig, null); Assert.assertEquals(tier.getName(), "tier1"); Assert.assertTrue(tier.getSegmentSelector() instanceof TimeBasedTierSegmentSelector); @@ -88,9 +105,28 @@ public class TierConfigUtilsTest { Assert.assertEquals(tier.getStorage().getType(), TierFactory.PINOT_SERVER_STORAGE_TYPE); Assert.assertEquals(((PinotServerTierStorage) tier.getStorage()).getServerTag(), "tier1_tag_OFFLINE"); + tierConfig = new TierConfig("tier1", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, + Lists.newArrayList("segment1", "segment2", "segment3"), TierFactory.PINOT_SERVER_STORAGE_TYPE, + "tier1_tag_OFFLINE", null, null); + tier = TierFactory.getTier(tierConfig, null); + Assert.assertEquals(tier.getName(), "tier1"); + Assert.assertTrue(tier.getSegmentSelector() instanceof FixedTierSegmentSelector); + Assert.assertEquals(tier.getSegmentSelector().getType(), TierFactory.FIXED_SEGMENT_SELECTOR_TYPE); + Assert.assertEquals(((FixedTierSegmentSelector) tier.getSegmentSelector()).getSegmentsToSelect(), + Sets.newHashSet("segment1", "segment2", "segment3")); + + tierConfig = new TierConfig("tier1", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, + null, TierFactory.PINOT_SERVER_STORAGE_TYPE, + "tier1_tag_OFFLINE", null, null); + tier = TierFactory.getTier(tierConfig, null); + Assert.assertEquals(tier.getName(), "tier1"); + Assert.assertTrue(tier.getSegmentSelector() instanceof FixedTierSegmentSelector); + Assert.assertEquals(tier.getSegmentSelector().getType(), TierFactory.FIXED_SEGMENT_SELECTOR_TYPE); + Assert.assertTrue(((FixedTierSegmentSelector) tier.getSegmentSelector()).getSegmentsToSelect().isEmpty()); + tierConfig = - new TierConfig("tier1", "unknown", "30d", TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, - null); + new TierConfig("tier1", "unknown", "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", + null, null); try { TierFactory.getTier(tierConfig, null); Assert.fail("Should have failed due to unsupported segmentSelectorType"); @@ -99,8 +135,8 @@ public class TierConfigUtilsTest { } tierConfig = - new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", "unknown", "tier1_tag_OFFLINE", null, - null); + new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, "unknown", "tier1_tag_OFFLINE", + null, null); try { TierFactory.getTier(tierConfig, null); Assert.fail("Should've failed due to unsupported storageType"); @@ -132,6 +168,11 @@ public class TierConfigUtilsTest { new Tier("tier5", new TimeBasedTierSegmentSelector(null, "1d"), new PinotServerTierStorage("tag_OFFLINE", null, null)); + Tier tier6 = new Tier("tier6", new FixedTierSegmentSelector(null, Sets.newHashSet("seg0")), + new PinotServerTierStorage("tag_OFFLINE", null, null)); + Tier tier7 = new Tier("tier6", new FixedTierSegmentSelector(null, Sets.newHashSet("seg1")), + new PinotServerTierStorage("tag_OFFLINE", null, null)); + Assert.assertEquals(tierComparator.compare(tier1, tier2), 1); Assert.assertEquals(tierComparator.compare(tier1, tier3), -1); Assert.assertEquals(tierComparator.compare(tier1, tier4), -1); @@ -140,5 +181,9 @@ public class TierConfigUtilsTest { Assert.assertEquals(tierComparator.compare(tier3, tier4), -1); Assert.assertEquals(tierComparator.compare(tier1, tier1), 0); Assert.assertEquals(tierComparator.compare(tier3, tier5), 0); + + Assert.assertEquals(tierComparator.compare(tier6, tier7), 0); + Assert.assertEquals(tierComparator.compare(tier6, tier5), -1); + Assert.assertEquals(tierComparator.compare(tier4, tier7), 1); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/tier/TierSegmentSelectorTest.java b/pinot-common/src/test/java/org/apache/pinot/common/tier/TierSegmentSelectorTest.java index ae6a8de..cee73f4 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/tier/TierSegmentSelectorTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/tier/TierSegmentSelectorTest.java @@ -18,6 +18,8 @@ */ package org.apache.pinot.common.tier; +import com.google.common.collect.Sets; +import java.util.Collections; import java.util.concurrent.TimeUnit; import org.apache.helix.HelixManager; import org.apache.helix.ZNRecord; @@ -111,4 +113,56 @@ public class TierSegmentSelectorTest { segmentSelector = new TimeBasedTierSegmentSelector(helixManager, "120h"); Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); } + + @Test + public void testFixedSegmentSelector() { + + // offline segment + String segmentName = "segment_0"; + String tableNameWithType = "myTable_OFFLINE"; + + FixedTierSegmentSelector segmentSelector = new FixedTierSegmentSelector(null, Collections.emptySet()); + Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); + + segmentSelector = new FixedTierSegmentSelector(null, Sets.newHashSet("segment_1", "segment_2")); + Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); + + segmentSelector = new FixedTierSegmentSelector(null, Sets.newHashSet("SEGMENT_0", "segment_1", "segment_2")); + Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); + + segmentSelector = new FixedTierSegmentSelector(null, Sets.newHashSet("segment_0", "segment_1", "segment_2")); + Assert.assertTrue(segmentSelector.selectSegment(tableNameWithType, segmentName)); + + segmentSelector = new FixedTierSegmentSelector(null, Sets.newHashSet("segment %", "segment_2")); + Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); + Assert.assertTrue(segmentSelector.selectSegment(tableNameWithType, "segment %")); + + // realtime segment + segmentName = "myTable__4__1__" + 123456789; + tableNameWithType = "myTable_REALTIME"; + SegmentZKMetadata realtimeSegmentZKMetadata = new SegmentZKMetadata(segmentName); + realtimeSegmentZKMetadata.setStartTime(System.currentTimeMillis() - 10000); + realtimeSegmentZKMetadata.setEndTime(System.currentTimeMillis() - 5000); + realtimeSegmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS); + realtimeSegmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE); + realtimeSegmentZKMetadata.setNumReplicas(1); + ZNRecord segmentZKMetadataZNRecord = realtimeSegmentZKMetadata.toZNRecord(); + + HelixManager helixManager = mock(HelixManager.class); + ZkHelixPropertyStore<ZNRecord> propertyStore = mock(ZkHelixPropertyStore.class); + when(propertyStore + .get(eq(ZKMetadataProvider.constructPropertyStorePathForSegment(tableNameWithType, segmentName)), any(), + anyInt())).thenReturn(segmentZKMetadataZNRecord); + when(helixManager.getHelixPropertyStore()).thenReturn(propertyStore); + segmentSelector = new FixedTierSegmentSelector(helixManager, Sets.newHashSet(segmentName, "foo", "bar")); + Assert.assertTrue(segmentSelector.selectSegment(tableNameWithType, segmentName)); + + realtimeSegmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS); + segmentZKMetadataZNRecord = realtimeSegmentZKMetadata.toZNRecord(); + when(propertyStore + .get(eq(ZKMetadataProvider.constructPropertyStorePathForSegment(tableNameWithType, segmentName)), any(), + anyInt())).thenReturn(segmentZKMetadataZNRecord); + when(helixManager.getHelixPropertyStore()).thenReturn(propertyStore); + Assert.assertFalse(segmentSelector.selectSegment(tableNameWithType, segmentName)); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java index 8d82b02..aa57465 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java @@ -300,10 +300,12 @@ public class TableConfigSerDeTest { { // With tier config List<TierConfig> tierConfigList = Lists.newArrayList( - new TierConfig("tierA", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", TierFactory.PINOT_SERVER_STORAGE_TYPE, - "tierA_tag_OFFLINE", null, null), - new TierConfig("tierB", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", TierFactory.PINOT_SERVER_STORAGE_TYPE, - "tierB_tag_OFFLINE", null, null)); + new TierConfig("tierA", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tierA_tag_OFFLINE", null, null), + new TierConfig("tierB", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tierB_tag_OFFLINE", null, null), + new TierConfig("tier0", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, Lists.newArrayList("seg0"), + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tierB_tag_OFFLINE", null, null)); TableConfig tableConfig = tableConfigBuilder.setTierConfigList(tierConfigList).build(); checkTierConfigList(tableConfig); @@ -461,7 +463,7 @@ public class TableConfigSerDeTest { private void checkTierConfigList(TableConfig tableConfig) { List<TierConfig> tierConfigsList = tableConfig.getTierConfigsList(); assertNotNull(tierConfigsList); - assertEquals(tierConfigsList.size(), 2); + assertEquals(tierConfigsList.size(), 3); assertEquals(tierConfigsList.get(0).getName(), "tierA"); assertEquals(tierConfigsList.get(0).getSegmentSelectorType(), TierFactory.TIME_SEGMENT_SELECTOR_TYPE); assertEquals(tierConfigsList.get(0).getSegmentAge(), "10d"); @@ -472,6 +474,12 @@ public class TableConfigSerDeTest { assertEquals(tierConfigsList.get(1).getSegmentAge(), "30d"); assertEquals(tierConfigsList.get(1).getStorageType(), TierFactory.PINOT_SERVER_STORAGE_TYPE); assertEquals(tierConfigsList.get(1).getServerTag(), "tierB_tag_OFFLINE"); + assertEquals(tierConfigsList.get(2).getName(), "tier0"); + assertEquals(tierConfigsList.get(2).getSegmentSelectorType(), TierFactory.FIXED_SEGMENT_SELECTOR_TYPE); + assertNull(tierConfigsList.get(2).getSegmentAge()); + assertEquals(tierConfigsList.get(2).getSegmentList(), Lists.newArrayList("seg0")); + assertEquals(tierConfigsList.get(2).getStorageType(), TierFactory.PINOT_SERVER_STORAGE_TYPE); + assertEquals(tierConfigsList.get(2).getServerTag(), "tierB_tag_OFFLINE"); } private void checkInstanceAssignmentConfig(TableConfig tableConfig) { diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineNonReplicaGroupTieredSegmentAssignmentTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineNonReplicaGroupTieredSegmentAssignmentTest.java index ba2c1c7..8454de4 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineNonReplicaGroupTieredSegmentAssignmentTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineNonReplicaGroupTieredSegmentAssignmentTest.java @@ -102,11 +102,11 @@ public class OfflineNonReplicaGroupTieredSegmentAssignmentTest { @BeforeClass public void setUp() { List<TierConfig> tierConfigList = Lists.newArrayList( - new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "50d", + new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "50d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_A_NAME, null, null), - new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "70d", + new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "70d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_B_NAME, null, null), - new TierConfig(TIER_C_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "120d", + new TierConfig(TIER_C_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "120d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_C_NAME, null, null)); TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setNumReplicas(NUM_REPLICAS) diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/RealtimeNonReplicaGroupTieredSegmentAssignmentTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/RealtimeNonReplicaGroupTieredSegmentAssignmentTest.java index 4863b96..ba33245 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/RealtimeNonReplicaGroupTieredSegmentAssignmentTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/RealtimeNonReplicaGroupTieredSegmentAssignmentTest.java @@ -110,11 +110,11 @@ public class RealtimeNonReplicaGroupTieredSegmentAssignmentTest { } List<TierConfig> tierConfigList = Lists.newArrayList( - new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", + new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "10d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_A_NAME, null, null), - new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "20d", + new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "20d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_B_NAME, null, null), - new TierConfig(TIER_C_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + new TierConfig(TIER_C_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, TAG_C_NAME, null, null)); TableConfig tableConfig = new TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setNumReplicas(NUM_REPLICAS) diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java index f2372da..f26cdea 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java @@ -19,6 +19,7 @@ package org.apache.pinot.controller.helix.core.rebalance; import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -65,6 +66,7 @@ public class TableRebalancerClusterStatelessTest extends ControllerTest { private static final String NO_TIER_NAME = "noTier"; private static final String TIER_A_NAME = "tierA"; private static final String TIER_B_NAME = "tierB"; + private static final String TIER_FIXED_NAME = "tierFixed"; @BeforeClass public void setUp() @@ -318,7 +320,7 @@ public class TableRebalancerClusterStatelessTest extends ControllerTest { /** * Tests rebalance with tier configs - * Add 10 segments, with segment metadat end time 3 days apart starting from now to 30 days ago + * Add 10 segments, with segment metadata end time 3 days apart starting from now to 30 days ago * 1. run rebalance - should see no change * 2. add nodes for tiers and run rebalance - should see no change * 3. add tier config and run rebalance - should see changed assignment @@ -374,11 +376,15 @@ public class TableRebalancerClusterStatelessTest extends ControllerTest { assertEquals(rebalanceResult.getSegmentAssignment(), oldSegmentAssignment); // add tier config + ArrayList<String> fixedTierSegments = + Lists.newArrayList(SEGMENT_NAME_PREFIX + 6, SEGMENT_NAME_PREFIX + 3, SEGMENT_NAME_PREFIX + 1); tableConfig.setTierConfigsList(Lists.newArrayList( - new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "7d", TierFactory.PINOT_SERVER_STORAGE_TYPE, - TIER_A_NAME + "_OFFLINE", null, null), - new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "15d", - TierFactory.PINOT_SERVER_STORAGE_TYPE, TIER_B_NAME + "_OFFLINE", null, null))); + new TierConfig(TIER_A_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "7d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, TIER_A_NAME + "_OFFLINE", null, null), + new TierConfig(TIER_B_NAME, TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "15d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, TIER_B_NAME + "_OFFLINE", null, null), + new TierConfig(TIER_FIXED_NAME, TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, fixedTierSegments, + TierFactory.PINOT_SERVER_STORAGE_TYPE, NO_TIER_NAME + "_OFFLINE", null, null))); _helixResourceManager.updateTableConfig(tableConfig); // rebalance should change assignment @@ -388,10 +394,14 @@ public class TableRebalancerClusterStatelessTest extends ControllerTest { // check that segments have moved to tiers Map<String, Map<String, String>> tierSegmentAssignment = rebalanceResult.getSegmentAssignment(); for (Map.Entry<String, Map<String, String>> entry : tierSegmentAssignment.entrySet()) { - int segId = Integer.parseInt(entry.getKey().split("_")[1]); + String segment = entry.getKey(); + int segId = Integer.parseInt(segment.split("_")[1]); Map<String, String> instanceStateMap = entry.getValue(); String expectedPrefix; - if (segId > 4) { + if (fixedTierSegments.contains(segment)) { + expectedPrefix = NO_TIER_NAME + "_" + SERVER_INSTANCE_ID_PREFIX; + } else + if (segId > 4) { expectedPrefix = TIER_B_NAME + "_" + SERVER_INSTANCE_ID_PREFIX; } else if (segId > 2) { expectedPrefix = TIER_A_NAME + "_" + SERVER_INSTANCE_ID_PREFIX; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java index bec8392..f3fa1d4 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java @@ -521,7 +521,7 @@ public final class TableConfigUtils { segmentSelectorType, tierName); Preconditions.checkState(TimeUtils.isPeriodValid(segmentAge), "segmentAge: %s must be a valid period string (eg. 30d, 24h) in tier: %s", segmentAge, tierName); - } else { + } else if (!segmentSelectorType.equalsIgnoreCase(TierFactory.FIXED_SEGMENT_SELECTOR_TYPE)) { throw new IllegalStateException( "Unsupported segmentSelectorType: " + segmentSelectorType + " in tier: " + tierName); } diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java index f78f9b2..51d1907 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java @@ -528,32 +528,32 @@ public class TableConfigUtilsTest { TableConfigUtils.validate(tableConfig, schema); // 1 tier configs - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null))).build(); TableConfigUtils.validate(tableConfig, schema); // 2 tier configs, case insensitive check - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE.toLowerCase(), "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE.toLowerCase(), "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), - new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE.toLowerCase(), "tier2_tag_OFFLINE", null, null))).build(); TableConfigUtils.validate(tableConfig, schema); //realtime table tableConfig = new TableConfigBuilder(TableType.REALTIME).setTableName(TABLE_NAME).setTimeColumnName(TIME_COLUMN) - .setTierConfigList(Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + .setTierConfigList(Lists.newArrayList( + new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE.toLowerCase(), "tier1_tag_OFFLINE", null, null), - new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE.toLowerCase(), "40d", + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE.toLowerCase(), "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); TableConfigUtils.validate(tableConfig, schema); // tier name empty - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList( - new TierConfig("", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", TierFactory.PINOT_SERVER_STORAGE_TYPE, - "tier1_tag_OFFLINE", null, null))).build(); + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); Assert.fail("Should have failed due to empty tier name"); @@ -562,10 +562,10 @@ public class TableConfigUtilsTest { } // tier name repeats - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("sameTierName", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("sameTierName", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), - new TierConfig("sameTierName", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "100d", + new TierConfig("sameTierName", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "100d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); @@ -575,10 +575,10 @@ public class TableConfigUtilsTest { } // segmentSelectorType invalid - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), - new TierConfig("tier2", "unsupportedSegmentSelector", "40d", TierFactory.PINOT_SERVER_STORAGE_TYPE, + new TierConfig("tier2", "unsupportedSegmentSelector", "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); @@ -588,10 +588,10 @@ public class TableConfigUtilsTest { } // segmentAge not provided for TIME segmentSelectorType - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList( - new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, null, TierFactory.PINOT_SERVER_STORAGE_TYPE, - "tier1_tag_OFFLINE", null, null), new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, null, null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); @@ -601,10 +601,10 @@ public class TableConfigUtilsTest { } // segmentAge invalid - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), - new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "3600", + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "3600", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { @@ -614,11 +614,28 @@ public class TableConfigUtilsTest { // expected } + // fixedSegmentSelector + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null))).build(); + TableConfigUtils.validate(tableConfig, schema); + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, "30d", Lists.newArrayList(), + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null))).build(); + TableConfigUtils.validate(tableConfig, schema); + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList( + new TierConfig("tier1", TierFactory.FIXED_SEGMENT_SELECTOR_TYPE, null, Lists.newArrayList("seg0", "seg1"), + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null))).build(); + TableConfigUtils.validate(tableConfig, schema); + // storageType invalid - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", "unsupportedStorageType", - "tier1_tag_OFFLINE", null, null), new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", - TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList( + new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, "unsupportedStorageType", + "tier1_tag_OFFLINE", null, null), + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", null, + TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); @@ -628,10 +645,10 @@ public class TableConfigUtilsTest { } // serverTag not provided for PINOT_SERVER storageType - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag_OFFLINE", null, null), - new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, null, null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); @@ -641,10 +658,10 @@ public class TableConfigUtilsTest { } // serverTag invalid - tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList(Lists - .newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", + tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setTierConfigList( + Lists.newArrayList(new TierConfig("tier1", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "30d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier1_tag", null, null), - new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", + new TierConfig("tier2", TierFactory.TIME_SEGMENT_SELECTOR_TYPE, "40d", null, TierFactory.PINOT_SERVER_STORAGE_TYPE, "tier2_tag_OFFLINE", null, null))).build(); try { TableConfigUtils.validate(tableConfig, schema); diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TierConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TierConfig.java index 755116b..a3a2d6a 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TierConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TierConfig.java @@ -21,6 +21,7 @@ package org.apache.pinot.spi.config.table; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.google.common.base.Preconditions; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; import org.apache.pinot.spi.config.BaseJsonConfig; @@ -39,6 +40,9 @@ public class TierConfig extends BaseJsonConfig { @JsonPropertyDescription("For 'TIME' segment selector, the period after which to select segments for this tier") private final String _segmentAge; + @JsonPropertyDescription("For 'FIXED' segment selector, the list of segments to select for this tier") + private final List<String> _segmentList; + @JsonPropertyDescription("The type of storage") private final String _storageType; @@ -58,6 +62,7 @@ public class TierConfig extends BaseJsonConfig { public TierConfig(@JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "segmentSelectorType", required = true) String segmentSelectorType, @JsonProperty("segmentAge") @Nullable String segmentAge, + @JsonProperty("segmentList") @Nullable List<String> segmentList, @JsonProperty(value = "storageType", required = true) String storageType, @JsonProperty("serverTag") @Nullable String serverTag, @JsonProperty("tierBackend") @Nullable String tierBackend, @@ -69,6 +74,7 @@ public class TierConfig extends BaseJsonConfig { _name = name; _segmentSelectorType = segmentSelectorType; _segmentAge = segmentAge; + _segmentList = segmentList; _storageType = storageType; _serverTag = serverTag; _tierBackend = tierBackend; @@ -83,10 +89,16 @@ public class TierConfig extends BaseJsonConfig { return _segmentSelectorType; } + @Nullable public String getSegmentAge() { return _segmentAge; } + @Nullable + public List<String> getSegmentList() { + return _segmentList; + } + public String getStorageType() { return _storageType; } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org