kylin git commit: KYLIN-1875 fix PK-FK derive from each other
Repository: kylin Updated Branches: refs/heads/KYLIN-1875 a9100bfc6 -> 48253d1ff KYLIN-1875 fix PK-FK derive from each other Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48253d1f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48253d1f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48253d1f Branch: refs/heads/KYLIN-1875 Commit: 48253d1ff03e137a8568b8ca880c45b5b8c524f3 Parents: a9100bf Author: Yang LiAuthored: Thu Dec 1 21:42:52 2016 +0800 Committer: Yang Li Committed: Thu Dec 1 21:42:52 2016 +0800 -- .../java/org/apache/kylin/cube/CubeManager.java | 8 +-- .../org/apache/kylin/cube/model/CubeDesc.java | 52 +++- .../storage/gtrecord/CubeTupleConverter.java| 2 +- .../gtrecord/GTCubeStorageQueryBase.java| 2 +- .../translate/DerivedFilterTranslator.java | 2 +- .../apache/kylin/query/ITKylinQueryTest.java| 2 +- .../query/enumerator/LookupTableEnumerator.java | 2 +- .../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +- .../hbase/cube/v1/CubeTupleConverter.java | 2 +- 9 files changed, 41 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index b4422d2..6a25594 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.DictionaryDesc; -import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.dict.DictionaryInfo; import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.lookup.LookupStringTable; @@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; +import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -657,10 +657,10 @@ public class CubeManager implements IRealizationProvider { cubeMap.removeLocal(cubeName); } -public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { +public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) { -String tableName = dim.getTableRef().getTableIdentity(); -String[] pkCols = dim.getJoin().getPrimaryKey(); +String tableName = join.getPKSide().getTableIdentity(); +String[] pkCols = join.getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index fa4da92..18b7e7a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; @@ -98,20 +99,20 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public static class DeriveInfo { public DeriveType type; -public DimensionDesc dimension; +public JoinDesc join; public TblColRef[] columns; public boolean isOneToOne; // only used when ref from derived to host -DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) { +
kylin git commit: minor, rollback ITKylinQueryTest
Repository: kylin Updated Branches: refs/heads/KYLIN-1875 48253d1ff -> 12230f4e3 minor, rollback ITKylinQueryTest Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/12230f4e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/12230f4e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/12230f4e Branch: refs/heads/KYLIN-1875 Commit: 12230f4e3b6a1557137e38b2e3c8c3b01288c248 Parents: 48253d1 Author: Yang LiAuthored: Fri Dec 2 06:32:25 2016 +0800 Committer: Yang Li Committed: Fri Dec 2 06:32:25 2016 +0800 -- .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/12230f4e/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 6d91753..90324b5 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; -//@Ignore("KylinQueryTest is contained by ITCombinationTest") +@Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { @Rule
kylin git commit: KYLIN-2180 minor, get owenr project in CubeDesc.init()
Repository: kylin Updated Branches: refs/heads/master 1a5295bad -> 9fc8f5e20 KYLIN-2180 minor, get owenr project in CubeDesc.init() Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2 Branch: refs/heads/master Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa Parents: 1a5295b Author: Li YangAuthored: Fri Dec 2 11:39:34 2016 +0800 Committer: Li Yang Committed: Fri Dec 2 11:39:34 2016 +0800 -- .../main/java/org/apache/kylin/cube/model/CubeDesc.java | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 327ce57..853571c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.realization.RealizationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public void init(KylinConfig config) { this.errors.clear(); -this.config = KylinConfigExt.createInstance(config, overrideKylinProps); checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank"); checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has blank modelName", name); +// note CubeDesc.name == CubeInstance.name +List ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name); +logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj); + +this.config = KylinConfigExt.createInstance(config, overrideKylinProps); + this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName); checkNotNull(this.model, "DateModelDesc(%s) not found", modelName); @@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return col; } +@SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return;
[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea Branch: refs/heads/yang21-hbase1.x Commit: 215329ea084465e654bfc6e2e32e91f68a64254a Parents: 8001887 Author: shaofengshiAuthored: Wed Mar 23 17:07:05 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 16:15:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 13 ++- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 36 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../hbase/steps/HBaseStreamingOutput.java | 9 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 22 ++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 .../kylin/tool/CubeMigrationCheckCLI.java | 16 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- .../apache/kylin/tool/StorageCleanupJob.java| 20 +++-- 53 files changed, 450 insertions(+), 500 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -
[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yang21-hbase1.x 02f37bd82 -> 215329ea0 (forced update) http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java index 0e95102..c59fb33 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java @@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.cube.CubeSegment; @@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput { try { CubeSegment cubeSegment = (CubeSegment) buildable; -final HTableInterface hTable; +final Table hTable; hTable = createHTable(cubeSegment); List cuboidWriters = Lists.newArrayList(); cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable)); @@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements IStreamingOutput { } } -private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException { +private Table createHTable(final CubeSegment cubeSegment) throws IOException { final String hTableName = cubeSegment.getStorageLocationIdentifier(); CubeHTableUtil.createHTable(cubeSegment, null); -final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName); +final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName)); logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!"); return hTable; } http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index 5b2441c..2f7e164 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,11 +24,11 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin admin = null; +Admin admin = null; try { -admin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +admin = conn.getAdmin(); + for (String table : oldTables) { -if (admin.tableExists(table)) { -HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); +if (admin.tableExists(TableName.valueOf(table))) { +HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { -
[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2
KYLIN-2233 Support HBase 1.0.2 Signed-off-by: Yang LiProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11b7fff7 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11b7fff7 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11b7fff7 Branch: refs/heads/yang21-hbase102 Commit: 11b7fff75f3b8c4091741ee68f468d207a07fa4a Parents: 215329e Author: Cheng Wang Authored: Wed Oct 12 14:39:24 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 16:18:42 2016 +0800 -- pom.xml | 2 +- .../storage/hbase/cube/v1/RegionScannerAdapter.java | 11 ++- .../coprocessor/observer/AggregateRegionObserver.java | 4 ++-- .../v1/coprocessor/observer/AggregationScanner.java | 14 -- .../observer/ObserverAggregationCache.java| 10 ++ .../observer/AggregateRegionObserverTest.java | 12 +++- .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +-- 7 files changed, 15 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/pom.xml -- diff --git a/pom.xml b/pom.xml index 1df14f4..ad70ee3 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 1.2.1 -1.1.1 +1.0.2 0.8.1 http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java index 0ade920..40da772 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; /** * @author yangli9 @@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override -public boolean next(List result, ScannerContext scannerContext) throws IOException { +public boolean next(List result, int limit) throws IOException { return next(result); } @@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override -public boolean nextRaw(List result, ScannerContext scannerContext) throws IOException { +public boolean nextRaw(List result, int limit) throws IOException { return next(result); } @@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner { public long getMvccReadPoint() { return Long.MAX_VALUE; } - -@Override -public int getBatch() { -return -1; -} - } http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java index 7e25e4c..7139ca7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.kylin.gridtable.StorageSideBehavior; @@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver { // start/end region operation & sync on scanner is suggested by the // javadoc of RegionScanner.nextRaw() // FIXME: will
[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea Branch: refs/heads/yang21-hbase102 Commit: 215329ea084465e654bfc6e2e32e91f68a64254a Parents: 8001887 Author: shaofengshiAuthored: Wed Mar 23 17:07:05 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 16:15:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 13 ++- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 36 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../hbase/steps/HBaseStreamingOutput.java | 9 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 22 ++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 .../kylin/tool/CubeMigrationCheckCLI.java | 16 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- .../apache/kylin/tool/StorageCleanupJob.java| 20 +++-- 53 files changed, 450 insertions(+), 500 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -
[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yang21-hbase102 5d7e7f239 -> 11b7fff75 (forced update) http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java index 0e95102..c59fb33 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java @@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.cube.CubeSegment; @@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput { try { CubeSegment cubeSegment = (CubeSegment) buildable; -final HTableInterface hTable; +final Table hTable; hTable = createHTable(cubeSegment); List cuboidWriters = Lists.newArrayList(); cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable)); @@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements IStreamingOutput { } } -private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException { +private Table createHTable(final CubeSegment cubeSegment) throws IOException { final String hTableName = cubeSegment.getStorageLocationIdentifier(); CubeHTableUtil.createHTable(cubeSegment, null); -final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName); +final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName)); logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!"); return hTable; } http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index 5b2441c..2f7e164 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,11 +24,11 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin admin = null; +Admin admin = null; try { -admin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +admin = conn.getAdmin(); + for (String table : oldTables) { -if (admin.tableExists(table)) { -HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); +if (admin.tableExists(TableName.valueOf(table))) { +HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { -
[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
KYLIN-1672 support kylin on cdh 5.7 Signed-off-by: Li YangProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0291138 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0291138 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0291138 Branch: refs/heads/yang21-cdh5.7 Commit: c029113811fe3cd1e2d1539afb7ad5903987b3d2 Parents: 215329e Author: Lynne Jiang Authored: Mon May 16 03:33:27 2016 -0700 Committer: Hongbin Ma Committed: Thu Dec 1 16:17:04 2016 +0800 -- build/conf/kylin.properties | 3 + dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0 .../kylin/engine/mr/steps/MockupMapContext.java | 15 +- examples/test_case_data/sandbox/core-site.xml | 146 +++--- examples/test_case_data/sandbox/hbase-site.xml | 162 ++ examples/test_case_data/sandbox/hdfs-site.xml | 259 ++ examples/test_case_data/sandbox/mapred-site.xml | 398 ++- examples/test_case_data/sandbox/yarn-site.xml | 496 ++- pom.xml | 16 +- server/pom.xml | 36 ++ .../storage/hbase/steps/MockupMapContext.java | 19 +- tool/pom.xml| 12 + 12 files changed, 431 insertions(+), 1131 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/build/conf/kylin.properties -- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index ed86bdb..ebd8cde 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100 kylin.job.step.timeout=7200 +# for secure cdh, filtering hive dependency is risky, so filter nothing +kylin.job.dependency.filterlist=[^,]+ + ### CUBE ### # 'auto', 'inmem', 'layer' or 'random' for testing http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/dev-support/test_all_against_hdp_2_2_4_2_2.sh -- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh old mode 100644 new mode 100755 http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java -- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java index 847071d..9900465 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java @@ -77,6 +77,7 @@ public class MockupMapContext { outKV[0] = key; outKV[1] = value; } + } @Override @@ -99,6 +100,7 @@ public class MockupMapContext { throw new NotImplementedException(); } + @Override public float getProgress() { throw new NotImplementedException(); @@ -195,17 +197,17 @@ public class MockupMapContext { } @Override -public RawComparator getSortComparator() { +public boolean userClassesTakesPrecedence() { throw new NotImplementedException(); } @Override -public String getJar() { +public RawComparator getSortComparator() { throw new NotImplementedException(); } @Override -public RawComparator getGroupingComparator() { +public String getJar() { throw new NotImplementedException(); } @@ -221,7 +223,7 @@ public class MockupMapContext { @Override public boolean getProfileEnabled() { -throw new NotImplementedException(); +return false; } @Override @@ -308,6 +310,11 @@ public class MockupMapContext { public RawComparator getCombinerKeyGroupingComparator() { throw new NotImplementedException(); } + +@Override +public RawComparator getGroupingComparator() { +return null; +} }); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/examples/test_case_data/sandbox/core-site.xml -- diff --git a/examples/test_case_data/sandbox/core-site.xml
[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yang21-cdh5.7 0b6d6662e -> c02911381 (forced update) http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java index 0e95102..c59fb33 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java @@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.cube.CubeSegment; @@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput { try { CubeSegment cubeSegment = (CubeSegment) buildable; -final HTableInterface hTable; +final Table hTable; hTable = createHTable(cubeSegment); List cuboidWriters = Lists.newArrayList(); cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable)); @@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements IStreamingOutput { } } -private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException { +private Table createHTable(final CubeSegment cubeSegment) throws IOException { final String hTableName = cubeSegment.getStorageLocationIdentifier(); CubeHTableUtil.createHTable(cubeSegment, null); -final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName); +final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName)); logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!"); return hTable; } http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index 5b2441c..2f7e164 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,11 +24,11 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin admin = null; +Admin admin = null; try { -admin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +admin = conn.getAdmin(); + for (String table : oldTables) { -if (admin.tableExists(table)) { -HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); +if (admin.tableExists(TableName.valueOf(table))) { +HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { -
[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea Branch: refs/heads/yang21-cdh5.7 Commit: 215329ea084465e654bfc6e2e32e91f68a64254a Parents: 8001887 Author: shaofengshiAuthored: Wed Mar 23 17:07:05 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 16:15:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 13 ++- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 36 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../hbase/steps/HBaseStreamingOutput.java | 9 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 22 ++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 .../kylin/tool/CubeMigrationCheckCLI.java | 16 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- .../apache/kylin/tool/StorageCleanupJob.java| 20 +++-- 53 files changed, 450 insertions(+), 500 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -
kylin git commit: refine mapper and reducer log
Repository: kylin Updated Branches: refs/heads/yang21 80018874c -> d3ecb0d9c refine mapper and reducer log Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3ecb0d9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3ecb0d9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3ecb0d9 Branch: refs/heads/yang21 Commit: d3ecb0d9c381dbb035c7cada7d3c798e24fef1d1 Parents: 8001887 Author: Hongbin MaAuthored: Thu Dec 1 18:01:55 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 18:01:55 2016 +0800 -- .../org/apache/kylin/engine/mr/KylinMapper.java | 17 +++--- .../apache/kylin/engine/mr/KylinReducer.java| 17 +++--- .../engine/mr/steps/BaseCuboidMapperBase.java | 1 - .../kylin/engine/mr/steps/CuboidReducer.java| 21 ++-- .../engine/mr/steps/HiveToBaseCuboidMapper.java | 10 ++ .../engine/mr/steps/InMemCuboidMapper.java | 34 .../engine/mr/steps/InMemCuboidReducer.java | 20 ++-- .../kylin/engine/mr/steps/NDCuboidMapper.java | 22 ++--- 8 files changed, 70 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ecb0d9/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java -- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java index a01f7a2..c5af2fe 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java @@ -18,18 +18,21 @@ package org.apache.kylin.engine.mr; -import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Mapper; +import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; + /** */ public class KylinMapper extends Mapper { private static final Logger logger = LoggerFactory.getLogger(KylinMapper.class); +protected int mapCounter = 0; + protected void bindCurrentConfiguration(Configuration conf) { logger.info("The conf for current mapper will be " + System.identityHashCode(conf)); HadoopUtil.setCurrentConfiguration(conf); @@ -38,6 +41,10 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { try { +if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { +logger.info("Accepting Mapper Key with ordinal: " + mapCounter); +} + doMap(key, value, context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); @@ -53,11 +60,11 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { super.map(key, value, context); } - + @Override final protected void cleanup(Mapper .Context context) throws IOException, InterruptedException { try { @@ -76,7 +83,7 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ecb0d9/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java -- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java index 2b63ce0..83266ea 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java @@ -18,18 +18,22 @@ package org.apache.kylin.engine.mr; -import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Reducer; +import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; + /** */ public class
kylin git commit: measure encoding length bug
Repository: kylin Updated Branches: refs/heads/master 58224921d -> 1a5295bad measure encoding length bug Signed-off-by: Hongbin MaProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba Branch: refs/heads/master Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185 Parents: 5822492 Author: luguosheng <550175...@qq.com> Authored: Thu Dec 1 17:54:35 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 18:17:35 2016 +0800 -- webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++-- webapp/app/js/controllers/cubeEdit.js | 2 +- webapp/app/js/controllers/cubeMeasures.js | 8 3 files changed, 7 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js -- diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js index f2b36fe..8192419 100644 --- a/webapp/app/js/controllers/cubeAdvanceSetting.js +++ b/webapp/app/js/controllers/cubeAdvanceSetting.js @@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi var _encoding = item.encoding; var _valueLength ; var baseKey=item.encoding.replace(/:\d+/,''); -if(needLengthKeyList.indexOf(baseKey)>=-1){ +if(needLengthKeyList.indexOf(baseKey)!=-1){ var result=/:(\d+)/.exec(item.encoding); _valueLength=result?result[1]:0; } @@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi var version=$scope.getTypeVersion(item.encoding); var encodingType=$scope.removeVersion(item.encoding); -if(needLengthKeyList.indexOf(encodingType)>=-1){ +if(needLengthKeyList.indexOf(encodingType)!=-1){ encoding = encodingType+":"+item.valueLength; }else{ encoding = encodingType; http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js -- diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js index 0704a62..85bd4b1 100755 --- a/webapp/app/js/controllers/cubeEdit.js +++ b/webapp/app/js/controllers/cubeEdit.js @@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio $scope.store.supportedEncoding = $scope.cubeConfig.encodings; }) $scope.createFilter=function(type){ - if(type.indexOf("varchar")<=0){ + if(type.indexOf("varchar")==-1){ return ['fixed_length_hex']; }else if(type!="date"){ return ['date']; http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js -- diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js index 085338f..18f53c2 100644 --- a/webapp/app/js/controllers/cubeMeasures.js +++ b/webapp/app/js/controllers/cubeMeasures.js @@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes } } $scope.createFilter=function(type){ -if(type.indexOf("varchar")<=0){ +if(type.indexOf("varchar")==-1){ return ['fixed_length_hex']; }else if(type!="date"){ return ['date']; @@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes var encodings =$scope.store.supportedEncoding,filterEncoding=[]; var filerList=$scope.createFilter(type); if($scope.isEdit) { - if (name && $scope.newMeasure.function.configuration) { + if (name && $scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.' + name]) { var version = $scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1; filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 'value', $scope.newMeasure.function.configuration['topn.encoding.' + name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 'suggest', true); }else{ @@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes var version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1; item=$scope.removeVersion(item); var baseKey=item.replace(/:\d+/,''); -if(needLengthKeyList.indexOf(baseKey)>=-1){ +if(needLengthKeyList.indexOf(baseKey)!=-1){
kylin git commit: KYLIN-1875 Support measure on non-root-fact-table
Repository: kylin Updated Branches: refs/heads/KYLIN-1875 28ae57b81 -> a9100bfc6 KYLIN-1875 Support measure on non-root-fact-table Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9100bfc Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9100bfc Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9100bfc Branch: refs/heads/KYLIN-1875 Commit: a9100bfc6e7d3b601f4721d0bafa198ebb810a98 Parents: 28ae57b Author: Li YangAuthored: Thu Dec 1 18:59:08 2016 +0800 Committer: Li Yang Committed: Thu Dec 1 18:59:08 2016 +0800 -- .../kylin/metadata/project/ProjectL2Cache.java | 26 +--- .../template/cube_desc/kylin_sales_cube.json| 18 +++--- .../kylin/query/relnode/OLAPTableScan.java | 4 +++ 3 files changed, 30 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a9100bfc/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index 82c0de3..14d7843 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; @@ -126,17 +127,17 @@ class ProjectL2Cache { return Collections.unmodifiableSet(tableCache.realizations); } -public List listEffectiveRewriteMeasures(String project, String factTable, boolean onlyRewriteMeasure) { -Set realizations = getRealizationsByTable(project, factTable); +public List listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) { +Set realizations = getRealizationsByTable(project, table); List result = Lists.newArrayList(); for (IRealization r : realizations) { -if (r.getModel().isFactTable(factTable) && r.isReady()) { -for (MeasureDesc m : r.getMeasures()) { -FunctionDesc func = m.getFunction(); -if (onlyRewriteMeasure) { -if (func.needRewrite()) -result.add(m); -} else { +if (!r.isReady()) +continue; + +for (MeasureDesc m : r.getMeasures()) { +FunctionDesc func = m.getFunction(); +if (belongToTable(func, table, r.getModel())) { +if (!onlyRewriteMeasure || func.needRewrite()) { result.add(m); } } @@ -145,6 +146,13 @@ class ProjectL2Cache { return result; } +private boolean belongToTable(FunctionDesc func, String table, DataModelDesc model) { +// measure belong to the first column parameter's table +List cols = func.getParameter().getColRefs(); +String belongTo = cols.isEmpty() ? model.getRootFactTable().getTableIdentity() : cols.get(0).getTable(); +return belongTo.equals(table); +} + // // build the cache // http://git-wip-us.apache.org/repos/asf/kylin/blob/a9100bfc/examples/sample_cube/template/cube_desc/kylin_sales_cube.json -- diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json index 48eef46..2a27305 100644 --- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json +++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json @@ -103,26 +103,26 @@ "returntype" : "decimal(19,4)" } }, { -"name" : "GMV_MIN", +"name" : "BUYER_LEVEL_SUM", "function" : { - "expression" : "MIN", + "expression" : "SUM", "parameter" : { "type" : "column", -"value" : "PRICE", +"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "next_parameter" : null }, - "returntype" : "decimal(19,4)" + "returntype" : "bigint" } }, { -"name" : "GMV_MAX", +"name" :
kylin git commit: refine mapper and reducer log
Repository: kylin Updated Branches: refs/heads/master 28ba1eaea -> 58224921d refine mapper and reducer log Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58224921 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58224921 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58224921 Branch: refs/heads/master Commit: 58224921d896e4479f5d034d43c044aacaf14200 Parents: 28ba1ea Author: Hongbin MaAuthored: Thu Dec 1 18:15:46 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 1 18:15:46 2016 +0800 -- .../java/org/apache/kylin/engine/mr/KylinMapper.java | 12 +--- .../java/org/apache/kylin/engine/mr/KylinReducer.java | 13 ++--- .../apache/kylin/engine/mr/steps/CuboidReducer.java| 11 --- .../kylin/engine/mr/steps/HiveToBaseCuboidMapper.java | 6 -- .../kylin/engine/mr/steps/InMemCuboidMapper.java | 7 +-- .../kylin/engine/mr/steps/InMemCuboidReducer.java | 10 +- .../apache/kylin/engine/mr/steps/NDCuboidMapper.java | 10 -- 7 files changed, 33 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java -- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java index a01f7a2..2b564e9 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Mapper; +import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory; public class KylinMapper extends Mapper { private static final Logger logger = LoggerFactory.getLogger(KylinMapper.class); +protected int mapCounter = 0; + protected void bindCurrentConfiguration(Configuration conf) { logger.info("The conf for current mapper will be " + System.identityHashCode(conf)); HadoopUtil.setCurrentConfiguration(conf); @@ -38,6 +41,9 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { try { +if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { +logger.info("Accepting Mapper Key with ordinal: " + mapCounter); +} doMap(key, value, context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); @@ -53,11 +59,11 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { super.map(key, value, context); } - + @Override final protected void cleanup(Mapper .Context context) throws IOException, InterruptedException { try { @@ -76,7 +82,7 @@ public class KylinMapper extends Mapper .Context context) throws IOException, InterruptedException { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java -- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java index 2b63ce0..cb2d7a7 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Reducer; +import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +30,9 @@ import org.slf4j.LoggerFactory; */ public class KylinReducer extends Reducer {
kylin git commit: KYLIN-2213 add test query
Repository: kylin Updated Branches: refs/heads/master 9fc8f5e20 -> 3091f06a6 KYLIN-2213 add test query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3091f06a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3091f06a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3091f06a Branch: refs/heads/master Commit: 3091f06a68e0b97b35639cbb65acbf532c114ca4 Parents: 9fc8f5e Author: Li YangAuthored: Fri Dec 2 13:21:54 2016 +0800 Committer: Li Yang Committed: Fri Dec 2 13:21:54 2016 +0800 -- .../test/resources/query/sql_like/query20.sql | 31 1 file changed, 31 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/3091f06a/kylin-it/src/test/resources/query/sql_like/query20.sql -- diff --git a/kylin-it/src/test/resources/query/sql_like/query20.sql b/kylin-it/src/test/resources/query/sql_like/query20.sql new file mode 100644 index 000..6101cb0 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query20.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select lstg_format_name as lstg_format_name, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where lstg_format_name not like '%BIN%' +group by lstg_format_name \ No newline at end of file
kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally
Repository: kylin Updated Branches: refs/heads/master 3091f06a6 -> e562aafee KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e562aafe Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e562aafe Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e562aafe Branch: refs/heads/master Commit: e562aafee10c8bbd125937503d42b958c1843aad Parents: 3091f06 Author: Hongbin MaAuthored: Fri Dec 2 13:33:02 2016 +0800 Committer: Hongbin Ma Committed: Fri Dec 2 13:33:18 2016 +0800 -- .../kylin/common/debug/BackdoorToggles.java | 18 ++ .../java/org/apache/kylin/jdbc/IRemoteClient.java | 3 ++- .../java/org/apache/kylin/jdbc/KylinClient.java | 7 --- .../org/apache/kylin/jdbc/KylinResultSet.java | 9 - .../org/apache/kylin/jdbc/json/QueryRequest.java | 12 .../java/org/apache/kylin/jdbc/DummyClient.java | 3 ++- .../apache/kylin/rest/service/QueryService.java | 11 ++- 7 files changed, 56 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java index f7c90aa..28f7697 100644 --- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java @@ -79,6 +79,14 @@ public class BackdoorToggles { } } +public static Integer getStatementMaxRows() { +String v = getString(ATTR_STATEMENT_MAX_ROWS); +if (v == null) +return null; +else +return Integer.valueOf(v); +} + private static String getString(String key) { Map toggles = _backdoorToggles.get(); if (toggles == null) { @@ -183,4 +191,14 @@ public class BackdoorToggles { */ public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT"; +// properties on statement may go with this "channel" too +/** + * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property + * + example:(put it into request body) + "backdoorToggles": { + "ATTR_STATEMENT_MAX_ROWS": "10" + } + */ +public final static String ATTR_STATEMENT_MAX_ROWS = "ATTR_STATEMENT_MAX_ROWS"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java -- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java index b6a13e5..dfd8d76 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java @@ -21,6 +21,7 @@ package org.apache.kylin.jdbc; import java.io.Closeable; import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.ColumnMetaData; @@ -51,6 +52,6 @@ public interface IRemoteClient extends Closeable { /** * Execute query remotely and get back result. */ -public QueryResult executeQuery(String sql, List params, List paramValues) throws IOException; +public QueryResult executeQuery(String sql, List params, List paramValues, Map queryToggles) throws IOException; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java -- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java index 2d06a92..86c3a5b 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java @@ -321,9 +321,9 @@ public class KylinClient implements IRemoteClient { } @Override -public QueryResult executeQuery(String sql, List params, List paramValues) throws IOException { +public QueryResult executeQuery(String sql, List params, List paramValues, Map queryToggles) throws IOException { -SQLResponseStub queryResp = executeKylinQuery(sql, convertParameters(params, paramValues)); +SQLResponseStub queryResp = executeKylinQuery(sql,
[2/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv -- diff --git a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv index caa8754..949599d 100644 --- a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv +++ b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv @@ -1,402 +1,402 @@ -2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,1001,1,Ebay -2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,1002,1,Ebay -2013-04-06,82494,15,BookMagazines,NULL,Comic Books,Auction,14,66.6,1003,1,Ebay -2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,1004,1,Ebay -2013-05-20,152801,0,Jewelry & Watches,NULL,Earrings,FP-GTC,5,132.33,1005,1,Ebay -2013-06-16,43398,0,Home & Garden,NULL,Cheese & Crackers,FP-GTC,13,7.12,1006,1,Ebay -2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & Kits,Auction,14,204.28,1007,1,Ebay -2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & Manga,Auction,13,35.72,1008,1,Ebay -2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: Other,Auction,12,4.13,1009,1,Ebay -2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed Items,FP-GTC,5,27.48,1010,1,Ebay -2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,FP-GTC,14,9.26,1011,1,Ebay -2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & Business,Auction,12,3.18,1012,1,Ebay -2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,Others,13,3.18,1013,1,Ebay -2013-06-15,87118,0,Sporting Goods,Outdoor Sports,Paintball,ABIN,14,377.94,1014,1,Ebay -2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,Auction,12,146.33,1015,1,Ebay -2013-09-01,170302,15,Crafts,Embroidery,Design CDs,FP-GTC,5,51.23,1016,1,Ebay -2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,FP-non GTC,13,72.65,1017,1,Ebay -2013-05-31,132939,0,Jewelry & Watches,Fashion Jewelry,Other,Auction,13,66.6,1018,1,Ebay -2013-03-18,113593,15,Phones,Mobile Phones,Mobile Phones,Auction,12,9.26,1019,1,Ebay -2013-07-19,34273,100,eBay Motors,Parts & Accessories,Motorcycle,Auction,14,583.44,1020,1,Ebay -2013-06-23,106340,15,Home & Garden,Gardening,Hand Tools,FP-GTC,14,638.72,1021,1,Ebay -2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,1022,1,Ebay -2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,FP-GTC,12,319.79,1023,1,Ebay -2013-03-11,37831,0,Collectibles,Advertising,Merchandise & Memorabilia,Auction,12,20.35,1024,1,Ebay -2013-01-30,1120,3,Books,First Editions,Other,FP-non GTC,5,223.63,1025,1,Ebay -2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV Parts,FP-GTC,13,204.28,1026,1,Ebay -2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,Auction,14,5.48,1027,1,Ebay -2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,1028,1,Ebay -2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's Bags,FP-GTC,5,21.72,1029,1,Ebay -2013-10-29,2023,0,Sporting Goods,Team Sports,Basketball,ABIN,12,3.18,1030,1,Ebay -2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car Video,FP-GTC,11,491.32,1031,1,Ebay -2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,1032,1,Ebay -2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & Fixtures,Auction,13,415.73,1033,1,Ebay -2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,Auction,13,377.94,1034,1,Ebay -2013-05-27,33038,15,Musical Instruments,Instruments,Guitars (Electric),FP-GTC,14,146.33,1035,1,Ebay -2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,5,7.12,1036,1,Ebay -2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,Auction,13,42.99,1037,1,Ebay -2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,Auction,13,12.85,1038,1,Ebay -2013-08-20,15115,0,Video Games & Consoles,Video Games,Video Games,FP-GTC,13,55.89,1039,1,Ebay -2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,FP-GTC,14,73.26,1040,1,Ebay -2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,Auction,11,112.56,1041,1,Ebay -2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,FP-non GTC,11,3.49,1042,1,Ebay -2013-08-14,121153,0,Baby,Nursery Decor,Night Lights,Auction,13,184.21,1043,1,Ebay -2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,Auction,13,157.14,1044,1,Ebay
[4/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary
KYLIN-2192 More Robust Global Dictionary Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798 Branch: refs/heads/master Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1 Parents: e562aaf Author: sunyeruiAuthored: Mon Nov 21 21:26:34 2016 +0800 Committer: gaodayue Committed: Fri Dec 2 13:33:59 2016 +0800 -- .../kylin/job/dataGen/FactTableGenerator.java | 12 +- .../apache/kylin/common/KylinConfigBase.java| 8 + .../apache/kylin/common/util/Dictionary.java| 2 +- .../model/validation/rule/DictionaryRule.java | 78 +- .../validation/rule/DictionaryRuleTest.java | 28 +- .../apache/kylin/dict/AppendTrieDictionary.java | 285 +-- .../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++ .../org/apache/kylin/dict/CachedTreeMap.java| 260 +++--- .../kylin/dict/GlobalDictionaryBuilder.java | 36 +- .../kylin/dict/AppendTrieDictionaryTest.java| 150 +++- .../apache/kylin/dict/CachedTreeMapTest.java| 320 +--- .../kylin/measure/bitmap/BitmapCounterTest.java | 6 +- ...t_kylin_cube_without_slr_left_join_desc.json | 16 +- .../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv | 804 +-- .../flatten_data_for_without_slr_left_join.csv | 804 +-- .../test_kylin_inner_join_model_desc.json | 3 +- .../test_kylin_inner_join_view_model_desc.json | 3 +- .../test_kylin_left_join_model_desc.json| 3 +- .../test_kylin_left_join_view_model_desc.json | 3 +- .../table/DEFAULT.TEST_KYLIN_FACT.json | 8 +- .../source/hive/ITHiveTableReaderTest.java | 2 +- .../query/sql_distinct_precisely/query00.sql| 2 +- .../query/sql_distinct_precisely/query01.sql| 2 +- .../query/sql_distinct_precisely/query02.sql| 2 +- .../query/sql_distinct_precisely/query03.sql| 3 +- .../query/sql_distinct_precisely/query04.sql| 3 +- .../query/sql_distinct_precisely/query05.sql| 25 - .../query/sql_distinct_precisely/query06.sql| 26 - .../query/sql_distinct_precisely/query07.sql| 24 - 29 files changed, 1737 insertions(+), 1283 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java -- diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java index 8068fd1..677b713 100644 --- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java +++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java @@ -403,13 +403,13 @@ public class FactTableGenerator { } private String createRandomCell(ColumnDesc cDesc) { -String type = cDesc.getTypeName(); -String s = type.toLowerCase(); -if (s.equals("string") || s.equals("char") || s.equals("varchar")) { +DataType type =cDesc.getType(); +String s = type.getName(); +if (s.equals("char") || s.equals("varchar")) { StringBuilder sb = new StringBuilder(); -for (int i = 0; i < 2; i++) { -sb.append((char) ('a' + r.nextInt(10)));// there are 10*10 -// possible strings +int len = Math.min(type.getPrecision(), 3); +for (int i = 0; i < len; i++) { +sb.append((char) ('a' + r.nextInt(10))); // cardinality at most 10x10x10 } return sb.toString(); } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3c10826..f46c185 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "1000")); } +public int getAppendDictMaxVersions() { +return Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3")); +} + +public int getAppendDictVersionTTL() { +return
[1/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary
Repository: kylin Updated Branches: refs/heads/master e562aafee -> 4a0ee7989 http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql index a3948c3..0c163a6 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql @@ -19,6 +19,6 @@ select lstg_format_name, cal_dt, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct seller_id) as seller_count + count(distinct user_id) as user_count from test_kylin_fact group by lstg_format_name, cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql index e8579ef..62142a2 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql @@ -19,7 +19,7 @@ select lstg_format_name, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct seller_id) as seller_count + count(distinct user_id) as user_count from test_kylin_fact where lstg_format_name='FP-GTC' group by lstg_format_name http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql index 48f49e9..88ce532 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql @@ -19,7 +19,7 @@ select lstg_format_name, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct seller_id) as seller_count + count(distinct user_id) as user_count from test_kylin_fact where lstg_format_name='FP-GTC' group by lstg_format_name http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql index dbc2fac..7f739e7 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql @@ -17,7 +17,8 @@ -- select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV - , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count + , count(1) as TRANS_CNT + , count(distinct user_id) as user_count , count(distinct site_name) as site_count from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql index 69006ce..13de5a7 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql @@ -17,7 +17,8 @@ -- select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV - , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count + , count(1) as TRANS_CNT + , count(distinct user_id) as user_count , count(distinct site_name) as site_count from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql -- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql deleted file mode 100644 index dea09f7..000 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql +++ /dev/null @@ -1,25 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version
[3/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java -- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java index df64f3f..381e0b1 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java @@ -17,11 +17,12 @@ */ package org.apache.kylin.dict; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.Test; import java.io.*; @@ -91,39 +92,26 @@ public class CachedTreeMapTest { public static class CachedFileFilter implements FileFilter { @Override public boolean accept(File pathname) { -return pathname.getName().startsWith("cached_"); +return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX); } } -public static final String baseDir = "/tmp/kylin_cachedtreemap_test/"; -public static final String backupDir = "/tmp/kylin_cachedtreemap_test.bak/"; -public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/"; - -private static void cleanup() { -File dir = new File(baseDir); -if (dir.exists()) { -for (File f : dir.listFiles()) { -f.delete(); -} -dir.delete(); -} - -dir = new File(tmpDir); -if (dir.exists()) { -for (File f : dir.listFiles()) { -f.delete(); -} -dir.delete(); +public static class VersionFilter implements FileFilter { +@Override +public boolean accept(File pathname) { +return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX); } +} -dir = new File(backupDir); -if (dir.exists()) { -for (File f : dir.listFiles()) { -f.delete(); -} -dir.delete(); -} +public static final String baseDir = "/tmp/kylin_cachedtreemap_test/"; +public static final String workingDir = "/tmp/kylin_cachedtreemap_test/working"; +private static void cleanup() { +Configuration conf = new Configuration(); +Path basePath = new Path(baseDir); +try { +FileSystem.get(basePath.toUri(), conf).delete(basePath, true); +} catch (IOException e) {} VALUE_WRITE_ERROR_TOGGLE = false; } @@ -139,154 +127,240 @@ public class CachedTreeMapTest { @Test public void testCachedTreeMap() throws IOException { -CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) - .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); +CachedTreeMap map = createMutableMap(); map.put(Key.of(1), Value.of("a")); map.put(Key.of(2), Value.of("b")); map.put(Key.of(3), Value.of("c")); map.put(Key.of(4), Value.of("d")); map.put(Key.of(5), Value.of("e")); -File dir = new File(tmpDir); +File dir = new File(workingDir); assertEquals(3, dir.listFiles(new CachedFileFilter()).length); -DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpDir+"/.index")); -map.write(out); -out.flush(); -out.close(); -map.commit(false); +flushAndCommit(map, true, true, false); +assertFalse(new File(workingDir).exists()); -dir = new File(baseDir); +dir = new File(map.getLatestVersion()); assertEquals(5, dir.listFiles(new CachedFileFilter()).length); -DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index")); -CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) - .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); -map2.readFields(in); +CachedTreeMap map2 = createImmutableMap(); assertEquals(5, map2.size()); assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr); try { map2.put(Key.of(6), Value.of("f")); fail("Should be error when put value into immutable map"); -} catch (AssertionError error) { +} catch (AssertionError error) {} +} + +@Test +public void testMultiVersions() throws IOException, InterruptedException { +CachedTreeMap map = createMutableMap(); +Thread.sleep(3000); +map.put(Key.of(1), Value.of("a")); +map.put(Key.of(2),