This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d494b8d05f75fd648d2dbb1dcb4b72f452de8b55 Author: Zhong, Yanghong <nju_y...@apache.org> AuthorDate: Fri Apr 10 10:43:02 2020 +0800 KYLIN-4421 Allow to update table & database name --- .../java/org/apache/kylin/cube/CubeInstance.java | 65 ++- .../java/org/apache/kylin/cube/CubeSegment.java | 41 ++ .../apache/kylin/cube/model/AggregationGroup.java | 2 + .../java/org/apache/kylin/cube/model/CubeDesc.java | 65 +++ .../apache/kylin/cube/model/DictionaryDesc.java | 17 + .../kylin/cube/model/HBaseColumnFamilyDesc.java | 16 + .../org/apache/kylin/cube/model/SelectRule.java | 40 ++ .../apache/kylin/cube/model/SnapshotTableDesc.java | 33 +- .../apache/kylin/metadata/model/DataModelDesc.java | 46 ++ .../apache/kylin/metadata/model/JoinTableDesc.java | 9 + .../kylin/metadata/model/ModelDimensionDesc.java | 18 + .../apache/kylin/metadata/model/ParameterDesc.java | 7 +- .../org/apache/kylin/metadata/model/TableDesc.java | 14 +- .../service/update/TableSchemaUpdateMapping.java | 71 +++ .../rest/service/update/TableSchemaUpdater.java | 191 +++++++ .../service/update/TableSchemaUpdaterTest.java | 154 +++++ .../resources/update/TableSchemaUpdateMapping.json | 19 + .../test_kylin_cube_with_slr_left_join_ready.json | 44 ++ .../update/cube_desc/ci_left_join_cube.json | 619 +++++++++++++++++++++ .../update/model_desc/ci_inner_join_model.json | 238 ++++++++ .../test/resources/update/table/EDW.CAL_DT.json | 413 ++++++++++++++ .../test/resources/update/table/TEST.COUNTRY.json | 26 + .../resources/update/table/TEST.KYLIN_FACT.json | 73 +++ .../resources/update/table/TEST.TEST_ACCOUNT.json | 36 ++ .../update/table/TEST.TEST_CATEGORY_GROUPINGS.json | 155 ++++++ 25 files changed, 2393 insertions(+), 19 deletions(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index eb90d56..454816d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -24,6 +24,7 @@ import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -273,6 +274,33 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return true; } + public boolean equalsRaw(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + CubeInstance that = (CubeInstance) o; + if (!java.util.Objects.equals(name, that.name)) + return false; + if (!java.util.Objects.equals(owner, that.owner)) + return false; + if (!java.util.Objects.equals(descName, that.descName)) + return false; + if (!java.util.Objects.equals(displayName, that.displayName)) + return false; + if (!java.util.Objects.equals(status, that.status)) + return false; + + if (!java.util.Objects.equals(segments, that.segments)) + return false; + if (!java.util.Arrays.equals(cuboidBytes, that.cuboidBytes)) + return false; + if (!java.util.Arrays.equals(cuboidBytesRecommend, that.cuboidBytesRecommend)) + return false; + return java.util.Objects.equals(snapshots, that.snapshots); + } + // ============================================================================ @Override @@ -695,6 +723,10 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return snapshots; } + public void resetSnapshots() { + snapshots = Maps.newHashMap(); + } + public String getSnapshotResPath(String tableName) { return getSnapshots().get(tableName); } @@ -703,18 +735,27 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, getSnapshots().put(table, snapshotResPath); } - public static CubeInstance getCopyOf(CubeInstance cubeInstance) { - CubeInstance newCube = new CubeInstance(); - newCube.setName(cubeInstance.getName()); - newCube.setSegments(cubeInstance.getSegments()); - newCube.setDescName(cubeInstance.getDescName()); - newCube.setConfig((KylinConfigExt) cubeInstance.getConfig()); - newCube.setStatus(cubeInstance.getStatus()); - newCube.setOwner(cubeInstance.getOwner()); - newCube.setCost(cubeInstance.getCost()); - newCube.setCreateTimeUTC(System.currentTimeMillis()); - newCube.updateRandomUuid(); - return newCube; + public static CubeInstance getCopyOf(CubeInstance other) { + CubeInstance ret = new CubeInstance(); + ret.setName(other.getName()); + ret.setOwner(other.getOwner()); + ret.setDescName(other.getDescName()); + ret.setCost(other.getCost()); + ret.setStatus(other.getStatus()); + ret.setSegments(other.getSegments()); + ret.setCreateTimeUTC(System.currentTimeMillis()); + if (other.cuboidBytes != null) { + ret.cuboidBytes = Arrays.copyOf(other.cuboidBytes, other.cuboidBytes.length); + } + if (other.cuboidBytesRecommend != null) { + ret.cuboidBytesRecommend = Arrays.copyOf(other.cuboidBytesRecommend, other.cuboidBytesRecommend.length); + } + ret.cuboidLastOptimized = other.cuboidLastOptimized; + ret.getSnapshots().putAll(other.getSnapshots()); + + ret.setConfig((KylinConfigExt) other.getConfig()); + ret.updateRandomUuid(); + return ret; } public static CubeSegment findSegmentWithJobId(String jobID, CubeInstance cubeInstance) { diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index 52f2034..e423f57 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -321,6 +321,10 @@ public class CubeSegment implements IBuildable, ISegment, Serializable { return snapshots; } + public void resetSnapshots() { + snapshots = new ConcurrentHashMap<String, String>(); + } + public String getSnapshotResPath(String table) { return getSnapshots().get(table); } @@ -660,4 +664,41 @@ public class CubeSegment implements IBuildable, ISegment, Serializable { public void setStreamSourceCheckpoint(String streamSourceCheckpoint) { this.streamSourceCheckpoint = streamSourceCheckpoint; } + + public static CubeSegment getCopyOf(CubeSegment other) { + CubeSegment copy = new CubeSegment(); + copy.cubeInstance = other.cubeInstance; + copy.uuid = other.uuid; + copy.name = other.name; + copy.storageLocationIdentifier = other.storageLocationIdentifier; + copy.dateRangeStart = other.dateRangeStart; + copy.dateRangeEnd = other.dateRangeEnd; + copy.sourceOffsetStart = other.sourceOffsetStart; + copy.sourceOffsetEnd = other.sourceOffsetEnd; + copy.status = other.status; + copy.sizeKB = other.sizeKB; + copy.isMerged = other.isMerged; + copy.estimateRatio = other.estimateRatio == null ? null : Lists.newArrayList(other.estimateRatio); + copy.inputRecords = other.inputRecords; + copy.inputRecordsSize = other.inputRecordsSize; + copy.lastBuildTime = other.lastBuildTime; + copy.lastBuildJobID = other.lastBuildJobID; + copy.createTimeUTC = other.createTimeUTC; + copy.cuboidShardNums.putAll(other.cuboidShardNums); + copy.totalShards = other.totalShards; + copy.blackoutCuboids.addAll(other.blackoutCuboids); + copy.getDictionaries().putAll(other.getDictionaries()); + copy.getSnapshots().putAll(other.getSnapshots()); + copy.rowkeyStats.addAll(other.rowkeyStats); + copy.sourcePartitionOffsetStart.putAll(other.sourcePartitionOffsetStart); + copy.sourcePartitionOffsetEnd.putAll(other.sourcePartitionOffsetEnd); + if (other.streamSourceCheckpoint != null) { + copy.streamSourceCheckpoint = other.streamSourceCheckpoint; + } + copy.additionalInfo.putAll(other.additionalInfo); + copy.dimensionRangeInfoMap = other.dimensionRangeInfoMap == null ? null + : Maps.newHashMap(other.dimensionRangeInfoMap); + copy.binarySignature = other.binarySignature; + return copy; + } } diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 7de211e..c03cca1 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -540,4 +540,6 @@ public class AggregationGroup implements Serializable { public int getDimCap() { return this.selectRule.dimCap == null ? 0 : this.selectRule.dimCap; } + + } diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 364ad6d..13311c0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -42,6 +42,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.stream.IntStream; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; @@ -484,6 +485,70 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return mandatoryCuboids; } + public boolean equalsRaw(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + CubeDesc that = (CubeDesc) o; + + if (!Objects.equals(name, that.name)) + return false; + if (!Objects.equals(modelName, that.modelName)) + return false; + if (!Objects.equals(description, that.description)) + return false; + if (!Objects.equals(dimensions, that.dimensions)) + return false; + if (!Objects.equals(measures, that.measures)) + return false; + if (!Objects.equals(dictionaries, that.dictionaries)) + return false; + if (!Arrays.equals(rowkey.getRowKeyColumns(), that.rowkey.getRowKeyColumns())) + return false; + if (!Objects.equals(nullStrings, that.nullStrings)) + return false; + if (!Arrays.equals(hbaseMapping.getColumnFamily(), that.hbaseMapping.getColumnFamily())) + return false; + if (aggregationGroups != that.aggregationGroups) { + if (aggregationGroups == null || that.aggregationGroups == null) { + return false; + } else if (!IntStream.range(0, aggregationGroups.size()) + .allMatch(i -> Arrays.equals(aggregationGroups.get(i).getIncludes(), + that.aggregationGroups.get(i).getIncludes()) + && Objects.equals(aggregationGroups.get(i).getSelectRule(), + that.aggregationGroups.get(i).getSelectRule()))) { + return false; + } + } + if (!Objects.equals(notifyList, that.notifyList)) + return false; + if (!Objects.equals(statusNeedNotify, that.statusNeedNotify)) + return false; + if (!Arrays.equals(autoMergeTimeRanges, that.autoMergeTimeRanges)) + return false; + if (!Objects.equals(retentionRange, that.retentionRange)) + return false; + if (!Objects.equals(engineType, that.engineType)) + return false; + if (!Objects.equals(storageType, that.storageType)) + return false; + if (!Objects.equals(overrideKylinProps, that.overrideKylinProps)) + return false; + if (!Objects.equals(snapshotTableDescList, that.snapshotTableDescList)) + return false; + if (!Objects.equals(partitionDateStart, that.partitionDateStart)) + return false; + if (!Objects.equals(partitionDateEnd, that.partitionDateEnd)) + return false; + if (!Objects.equals(parentForward, that.parentForward)) + return false; + if (!Objects.equals(mandatoryDimensionSetList, that.mandatoryDimensionSetList)) + return false; + return Objects.equals(cuboidBlackSet, that.cuboidBlackSet); + } + @Override public boolean equals(Object o) { if (this == o) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java index 2d1ba99..248984e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java @@ -118,4 +118,21 @@ public class DictionaryDesc implements java.io.Serializable { desc.builderClass = builderClass; return desc; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DictionaryDesc that = (DictionaryDesc) o; + return Objects.equals(column, that.column) && + Objects.equals(reuseColumn, that.reuseColumn) && + Objects.equals(builderClass, that.builderClass) && + Objects.equals(cube, that.cube) && + Objects.equals(model, that.model); + } + + @Override + public int hashCode() { + return Objects.hash(column, reuseColumn, builderClass, cube, model); + } } diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 85c2c17..7550751 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.kylin.metadata.model.MeasureDesc; import java.util.Arrays; +import java.util.Objects; /** */ @@ -67,4 +68,19 @@ public class HBaseColumnFamilyDesc implements java.io.Serializable { return false; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HBaseColumnFamilyDesc that = (HBaseColumnFamilyDesc) o; + return Objects.equals(name, that.name) && + Arrays.equals(columns, that.columns); + } + + @Override + public int hashCode() { + int result = Objects.hash(name); + result = 31 * result + Arrays.hashCode(columns); + return result; + } } diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java index d78da9f..b3527ae 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java @@ -19,6 +19,9 @@ package org.apache.kylin.cube.model; import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.IntStream; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,4 +38,41 @@ public class SelectRule implements Serializable { @JsonProperty("dim_cap") @JsonInclude(JsonInclude.Include.NON_NULL) public Integer dimCap; + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + SelectRule that = (SelectRule) o; + if (hierarchyDims != that.hierarchyDims) { + if (hierarchyDims == null || that.hierarchyDims == null) { + return false; + } else if (!IntStream.range(0, hierarchyDims.length) + .allMatch(i -> Arrays.equals(hierarchyDims[i], that.hierarchyDims[i]))) { + return false; + } + } + + if (jointDims != that.jointDims) { + if (jointDims == null || that.jointDims == null) { + return false; + } else if (!IntStream.range(0, jointDims.length) + .allMatch(i -> Arrays.equals(jointDims[i], that.jointDims[i]))) { + return false; + } + } + return Arrays.equals(mandatoryDims, that.mandatoryDims) && Objects.equals(dimCap, that.dimCap); + } + + @Override + public int hashCode() { + int result = Objects.hash(dimCap); + result = 31 * result + Arrays.hashCode(hierarchyDims); + result = 31 * result + Arrays.hashCode(mandatoryDims); + result = 31 * result + Arrays.hashCode(jointDims); + return result; + } } diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java index e61240b..30f533b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java @@ -18,13 +18,17 @@ package org.apache.kylin.cube.model; +import org.apache.kylin.dict.lookup.SnapshotTable; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.kylin.dict.lookup.SnapshotTable; + +import java.util.Objects; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class SnapshotTableDesc implements java.io.Serializable{ +public class +SnapshotTableDesc implements java.io.Serializable{ @JsonProperty("table_name") private String tableName; @@ -72,4 +76,29 @@ public class SnapshotTableDesc implements java.io.Serializable{ public void setEnableLocalCache(boolean enableLocalCache) { this.enableLocalCache = enableLocalCache; } + + public static SnapshotTableDesc getCopyOf(SnapshotTableDesc other) { + SnapshotTableDesc copy = new SnapshotTableDesc(); + copy.tableName = other.tableName; + copy.storageType = other.storageType; + copy.enableLocalCache = other.enableLocalCache; + copy.global = other.global; + return copy; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SnapshotTableDesc that = (SnapshotTableDesc) o; + return enableLocalCache == that.enableLocalCache && + global == that.global && + Objects.equals(tableName, that.tableName) && + Objects.equals(storageType, that.storageType); + } + + @Override + public int hashCode() { + return Objects.hash(tableName, storageType, enableLocalCache, global); + } } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index e117176..d126d3d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -720,6 +720,52 @@ public class DataModelDesc extends RootPersistentEntity { return this.errors; } + private Map<String, JoinTableDesc> getJoinTableMap(JoinTableDesc[] joinTables) { + if (joinTables == null) { + return Maps.newHashMap(); + } + Map<String, JoinTableDesc> ret = Maps.newHashMapWithExpectedSize(joinTables.length); + for (JoinTableDesc joinTable : joinTables) { + ret.put(joinTable.getAlias(), joinTable); + } + return ret; + } + + public boolean equalsRaw(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + DataModelDesc that = (DataModelDesc) o; + + if (isDraft != that.isDraft) + return false; + if (name != null ? !name.equals(that.name) : that.name != null) + return false; + if (owner != null ? !owner.equals(that.owner) : that.owner != null) + return false; + if (description != null ? !description.equals(that.description) : that.description != null) + return false; + if (rootFactTable != null ? !rootFactTable.equals(that.rootFactTable) : that.rootFactTable != null) + return false; + if (rootFactTableAlias != null ? !rootFactTableAlias.equals(that.rootFactTableAlias) + : that.rootFactTableAlias != null) + return false; + if (!getJoinTableMap(joinTables).equals(getJoinTableMap(that.joinTables))) + return false; + if (dimensions != null ? !dimensions.equals(that.dimensions) : that.dimensions != null) + return false; + // Probably incorrect - comparing Object[] arrays with Arrays.equals + if (!Arrays.equals(metrics, that.metrics)) + return false; + if (filterCondition != null ? !filterCondition.equals(that.filterCondition) : that.filterCondition != null) + return false; + if (partitionDesc != null ? !partitionDesc.equalsRaw(that.partitionDesc) : that.partitionDesc != null) + return false; + return capacity == that.capacity; + } + @Override public boolean equals(Object o) { if (this == o) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java index dc4710e..f0d9111 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java @@ -87,6 +87,15 @@ public class JoinTableDesc implements Serializable { this.tableRef = ref; } + public static JoinTableDesc getCopyOf(JoinTableDesc other) { + JoinTableDesc copy = new JoinTableDesc(); + copy.table = other.table; + copy.kind = other.kind; + copy.alias = other.alias; + copy.join = other.join; + return copy; + } + @Override public boolean equals(Object o) { if (this == o) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java index caadbd6..cf729a4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java @@ -19,9 +19,12 @@ package org.apache.kylin.metadata.model; import java.io.Serializable; +import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Objects; + import org.apache.kylin.common.util.StringUtil; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -94,4 +97,19 @@ public class ModelDimensionDesc implements Serializable { return count; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ModelDimensionDesc that = (ModelDimensionDesc) o; + return Objects.equals(table, that.table) && + Arrays.equals(columns, that.columns); + } + + @Override + public int hashCode() { + int result = Objects.hash(table); + result = 31 * result + Arrays.hashCode(columns); + return result; + } } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java index 0633d1e..7748719 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Set; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -266,19 +267,19 @@ public class ParameterDesc implements Serializable { PlainParameter that = (PlainParameter) o; - if (type != null ? !type.equals(that.type) : that.type != null) + if (!Objects.equals(type, that.type)) return false; if (this.isColumnType()) { if (!that.isColumnType()) return false; - if (!this.colRef.equals(that.colRef)) { + if (!Objects.equals(colRef, that.colRef)) { return false; } } else { if (that.isColumnType()) return false; - if (!this.value.equals(that.value)) + if (!Objects.equals(value, that.value)) return false; } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index 588cf1e..d99ff54 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -237,8 +237,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { public String getIdentity() { if (identity == null) { - identity = String.format(Locale.ROOT, "%s.%s", this.getDatabase().toUpperCase(Locale.ROOT), this.getName()) - .toUpperCase(Locale.ROOT); + setIdentity(); } return identity; } @@ -279,6 +278,9 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { } else { this.name = null; } + if (identity != null) { + setIdentity(); + } } @JsonProperty("database") @@ -289,6 +291,14 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { @JsonProperty("database") public void setDatabase(String database) { this.database.setName(database); + if (identity != null) { + setIdentity(); + } + } + + private void setIdentity() { + identity = String.format(Locale.ROOT, "%s.%s", this.getDatabase().toUpperCase(Locale.ROOT), this.getName()) + .toUpperCase(Locale.ROOT); } public ColumnDesc[] getColumns() { diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdateMapping.java b/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdateMapping.java new file mode 100644 index 0000000..62b4a35 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdateMapping.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service.update; + +import java.util.Locale; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +public class TableSchemaUpdateMapping { + + private String database; + + private String tableName; + + public boolean isDatabaseChanged() { + return !Strings.isNullOrEmpty(database); + } + + public String getDatabase(String dbName) { + String ret = isDatabaseChanged() ? database : dbName; + return ret.toUpperCase(Locale.ROOT); + } + + public void setDatabase(String database) { + this.database = database; + } + + public boolean isTableNameChanged() { + return !Strings.isNullOrEmpty(tableName); + } + + public String getTableName(String tblName) { + String ret = isTableNameChanged() ? tableName : tblName; + return ret.toUpperCase(Locale.ROOT); + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public boolean isTableIdentityChanged() { + return isDatabaseChanged() || isTableNameChanged(); + } + + public String getTableIdentity(String tableIdentity) { + String[] tableNameEs = tableIdentity.split("\\."); + Preconditions.checkArgument(tableNameEs.length == 2); + return getTableIdentity(tableNameEs[0], tableNameEs[1]); + } + + public String getTableIdentity(String database, String tableName) { + return getDatabase(database) + "." + getTableName(tableName); + } +} \ No newline at end of file diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdater.java b/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdater.java new file mode 100644 index 0000000..99c88a0 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/service/update/TableSchemaUpdater.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service.update; + +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.SnapshotTableDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.JoinTableDesc; +import org.apache.kylin.metadata.model.PartitionDesc; +import org.apache.kylin.metadata.model.Segments; +import org.apache.kylin.metadata.model.TableDesc; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class TableSchemaUpdater { + + public static TableDesc dealWithMappingForTable(TableDesc other, Map<String, TableSchemaUpdateMapping> mappings) { + TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, other.getIdentity()); + if (mapping == null) { + return other; + } + + TableDesc copy = new TableDesc(other); + + copy.setDatabase(mapping.getDatabase(other.getDatabase())); + + copy.setName(mapping.getTableName(other.getName())); + + // It will always be a new one + copy.setLastModified(0L); + + return copy; + } + + // the input data model should be initialized, then table names & col names will be normalized + public static DataModelDesc dealWithMappingForModel(DataModelDesc other, + Map<String, TableSchemaUpdateMapping> mappings) { + // For filter condition, not support + if (!Strings.isNullOrEmpty(other.getFilterCondition())) { + throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition()); + } + + DataModelDesc copy = DataModelDesc.getCopyOf(other); + copy.setLastModified(other.getLastModified()); + + // mapping for root fact table identity + TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName()); + if (rootMapping != null) { + TableDesc rootFactTable = other.getRootFactTable().getTableDesc(); + copy.setRootFactTableName( + rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName())); + } + + // mapping for joins + JoinTableDesc[] joinTables = other.getJoinTables(); + JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length]; + for (int i = 0; i < joinTables.length; i++) { + JoinTableDesc joinTable = joinTables[i]; + joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable); + String tableIdentity = joinTable.getTable(); + TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity); + if (mapping != null && mapping.isTableIdentityChanged()) { + joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity)); + } + } + copy.setJoinTables(joinTablesCopy); + + // mapping for partition columns + PartitionDesc partDesc = other.getPartitionDesc(); + PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc); + if (partDesc.getPartitionDateColumnRef() != null) { + partCopy.setPartitionDateColumn( + replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings)); + } + if (partDesc.getPartitionTimeColumnRef() != null) { + partCopy.setPartitionTimeColumn( + replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings)); + } + copy.setPartitionDesc(partCopy); + + return copy; + } + + public static CubeDesc dealWithMappingForCubeDesc(CubeDesc other, Map<String, TableSchemaUpdateMapping> mappings) { + CubeDesc copy = CubeDesc.getCopyOf(other); + copy.setLastModified(other.getLastModified()); + + // mapping for cube-level snapshot tables + if (other.getSnapshotTableDescList() != null && !other.getSnapshotTableDescList().isEmpty()) { + List<SnapshotTableDesc> snapshotTableDescListCopy = Lists + .newArrayListWithExpectedSize(other.getSnapshotTableDescList().size()); + for (SnapshotTableDesc snapshotDesc : other.getSnapshotTableDescList()) { + TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, snapshotDesc.getTableName()); + if (mapping != null && mapping.isTableIdentityChanged()) { + snapshotDesc = SnapshotTableDesc.getCopyOf(snapshotDesc); + snapshotDesc.setTableName(mapping.getTableIdentity(snapshotDesc.getTableName())); + } + snapshotTableDescListCopy.add(snapshotDesc); + } + copy.setSnapshotTableDescList(snapshotTableDescListCopy); + } + + return copy; + } + + public static CubeInstance dealWithMappingForCube(CubeInstance other, + Map<String, TableSchemaUpdateMapping> mappings) { + CubeInstance copy = CubeInstance.getCopyOf(other); + copy.setLastModified(other.getLastModified()); + + // mapping for cube-level snapshot tables + if (other.getSnapshots() != null && !other.getSnapshots().isEmpty()) { + Map<String, String> snapshotsCopy = replaceTableIdentityForTableSnapshots(other.getSnapshots(), mappings); + copy.resetSnapshots(); + copy.getSnapshots().putAll(snapshotsCopy); + } + + // mapping for segment-level snapshot tables + if (other.getSegments() != null && !other.getSegments().isEmpty()) { + Segments<CubeSegment> segmentsCopy = new Segments<>(); + for (CubeSegment segment : other.getSegments()) { + CubeSegment segmentCopy = CubeSegment.getCopyOf(segment); + segmentCopy.setCubeInstance(copy); + Map<String, String> snapshotsCopy = replaceTableIdentityForTableSnapshots(segment.getSnapshots(), + mappings); + segmentCopy.resetSnapshots(); + segmentCopy.getSnapshots().putAll(snapshotsCopy); + segmentsCopy.add(segmentCopy); + } + copy.setSegments(segmentsCopy); + } + + return copy; + } + + private static Map<String, String> replaceTableIdentityForTableSnapshots(Map<String, String> snapshots, + Map<String, TableSchemaUpdateMapping> mappings) { + Map<String, String> snapshotsCopy = Maps.newHashMapWithExpectedSize(snapshots.size()); + for (String tableIdentity : snapshots.keySet()) { + String resPath = snapshots.get(tableIdentity); + TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity); + if (mapping != null && mapping.isTableIdentityChanged()) { + tableIdentity = mapping.getTableIdentity(tableIdentity); + } + snapshotsCopy.put(tableIdentity, resPath); + } + return snapshotsCopy; + } + + private static String replacePartitionCol(String partCol, Map<String, TableSchemaUpdateMapping> mappings) { + int cut = partCol.lastIndexOf('.'); + if (cut < 0) { + return partCol; + } + String partTableIdentity = partCol.substring(0, cut); + TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, partTableIdentity); + if (mapping != null) { + return mapping.getTableIdentity(partTableIdentity) + "." + partCol.substring(cut + 1); + } + return partCol; + } + + public static TableSchemaUpdateMapping getTableSchemaUpdateMapping(Map<String, TableSchemaUpdateMapping> mappings, + String key) { + return mappings.get(key.toUpperCase(Locale.ROOT)); + } +} diff --git a/server-base/src/test/java/org/apache/kylin/rest/service/update/TableSchemaUpdaterTest.java b/server-base/src/test/java/org/apache/kylin/rest/service/update/TableSchemaUpdaterTest.java new file mode 100644 index 0000000..7b8eecb --- /dev/null +++ b/server-base/src/test/java/org/apache/kylin/rest/service/update/TableSchemaUpdaterTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service.update; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Map; +import java.util.Set; + +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.TableMetadataManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; +import org.apache.kylin.metadata.model.TableDesc; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; + +public class TableSchemaUpdaterTest extends LocalFileMetadataTestCase { + + private final String mappingRootPath = "src/test/resources/update"; + private final String mappingFileName = "TableSchemaUpdateMapping.json"; + private Map<String, TableSchemaUpdateMapping> mappings; + + @Before + public void setUp() throws IOException { + this.createTestMetadata(); + + File mappingFile = new File(mappingRootPath + "/" + mappingFileName); + String content = new String(Files.readAllBytes(mappingFile.toPath()), StandardCharsets.UTF_8); + mappings = JsonUtil.readValue(content, new TypeReference<Map<String, TableSchemaUpdateMapping>>() { + }); + } + + @Test + public void testDealWithMappingForTable() throws IOException { + TableMetadataManager tableMetaManager = TableMetadataManager.getInstance(getTestConfig()); + ResourceStore store = tableMetaManager.getStore(); + + Set<TableDesc> tables = Sets.newHashSet(); + for (String tableIdentity : mappings.keySet()) { + tables.add(store.getResource(TableDesc.concatResourcePath(tableIdentity, null), + TableMetadataManager.TABLE_SERIALIZER)); + } + + for (TableDesc tableDesc : tables) { + TableDesc updated = TableSchemaUpdater.dealWithMappingForTable(tableDesc, mappings); + updated = reinit(updated, TableMetadataManager.TABLE_SERIALIZER); + + try (DataInputStream bis = new DataInputStream(new FileInputStream( + new File(mappingRootPath + TableDesc.concatResourcePath(updated.getIdentity(), null))))) { + TableDesc expected = TableMetadataManager.TABLE_SERIALIZER.deserialize(bis); + Assert.assertEquals(expected, updated); + } catch (Exception e) { + Assert.fail("Table identity is not updated correctly"); + } + } + } + + @Test + public void testDealWithMappingForModel() throws IOException { + DataModelManager dataModelManager = DataModelManager.getInstance(getTestConfig()); + DataModelDesc model = dataModelManager.getDataModelDesc("ci_inner_join_model"); + + DataModelDesc updated = TableSchemaUpdater.dealWithMappingForModel(model, mappings); + updated = reinit(updated, dataModelManager.getDataModelSerializer()); + + try (DataInputStream bis = new DataInputStream( + new FileInputStream(new File(mappingRootPath + DataModelDesc.concatResourcePath(updated.getName()))))) { + DataModelDesc expected = dataModelManager.getDataModelSerializer().deserialize(bis); + Assert.assertTrue(expected.equalsRaw(updated)); + } catch (Exception e) { + Assert.fail("Model is not updated correctly"); + } + } + + @Test + public void testDealWithMappingForCubeDesc() throws IOException { + CubeDescManager cubeDescManager = CubeDescManager.getInstance(getTestConfig()); + CubeDesc cubeDesc = cubeDescManager.getCubeDesc("ci_left_join_cube"); + + CubeDesc updated = TableSchemaUpdater.dealWithMappingForCubeDesc(cubeDesc, mappings); + updated = reinit(updated, cubeDescManager.CUBE_DESC_SERIALIZER); + + try (DataInputStream bis = new DataInputStream( + new FileInputStream(new File(mappingRootPath + CubeDesc.concatResourcePath(updated.getName()))))) { + CubeDesc expected = cubeDescManager.CUBE_DESC_SERIALIZER.deserialize(bis); + Assert.assertTrue(expected.equalsRaw(updated)); + } catch (Exception e) { + Assert.fail("CubeDesc is not updated correctly"); + } + } + + @Test + public void testDealWithMappingForCube() throws IOException { + CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); + CubeInstance cube = cubeManager.getCube("test_kylin_cube_with_slr_left_join_ready"); + + CubeInstance updated = TableSchemaUpdater.dealWithMappingForCube(cube, mappings); + updated = reinit(updated, cubeManager.CUBE_SERIALIZER); + + try (DataInputStream bis = new DataInputStream( + new FileInputStream(new File(mappingRootPath + CubeInstance.concatResourcePath(updated.getName()))))) { + CubeInstance expected = cubeManager.CUBE_SERIALIZER.deserialize(bis); + Assert.assertTrue(expected.equalsRaw(updated)); + } catch (Exception e) { + Assert.fail("CubeInstance is not updated correctly"); + } + } + + private <T extends RootPersistentEntity> T reinit(T obj, Serializer<T> serializer) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + serializer.serialize(obj, dos); + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + return serializer.deserialize(dis); + } +} diff --git a/server-base/src/test/resources/update/TableSchemaUpdateMapping.json b/server-base/src/test/resources/update/TableSchemaUpdateMapping.json new file mode 100644 index 0000000..bd601c8 --- /dev/null +++ b/server-base/src/test/resources/update/TableSchemaUpdateMapping.json @@ -0,0 +1,19 @@ +{ + "DEFAULT.TEST_KYLIN_FACT": { + "database": "TEST", + "tableName": "KYLIN_FACT" + }, + "DEFAULT.TEST_ACCOUNT": { + "database": "TEST" + }, + "DEFAULT.TEST_COUNTRY": { + "database": "TEST", + "tableName": "COUNTRY" + }, + "EDW.TEST_CAL_DT": { + "tableName": "CAL_DT" + }, + "DEFAULT.TEST_CATEGORY_GROUPINGS": { + "database": "TEST" + } +} \ No newline at end of file diff --git a/server-base/src/test/resources/update/cube/test_kylin_cube_with_slr_left_join_ready.json b/server-base/src/test/resources/update/cube/test_kylin_cube_with_slr_left_join_ready.json new file mode 100644 index 0000000..8cd7548 --- /dev/null +++ b/server-base/src/test/resources/update/cube/test_kylin_cube_with_slr_left_join_ready.json @@ -0,0 +1,44 @@ +{ + "uuid" : "kkkka32a-a33e-4b69-83dd-0bb8b1f8c53b", + + "last_modified" : 1404097095621, + "name" : "test_kylin_cube_with_slr_left_join_ready", + "owner" : null, + "descriptor" : "test_kylin_cube_with_slr_left_join_desc", + "cost" : 50, + "segments" : [ { + "uuid" : "1eaca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "1eaca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_LEFT_JOIN_READY-BCF2F125-9B0B-40DD-9509-95EC59B31333", + "date_range_start" : 0, + "date_range_end" : 1384243200000, + "status" : "READY", + "size_kb" : 7690, + "source_records" : 10000, + "source_records_size" : 610288, + "last_build_time" : 1404097095455, + "last_build_job_id" : "bcf2f125-9b0b-40dd-9509-95ec59b31333", + "binary_signature" : null, + "dictionaries" : { + "TEST.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict", + "TEST.TEST_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict", + "TEST.TEST_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict", + "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict", + "TEST.TEST_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict", + "TEST.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict", + "EDW.CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict", + "TEST.TEST_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict", + "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict", + "TEST.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict" + }, + "snapshots" : { + "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot", + "EDW.CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot", + "TEST.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot", + "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot" + } + } ], + "status" : "READY", + "create_time" : null, + "notify_list" : null +} \ No newline at end of file diff --git a/server-base/src/test/resources/update/cube_desc/ci_left_join_cube.json b/server-base/src/test/resources/update/cube_desc/ci_left_join_cube.json new file mode 100644 index 0000000..f7f5959 --- /dev/null +++ b/server-base/src/test/resources/update/cube_desc/ci_left_join_cube.json @@ -0,0 +1,619 @@ +{ + "uuid": "629ab7a8-3929-4dff-b59d-2100aadccd1a", + "name": "ci_left_join_cube", + "model_name": "ci_left_join_model", + "description": null, + "dimensions": [ + { + "name": "CAL_DT", + "table": "TEST_CAL_DT", + "column": "{FK}", + "derived": [ + "WEEK_BEG_DT" + ] + }, + { + "name": "ORDER_ID", + "table": "TEST_KYLIN_FACT", + "column": "ORDER_ID" + }, + { + "name": "TEST_DATE_ENC", + "table": "TEST_ORDER", + "column": "TEST_DATE_ENC" + }, + { + "name": "TEST_TIME_ENC", + "table": "TEST_ORDER", + "column": "TEST_TIME_ENC" + }, + { + "name": "CATEGORY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "{FK}", + "derived": [ + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null + }, + { + "name": "LSTG_FORMAT_NAME", + "table": "TEST_KYLIN_FACT", + "column": "LSTG_FORMAT_NAME", + "derived": null + }, + { + "name": "SITE_ID", + "table": "TEST_SITES", + "column": "{FK}", + "derived": [ + "SITE_NAME", + "CRE_USER" + ] + }, + { + "name": "SELLER_TYPE_CD", + "table": "TEST_SELLER_TYPE_DIM", + "column": "{FK}", + "derived": [ + "SELLER_TYPE_DESC" + ] + }, + { + "name": "SELLER_ID", + "table": "TEST_KYLIN_FACT", + "column": "SELLER_ID" + }, + { + "name": "SELLER_BUYER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "SELLER_SELLER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "SELLER_COUNTRY", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "SELLER_COUNTRY_NAME", + "table": "SELLER_COUNTRY", + "column": "NAME" + }, + { + "name": "BUYER_ID", + "table": "TEST_ORDER", + "column": "BUYER_ID" + }, + { + "name": "BUYER_BUYER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "BUYER_SELLER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "BUYER_COUNTRY", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "BUYER_COUNTRY_NAME", + "table": "BUYER_COUNTRY", + "column": "NAME" + } + ], + "measures": [ + { + "name": "TRANS_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "constant", + "value": "1" + }, + "returntype": "bigint" + } + }, + { + "name": "ITEM_COUNT_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ITEM_COUNT" + }, + "returntype": "bigint" + } + }, + { + "name": "GMV_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "bigint" + } + }, + { + "name": "GMV_MIN", + "function": { + "expression": "MIN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_MAX", + "function": { + "expression": "MAX", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "SELLER_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + }, + "returntype": "hllc(10)" + } + }, + { + "name": "SELLER_FORMAT_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "hllc(10)" + } + }, + { + "name": "TOP_SELLER", + "function": { + "expression": "TOP_N", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "topn(100, 4)", + "configuration": { + "topn.encoding.TEST_KYLIN_FACT.SELLER_ID": "int:4" + } + } + }, + { + "name": "TEST_COUNT_DISTINCT_BITMAP", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" + }, + "returntype": "bitmap" + } + }, + { + "name": "TEST_EXTENDED_COLUMN", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "BUYER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_ORDER.BUYER_ID", + "next_parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "SELLER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID", + "next_parameter": { + "type": "column", + "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "TRANS_ID_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TRANS_ID" + }, + "returntype": "raw" + } + }, + { + "name": "PRICE_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "raw" + } + }, + { + "name": "CAL_DT_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.CAL_DT" + }, + "returntype": "raw" + } + }, + { + "name": "GVM_PERCENTILE", + "function": { + "expression": "PERCENTILE_APPROX", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "percentile(100)" + } + }, + { + "name": "TEST_COUNT_COLUMN_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TEST_COUNT_COLUMN" + }, + "returntype": "bigint" + } + } + ], + "dictionaries": [ + { + "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", + "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" + } + ], + "rowkey": { + "rowkey_columns": [ + { + "column": "TEST_KYLIN_FACT.SELLER_ID", + "encoding": "int:4" + }, + { + "column": "TEST_KYLIN_FACT.ORDER_ID", + "encoding": "int:4" + }, + { + "column": "TEST_KYLIN_FACT.CAL_DT", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "encoding": "dict" + }, + { + "column": "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "encoding": "dict" + }, + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "encoding": "dict" + }, + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "encoding": "fixed_length:12" + }, + { + "column": "TEST_KYLIN_FACT.LSTG_SITE_ID", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "encoding": "dict" + }, + { + "column": "TEST_ORDER.TEST_TIME_ENC", + "encoding": "time" + }, + { + "column": "TEST_ORDER.TEST_DATE_ENC", + "encoding": "date" + }, + { + "column": "TEST_ORDER.BUYER_ID", + "encoding": "int:4" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "BUYER_COUNTRY.NAME", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "SELLER_COUNTRY.NAME", + "encoding": "dict" + } + ] + }, + "signature": null, + "last_modified": 1448959801311, + "null_string": null, + "hbase_mapping": { + "column_family": [ + { + "name": "F1", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "TRANS_CNT", + "ITEM_COUNT_SUM", + "GMV_SUM", + "GMV_CNT", + "GMV_MIN", + "GMV_MAX", + "TEST_COUNT_COLUMN_CNT" + ] + } + ] + }, + { + "name": "F2", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "SELLER_HLL", + "SELLER_FORMAT_HLL", + "TOP_SELLER", + "TEST_COUNT_DISTINCT_BITMAP" + ] + } + ] + }, + { + "name": "F3", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "TEST_EXTENDED_COLUMN", + "BUYER_CONTACT", + "SELLER_CONTACT", + "TRANS_ID_RAW", + "PRICE_RAW", + "CAL_DT_RAW", + "GVM_PERCENTILE" + ] + } + ] + } + ] + }, + "aggregation_groups": [ + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" + ], + "select_rule": { + "hierarchy_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ] + ], + "mandatory_dims": [], + "joint_dims": [ + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" + ] + ], + "dim_cap": 5 + } + }, + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME", + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ], + "select_rule": { + "hierarchy_dims": [], + "mandatory_dims": [ + "TEST_KYLIN_FACT.CAL_DT" + ], + "joint_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ], + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" + ], + [ + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" + ], + [ + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ] + ], + "dim_cap": 3 + } + } + ], + "notify_list": null, + "status_need_notify": [], + "auto_merge_time_ranges": null, + "retention_range": 0, + "engine_type": 2, + "storage_type": 2, + "override_kylin_properties": { + "kylin.cube.algorithm": "INMEM", + "kylin.dictionary.shrunken-from-global-enabled": "true" + }, + "snapshot_table_desc_list": [ + { + "table_name": "TEST.TEST_CATEGORY_GROUPINGS", + "storage_type": "hbase", + "local_cache_enable": true, + "global": true + } + ], + "partition_date_start": 0 +} diff --git a/server-base/src/test/resources/update/model_desc/ci_inner_join_model.json b/server-base/src/test/resources/update/model_desc/ci_inner_join_model.json new file mode 100644 index 0000000..8a74b36 --- /dev/null +++ b/server-base/src/test/resources/update/model_desc/ci_inner_join_model.json @@ -0,0 +1,238 @@ +{ + "uuid": "72ab4ee2-2cdb-4b07-b39e-4c298563ae27", + "name": "ci_inner_join_model", + "fact_table": "TEST.KYLIN_FACT", + "fact_table_alias": "TEST_KYLIN_FACT", + "lookups": [ + { + "table": "DEFAULT.TEST_ORDER", + "alias": "TEST_ORDER", + "kind": "FACT", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_ORDER.ORDER_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.ORDER_ID" + ] + } + }, + { + "table": "TEST.TEST_ACCOUNT", + "alias": "BUYER_ACCOUNT", + "kind": "FACT", + "join": { + "type": "INNER", + "primary_key": [ + "BUYER_ACCOUNT.ACCOUNT_ID" + ], + "foreign_key": [ + "TEST_ORDER.BUYER_ID" + ] + } + }, + { + "table": "EDW.CAL_DT", + "alias": "TEST_CAL_DT", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_CAL_DT.CAL_DT" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.CAL_DT" + ] + } + }, + { + "table": "TEST.TEST_CATEGORY_GROUPINGS", + "alias": "TEST_CATEGORY_GROUPINGS", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID", + "TEST_CATEGORY_GROUPINGS.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "alias": "TEST_SITES", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_SITES.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "alias": "TEST_SELLER_TYPE_DIM", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" + ] + } + }, + { + "table": "TEST.COUNTRY", + "alias": "BUYER_COUNTRY", + "join": { + "type": "INNER", + "primary_key": [ + "BUYER_COUNTRY.COUNTRY" + ], + "foreign_key": [ + "BUYER_ACCOUNT.ACCOUNT_COUNTRY" + ] + } + }, + { + "table": "TEST.COUNTRY", + "alias": "SELLER_COUNTRY", + "join": { + "type": "INNER", + "primary_key": [ + "SELLER_COUNTRY.COUNTRY" + ], + "foreign_key": [ + "SELLER_ACCOUNT.ACCOUNT_COUNTRY" + ] + } + }, + { + "table": "TEST.TEST_ACCOUNT", + "alias": "SELLER_ACCOUNT", + "kind": "FACT", + "join": { + "type": "INNER", + "primary_key": [ + "SELLER_ACCOUNT.ACCOUNT_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.SELLER_ID" + ] + } + } + ], + "dimensions": [ + { + "table": "TEST_KYLIN_FACT", + "columns": [ + "TRANS_ID", + "ORDER_ID", + "CAL_DT", + "LSTG_FORMAT_NAME", + "LSTG_SITE_ID", + "LEAF_CATEG_ID", + "SLR_SEGMENT_CD", + "SELLER_ID", + "TEST_COUNT_DISTINCT_BITMAP" + ] + }, + { + "table": "TEST_ORDER", + "columns": [ + "ORDER_ID", + "BUYER_ID", + "TEST_DATE_ENC", + "TEST_TIME_ENC", + "TEST_EXTENDED_COLUMN" + ] + }, + { + "table": "BUYER_ACCOUNT", + "columns": [ + "ACCOUNT_ID", + "ACCOUNT_BUYER_LEVEL", + "ACCOUNT_SELLER_LEVEL", + "ACCOUNT_COUNTRY", + "ACCOUNT_CONTACT" + ] + }, + { + "table": "SELLER_ACCOUNT", + "columns": [ + "ACCOUNT_ID", + "ACCOUNT_BUYER_LEVEL", + "ACCOUNT_SELLER_LEVEL", + "ACCOUNT_COUNTRY", + "ACCOUNT_CONTACT" + ] + }, + { + "table": "TEST_CATEGORY_GROUPINGS", + "columns": [ + "LEAF_CATEG_ID", + "SITE_ID", + "META_CATEG_NAME", + "CATEG_LVL2_NAME", + "CATEG_LVL3_NAME", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "table": "TEST_SITES", + "columns": [ + "SITE_ID", + "SITE_NAME", + "CRE_USER" + ] + }, + { + "table": "TEST_SELLER_TYPE_DIM", + "columns": [ + "SELLER_TYPE_CD", + "SELLER_TYPE_DESC" + ] + }, + { + "table": "TEST_CAL_DT", + "columns": [ + "CAL_DT", + "WEEK_BEG_DT" + ] + }, + { + "table": "BUYER_COUNTRY", + "columns": [ + "COUNTRY", + "NAME" + ] + }, + { + "table": "SELLER_COUNTRY", + "columns": [ + "COUNTRY", + "NAME" + ] + } + ], + "metrics": [ + "TEST_KYLIN_FACT.TEST_COUNT_COLUMN", + "TEST_KYLIN_FACT.PRICE", + "TEST_KYLIN_FACT.ITEM_COUNT" + ], + "last_modified": 1422435345352, + "filter_condition": null, + "partition_desc": { + "partition_date_column": "TEST.KYLIN_FACT.CAL_DT", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} diff --git a/server-base/src/test/resources/update/table/EDW.CAL_DT.json b/server-base/src/test/resources/update/table/EDW.CAL_DT.json new file mode 100644 index 0000000..8cbcec8 --- /dev/null +++ b/server-base/src/test/resources/update/table/EDW.CAL_DT.json @@ -0,0 +1,413 @@ +{ + + "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a", + "name" : "CAL_DT", + "columns" : [ { + "id" : "1", + "name" : "CAL_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "2", + "name" : "YEAR_BEG_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "3", + "name" : "QTR_BEG_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "4", + "name" : "MONTH_BEG_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "5", + "name" : "WEEK_BEG_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "6", + "name" : "AGE_FOR_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "7", + "name" : "AGE_FOR_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "8", + "name" : "AGE_FOR_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "9", + "name" : "AGE_FOR_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "10", + "name" : "AGE_FOR_DT_ID", + "datatype" : "smallint" + }, { + "id" : "11", + "name" : "AGE_FOR_RTL_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "12", + "name" : "AGE_FOR_RTL_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "13", + "name" : "AGE_FOR_RTL_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "14", + "name" : "AGE_FOR_RTL_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "15", + "name" : "AGE_FOR_CS_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "16", + "name" : "DAY_OF_CAL_ID", + "datatype" : "integer" + }, { + "id" : "17", + "name" : "DAY_OF_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "18", + "name" : "DAY_OF_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "19", + "name" : "DAY_OF_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "20", + "name" : "DAY_OF_WEEK_ID", + "datatype" : "integer" + }, { + "id" : "21", + "name" : "WEEK_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "22", + "name" : "WEEK_OF_CAL_ID", + "datatype" : "integer" + }, { + "id" : "23", + "name" : "MONTH_OF_QTR_ID", + "datatype" : "tinyint" + }, { + "id" : "24", + "name" : "MONTH_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "25", + "name" : "MONTH_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "26", + "name" : "QTR_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "27", + "name" : "QTR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "28", + "name" : "YEAR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "29", + "name" : "YEAR_END_DT", + "datatype" : "varchar(256)" + }, { + "id" : "30", + "name" : "QTR_END_DT", + "datatype" : "varchar(256)" + }, { + "id" : "31", + "name" : "MONTH_END_DT", + "datatype" : "varchar(256)" + }, { + "id" : "32", + "name" : "WEEK_END_DT", + "datatype" : "varchar(256)" + }, { + "id" : "33", + "name" : "CAL_DT_NAME", + "datatype" : "varchar(256)" + }, { + "id" : "34", + "name" : "CAL_DT_DESC", + "datatype" : "varchar(256)" + }, { + "id" : "35", + "name" : "CAL_DT_SHORT_NAME", + "datatype" : "varchar(256)" + }, { + "id" : "36", + "name" : "YTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "37", + "name" : "QTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "38", + "name" : "MTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "39", + "name" : "WTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "40", + "name" : "SEASON_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "41", + "name" : "DAY_IN_YEAR_COUNT", + "datatype" : "smallint" + }, { + "id" : "42", + "name" : "DAY_IN_QTR_COUNT", + "datatype" : "tinyint" + }, { + "id" : "43", + "name" : "DAY_IN_MONTH_COUNT", + "datatype" : "tinyint" + }, { + "id" : "44", + "name" : "DAY_IN_WEEK_COUNT", + "datatype" : "tinyint" + }, { + "id" : "45", + "name" : "RTL_YEAR_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "46", + "name" : "RTL_QTR_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "47", + "name" : "RTL_MONTH_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "48", + "name" : "RTL_WEEK_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "49", + "name" : "CS_WEEK_BEG_DT", + "datatype" : "varchar(256)" + }, { + "id" : "50", + "name" : "CAL_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "51", + "name" : "DAY_OF_WEEK", + "datatype" : "varchar(256)" + }, { + "id" : "52", + "name" : "MONTH_ID", + "datatype" : "varchar(256)" + }, { + "id" : "53", + "name" : "PRD_DESC", + "datatype" : "varchar(256)" + }, { + "id" : "54", + "name" : "PRD_FLAG", + "datatype" : "varchar(256)" + }, { + "id" : "55", + "name" : "PRD_ID", + "datatype" : "varchar(256)" + }, { + "id" : "56", + "name" : "PRD_IND", + "datatype" : "varchar(256)" + }, { + "id" : "57", + "name" : "QTR_DESC", + "datatype" : "varchar(256)" + }, { + "id" : "58", + "name" : "QTR_ID", + "datatype" : "varchar(256)" + }, { + "id" : "59", + "name" : "QTR_IND", + "datatype" : "varchar(256)" + }, { + "id" : "60", + "name" : "RETAIL_WEEK", + "datatype" : "varchar(256)" + }, { + "id" : "61", + "name" : "RETAIL_YEAR", + "datatype" : "varchar(256)" + }, { + "id" : "62", + "name" : "RETAIL_START_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "63", + "name" : "RETAIL_WK_END_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "64", + "name" : "WEEK_IND", + "datatype" : "varchar(256)" + }, { + "id" : "65", + "name" : "WEEK_NUM_DESC", + "datatype" : "varchar(256)" + }, { + "id" : "66", + "name" : "WEEK_BEG_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "67", + "name" : "WEEK_END_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "68", + "name" : "WEEK_IN_YEAR_ID", + "datatype" : "varchar(256)" + }, { + "id" : "69", + "name" : "WEEK_ID", + "datatype" : "varchar(256)" + }, { + "id" : "70", + "name" : "WEEK_BEG_END_DESC_MDY", + "datatype" : "varchar(256)" + }, { + "id" : "71", + "name" : "WEEK_BEG_END_DESC_MD", + "datatype" : "varchar(256)" + }, { + "id" : "72", + "name" : "YEAR_ID", + "datatype" : "varchar(256)" + }, { + "id" : "73", + "name" : "YEAR_IND", + "datatype" : "varchar(256)" + }, { + "id" : "74", + "name" : "CAL_DT_MNS_1YEAR_DT", + "datatype" : "varchar(256)" + }, { + "id" : "75", + "name" : "CAL_DT_MNS_2YEAR_DT", + "datatype" : "varchar(256)" + }, { + "id" : "76", + "name" : "CAL_DT_MNS_1QTR_DT", + "datatype" : "varchar(256)" + }, { + "id" : "77", + "name" : "CAL_DT_MNS_2QTR_DT", + "datatype" : "varchar(256)" + }, { + "id" : "78", + "name" : "CAL_DT_MNS_1MONTH_DT", + "datatype" : "varchar(256)" + }, { + "id" : "79", + "name" : "CAL_DT_MNS_2MONTH_DT", + "datatype" : "varchar(256)" + }, { + "id" : "80", + "name" : "CAL_DT_MNS_1WEEK_DT", + "datatype" : "varchar(256)" + }, { + "id" : "81", + "name" : "CAL_DT_MNS_2WEEK_DT", + "datatype" : "varchar(256)" + }, { + "id" : "82", + "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "83", + "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "84", + "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "85", + "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "86", + "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "87", + "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "88", + "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "89", + "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "90", + "name" : "RTL_MONTH_OF_RTL_YEAR_ID", + "datatype" : "varchar(256)" + }, { + "id" : "91", + "name" : "RTL_QTR_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "92", + "name" : "RTL_WEEK_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "93", + "name" : "SEASON_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "94", + "name" : "YTM_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "95", + "name" : "YTQ_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "96", + "name" : "YTW_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "97", + "name" : "CAL_DT_CRE_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "98", + "name" : "CAL_DT_CRE_USER", + "datatype" : "varchar(256)" + }, { + "id" : "99", + "name" : "CAL_DT_UPD_DATE", + "datatype" : "varchar(256)" + }, { + "id" : "100", + "name" : "CAL_DT_UPD_USER", + "datatype" : "varchar(256)" + } ], + "database" : "edw", + "last_modified" : 0 +} \ No newline at end of file diff --git a/server-base/src/test/resources/update/table/TEST.COUNTRY.json b/server-base/src/test/resources/update/table/TEST.COUNTRY.json new file mode 100644 index 0000000..b6c1ee0 --- /dev/null +++ b/server-base/src/test/resources/update/table/TEST.COUNTRY.json @@ -0,0 +1,26 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882", + + "name" : "COUNTRY", + "columns" : [ { + "id" : "1", + "name" : "COUNTRY", + "datatype" : "varchar(256)", + "index": "T" + }, { + "id" : "2", + "name" : "LATITUDE", + "datatype" : "double" + }, { + "id" : "3", + "name" : "LONGITUDE", + "datatype" : "double" + }, { + "id" : "4", + "name" : "NAME", + "datatype" : "varchar(256)", + "index": "T" + } ], + "database" : "TEST", + "last_modified" : 0 +} \ No newline at end of file diff --git a/server-base/src/test/resources/update/table/TEST.KYLIN_FACT.json b/server-base/src/test/resources/update/table/TEST.KYLIN_FACT.json new file mode 100644 index 0000000..879f969 --- /dev/null +++ b/server-base/src/test/resources/update/table/TEST.KYLIN_FACT.json @@ -0,0 +1,73 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771", + "name" : "KYLIN_FACT", + "data_gen" : "1", + "columns" : [ { + "id" : "1", + "name" : "TRANS_ID", + "datatype" : "bigint", + "data_gen" : "ID" + }, { + "id" : "2", + "name" : "ORDER_ID", + "datatype" : "bigint", + "index": "T" + }, { + "id" : "3", + "name" : "CAL_DT", + "datatype" : "date", + "data_gen" : "FK,order", + "index": "T" + }, { + "id" : "4", + "name" : "LSTG_FORMAT_NAME", + "datatype" : "varchar(256)", + "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others", + "index": "T" + }, { + "id" : "5", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint", + "data_gen" : "FK,null,nullstr=0", + "index": "T" + }, { + "id" : "6", + "name" : "LSTG_SITE_ID", + "datatype" : "integer", + "index": "T" + }, { + "id" : "7", + "name" : "SLR_SEGMENT_CD", + "datatype" : "smallint", + "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD", + "index": "T" + }, { + "id" : "8", + "name" : "SELLER_ID", + "datatype" : "integer", + "data_gen" : "RAND||10000000|10001000", + "index": "T" + }, { + "id" : "9", + "name" : "PRICE", + "datatype" : "decimal(19,4)", + "data_gen" : "RAND|.##|-100|1000" + }, { + "id" : "10", + "name" : "ITEM_COUNT", + "datatype" : "integer", + "data_gen" : "RAND" + }, { + "id" : "11", + "name" : "TEST_COUNT_DISTINCT_BITMAP", + "datatype" : "varchar(256)", + "data_gen" : "RAND" + }, { + "id" : "12", + "name" : "TEST_COUNT_COLUMN", + "datatype" : "varchar(256)", + "data_gen" : "RAND,null=true,nullpct=0.2" + } ], + "database" : "TEST", + "last_modified" : 0 +} diff --git a/server-base/src/test/resources/update/table/TEST.TEST_ACCOUNT.json b/server-base/src/test/resources/update/table/TEST.TEST_ACCOUNT.json new file mode 100644 index 0000000..8facbea --- /dev/null +++ b/server-base/src/test/resources/update/table/TEST.TEST_ACCOUNT.json @@ -0,0 +1,36 @@ +{ + "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231", + + "name" : "TEST_ACCOUNT", + "data_gen" : "2000", + + "columns" : [ { + "id" : "1", + "name" : "ACCOUNT_ID", + "datatype" : "bigint", + "data_gen" : "ID|10000000", + "index": "T" + }, { + "id" : "2", + "name" : "ACCOUNT_BUYER_LEVEL", + "datatype" : "integer", + "data_gen" : "RAND||0|5" + }, { + "id" : "3", + "name" : "ACCOUNT_SELLER_LEVEL", + "datatype" : "integer", + "data_gen" : "RAND||0|5" + }, { + "id" : "4", + "name" : "ACCOUNT_COUNTRY", + "datatype" : "varchar(256)", + "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US", + "index": "T" + }, { + "id" : "5", + "name" : "ACCOUNT_CONTACT", + "datatype" : "varchar(256)" + } ], + "database" : "TEST", + "last_modified" : 0 +} \ No newline at end of file diff --git a/server-base/src/test/resources/update/table/TEST.TEST_CATEGORY_GROUPINGS.json b/server-base/src/test/resources/update/table/TEST.TEST_CATEGORY_GROUPINGS.json new file mode 100644 index 0000000..bc7eebb --- /dev/null +++ b/server-base/src/test/resources/update/table/TEST.TEST_CATEGORY_GROUPINGS.json @@ -0,0 +1,155 @@ +{ + + "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f", + "name" : "TEST_CATEGORY_GROUPINGS", + "columns" : [ { + "id" : "1", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint", + "index": "T" + }, { + "id" : "2", + "name" : "LEAF_CATEG_NAME", + "datatype" : "string", + "index": "T" + }, { + "id" : "3", + "name" : "SITE_ID", + "datatype" : "int", + "index": "T" + }, { + "id" : "4", + "name" : "CATEG_BUSN_MGR", + "datatype" : "string" + }, { + "id" : "5", + "name" : "CATEG_BUSN_UNIT", + "datatype" : "string" + }, { + "id" : "6", + "name" : "REGN_CATEG", + "datatype" : "string" + }, { + "id" : "7", + "name" : "USER_DEFINED_FIELD1", + "datatype" : "string" + }, { + "id" : "8", + "name" : "USER_DEFINED_FIELD3", + "datatype" : "string" + }, { + "id" : "9", + "name" : "GROUPINGS_CRE_DATE", + "datatype" : "string" + }, { + "id" : "10", + "name" : "UPD_DATE", + "datatype" : "string" + }, { + "id" : "11", + "name" : "GROUPINGS_CRE_USER", + "datatype" : "string" + }, { + "id" : "12", + "name" : "UPD_USER", + "datatype" : "string" + }, { + "id" : "13", + "name" : "META_CATEG_ID", + "datatype" : "decimal" + }, { + "id" : "14", + "name" : "META_CATEG_NAME", + "datatype" : "string" + }, { + "id" : "15", + "name" : "CATEG_LVL2_ID", + "datatype" : "decimal" + }, { + "id" : "16", + "name" : "CATEG_LVL3_ID", + "datatype" : "decimal" + }, { + "id" : "17", + "name" : "CATEG_LVL4_ID", + "datatype" : "decimal" + }, { + "id" : "18", + "name" : "CATEG_LVL5_ID", + "datatype" : "decimal" + }, { + "id" : "19", + "name" : "CATEG_LVL6_ID", + "datatype" : "decimal" + }, { + "id" : "20", + "name" : "CATEG_LVL7_ID", + "datatype" : "decimal" + }, { + "id" : "21", + "name" : "CATEG_LVL2_NAME", + "datatype" : "string" + }, { + "id" : "22", + "name" : "CATEG_LVL3_NAME", + "datatype" : "string" + }, { + "id" : "23", + "name" : "CATEG_LVL4_NAME", + "datatype" : "string" + }, { + "id" : "24", + "name" : "CATEG_LVL5_NAME", + "datatype" : "string" + }, { + "id" : "25", + "name" : "CATEG_LVL6_NAME", + "datatype" : "string" + }, { + "id" : "26", + "name" : "CATEG_LVL7_NAME", + "datatype" : "string" + }, { + "id" : "27", + "name" : "CATEG_FLAGS", + "datatype" : "decimal" + }, { + "id" : "28", + "name" : "ADULT_CATEG_YN", + "datatype" : "string" + }, { + "id" : "29", + "name" : "DOMAIN_ID", + "datatype" : "decimal" + }, { + "id" : "30", + "name" : "USER_DEFINED_FIELD5", + "datatype" : "string" + }, { + "id" : "31", + "name" : "VCS_ID", + "datatype" : "decimal" + }, { + "id" : "32", + "name" : "GCS_ID", + "datatype" : "decimal" + }, { + "id" : "33", + "name" : "MOVE_TO", + "datatype" : "decimal" + }, { + "id" : "34", + "name" : "SAP_CATEGORY_ID", + "datatype" : "decimal" + }, { + "id" : "35", + "name" : "SRC_ID", + "datatype" : "tinyint" + }, { + "id" : "36", + "name" : "BSNS_VRTCL_NAME", + "datatype" : "string" + } ], + "database" : "TEST", + "last_modified" : 0 +} \ No newline at end of file