Repository: lens Updated Branches: refs/heads/master 6be4d48fd -> d81211723
LENS-932 : Skip partition registration for very old and future dates for a fact tables Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d8121172 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d8121172 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d8121172 Branch: refs/heads/master Commit: d812117238962c4d6c11103c221d808034f631e9 Parents: 6be4d48 Author: Sushil Mohanty <[email protected]> Authored: Mon Feb 22 16:18:47 2016 +0530 Committer: Sushil Mohanty <[email protected]> Committed: Mon Feb 22 16:18:47 2016 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/CubeMetastoreClient.java | 107 +++++++++++--- .../cube/metadata/TestCubeMetastoreClient.java | 145 ++++++++++++++++++- .../server/metastore/TestMetastoreService.java | 58 +++++++- 3 files changed, 283 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d8121172/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 8969d1f..02bbbbd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -863,18 +863,36 @@ public class CubeMetastoreClient { latestLookupCache.add(storageTableName); return partsAdded; } else { + List<Partition> partsAdded = new ArrayList<>(); // first update in memory, then add to hive table's partitions. delete is reverse. partitionTimelineCache.updateForAddition(factOrDimTable, storageName, updatePeriod, - getTimePartSpecs(storagePartitionDescs)); + getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName, factOrDimTable), + getStorageTableEndDate(storageTableName, factOrDimTable))); // Adding partition in fact table. - List<Partition> partsAdded = - getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs, null); + if (storagePartitionDescs.size() > 0) { + partsAdded = getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, + storagePartitionDescs, null); + } // update hive table alterTablePartitionCache(getStorageTableName(factOrDimTable, Storage.getPrefix(storageName))); return partsAdded; } } + private Date getStorageTableStartDate(String storageTable, String factTableName) + throws HiveException, LensException { + List<Date> startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); + startDates.add(getFactTable(factTableName).getStartTime()); + return Collections.max(startDates); + } + + private Date getStorageTableEndDate(String storageTable, String factTableName) + throws HiveException, LensException { + List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); + endDates.add(getFactTable(factTableName).getEndTime()); + return Collections.min(endDates); + } + private Map<String, TreeSet<Date>> getTimePartSpecs(List<StoragePartitionDesc> storagePartitionDescs) { Map<String, TreeSet<Date>> timeSpecs = Maps.newHashMap(); for (StoragePartitionDesc storagePartitionDesc : storagePartitionDescs) { @@ -888,6 +906,41 @@ public class CubeMetastoreClient { return timeSpecs; } + + private Map<String, TreeSet<Date>> getTimePartSpecs(List<StoragePartitionDesc> storagePartitionDescs, + Date storageStartDate, Date storageEndDate) throws LensException { + Date now = new Date(); + Map<String, HashSet<Date>> skippedParts = Maps.newHashMap(); + Map<String, TreeSet<Date>> timeSpecs = Maps.newHashMap(); + Iterator<StoragePartitionDesc> itr = storagePartitionDescs.iterator(); + while (itr.hasNext()) { + StoragePartitionDesc storageDesc = itr.next(); + for (Map.Entry<String, Date> entry : storageDesc.getTimePartSpec().entrySet()) { + if (!timeSpecs.containsKey(entry.getKey())) { + timeSpecs.put(entry.getKey(), Sets.<Date>newTreeSet()); + } + // check whether partition falls between storage table start_time and + // end_time or d+2, in such case partition is eligible for registration. + if ((entry.getValue().compareTo(storageStartDate) >= 0 && entry.getValue().compareTo(storageEndDate) < 0) + && entry.getValue().compareTo(DateUtil.resolveRelativeDate("now +2 days", now)) < 0) { + timeSpecs.get(entry.getKey()).add(entry.getValue()); + } else { + if (!skippedParts.containsKey(entry.getKey())) { + skippedParts.put(entry.getKey(), Sets.<Date>newHashSet(entry.getValue())); + } else { + skippedParts.get(entry.getKey()).add(entry.getValue()); + } + itr.remove(); + break; + } + } + } + log.info("List of partitions skipped : {}, because they fall before fact start time : {} " + + "and after fact end time : {}", + skippedParts, storageStartDate, storageEndDate); + return timeSpecs; + } + private Map<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> groupPartitionDescs( List<StoragePartitionDesc> partitionDescs) { Map<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> ret = Maps.newHashMap(); @@ -1162,7 +1215,8 @@ public class CubeMetastoreClient { } public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, - Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException { + Map<String, Date> partitionTimestamp, + Map<String, String> partSpec) throws HiveException { String storageTableName = getFactOrDimtableStorageTableName(factName, storageName); return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec); } @@ -1235,7 +1289,7 @@ public class CubeMetastoreClient { throws HiveException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), - partitionTimestamps); + partitionTimestamps); } boolean latestPartitionExists(String factOrDimTblName, String storageName, String latestPartCol) @@ -2172,27 +2226,36 @@ public class CubeMetastoreClient { throw new HiveException(dimTableName + " is not a dimension table"); } } + + private List<Date> getStorageTimes(String storageTableName, String timeKey) throws HiveException, LensException { + Date now = new Date(); + List<Date> storageTimes = new ArrayList<Date>(); + String property = getTable(storageTableName).getProperty(timeKey); + if (StringUtils.isNotBlank(property)) { + for (String timeStr : property.split("\\s*,\\s*")) { + storageTimes.add(resolveDate(timeStr, now)); + } + } + return storageTimes; + } + public boolean isStorageTableCandidateForRange(String storageTableName, Date fromDate, Date toDate) throws HiveException, LensException { - Date now = new Date(); - String startProperty = getTable(storageTableName).getProperty(getStoragetableStartTimesKey()); - if (StringUtils.isNotBlank(startProperty)) { - for (String timeStr : startProperty.split("\\s*,\\s*")) { - if (fromDate.before(resolveDate(timeStr, now))) { - log.info("from date {} is before validity start time: {}, hence discarding {}", - fromDate, timeStr, storageTableName); - return false; - } + List<Date> storageStartDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableStartTimesKey()); + for(Date startDate : storageStartDates) { + if (fromDate.before(startDate)) { + log.info("from date {} is before validity start time: {}, hence discarding {}", + fromDate, startDate, storageTableName); + return false; } } - String endProperty = getTable(storageTableName).getProperty(getStoragetableEndTimesKey()); - if (StringUtils.isNotBlank(endProperty)) { - for (String timeStr : endProperty.split("\\s*,\\s*")) { - if (toDate.after(resolveDate(timeStr, now))) { - log.info("to date {} is after validity end time: {}, hence discarding {}", - toDate, timeStr, storageTableName); - return false; - } + + List<Date> storageEndDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableEndTimesKey()); + for(Date endDate : storageEndDates) { + if (toDate.after(endDate)) { + log.info("to date {} is after validity end time: {}, hence discarding {}", + toDate, endDate, storageTableName); + return false; } } return true; http://git-wip-us.apache.org/repos/asf/lens/blob/d8121172/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 0c4871c..65387c6 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -26,6 +26,7 @@ import static org.apache.lens.server.api.util.LensUtil.getHashMap; import static org.testng.Assert.*; +import java.text.SimpleDateFormat; import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; @@ -928,7 +929,7 @@ public class TestCubeMetastoreClient { assertRangeValidityForStorageTable(storageTable); - Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0, "non_existing_part_col", 0); + Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), -50, "non_existing_part_col", 0); // test error on adding invalid partition // test partition StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); @@ -958,14 +959,13 @@ public class TestCubeMetastoreClient { partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(partSpec2, c1); - assertEquals(client.getAllParts(storageTableName).size(), 2); + assertEquals(client.getAllParts(storageTableName).size(), 1); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); - assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); + assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); - client.dropPartition(cubeFact.getName(), c1, timeParts2, null, HOURLY); assertEquals(client.getAllParts(storageTableName).size(), 1); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); @@ -1655,6 +1655,143 @@ public class TestCubeMetastoreClient { } @Test(priority = 2) + public void testSkipPartitionsOlderThanFactStartTime() throws Exception { + Date now = new Date(); + SimpleDateFormat dateFormat = new SimpleDateFormat(DateUtil.ABSDATE_FMT); + + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); + String factNameSkipPart = "testFactSkipPart"; + + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + + // add some dimensions of the cube + factColumns.add(new FieldSchema("dim1", "string", "dim1")); + factColumns.add(new FieldSchema("dim2", "string", "dim2")); + + List<FieldSchema> factPartColumns = Lists.newArrayList(new FieldSchema("region", "string", "region part")); + + Set<UpdatePeriod> updates = Sets.newHashSet(HOURLY); + ArrayList<FieldSchema> partCols = Lists.newArrayList(getDatePartition(), factPartColumns.get(0)); + Map<String, String> factProps = new HashMap<>(); + + factProps.put(MetastoreConstants.FACT_RELATIVE_START_TIME, "now -30 days"); + factProps.put(MetastoreConstants.FACT_RELATIVE_END_TIME, "now +10 days"); + + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, partCols, + datePartKeySingleton); + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, updates); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); + + CubeFactTable cubeFactWithParts = new CubeFactTable(CUBE_NAME, factNameSkipPart, factColumns, updatePeriods); + // create cube fact + client.createCubeFactTable(CUBE_NAME, factNameSkipPart, factColumns, + updatePeriods, 0L, factProps, storageTables); + + assertTrue(client.tableExists(factNameSkipPart)); + Table cubeTbl = client.getHiveTable(factNameSkipPart); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); + CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); + assertTrue(cubeFactWithParts.equals(cubeFact2)); + + // Assert for storage tables + for (String entry : storageTables.keySet()) { + String storageTableName = getFactOrDimtableStorageTableName(factNameSkipPart, entry); + assertTrue(client.tableExists(storageTableName)); + } + + Map<String, String> partSpec = getHashMap(factPartColumns.get(0).getName(), "APAC"); + Map<String, Date> timePartsNow = getHashMap(getDatePartitionKey(), NOW); + Map<String, Date> timePartsBeforeTwoMonths = getHashMap(getDatePartitionKey(), TWO_MONTHS_BACK); + + // test partition + List<StoragePartitionDesc> storageDescs = new ArrayList<>(); + StoragePartitionDesc sPartSpecNow = + new StoragePartitionDesc(cubeFactWithParts.getName(), timePartsNow, partSpec, HOURLY); + StoragePartitionDesc sPartSpecTwoMonthsBack = + new StoragePartitionDesc(cubeFactWithParts.getName(), timePartsBeforeTwoMonths, partSpec, HOURLY); + storageDescs.add(sPartSpecNow); + storageDescs.add(sPartSpecTwoMonthsBack); + + client.addPartitions(storageDescs, c1); + assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); + assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, + timePartsBeforeTwoMonths, partSpec)); + } + + @Test(priority = 2) + public void testSkipPartitionsOlderThanStorageStartTime() throws Exception { + Date now = new Date(); + SimpleDateFormat dateFormat = new SimpleDateFormat(DateUtil.ABSDATE_FMT); + + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); + String factNameSkipPart = "testStorageSkipPart"; + + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + + // add some dimensions of the cube + factColumns.add(new FieldSchema("dim1", "string", "dim1")); + factColumns.add(new FieldSchema("dim2", "string", "dim2")); + + List<FieldSchema> factPartColumns = Lists.newArrayList(new FieldSchema("region", "string", "region part")); + + Set<UpdatePeriod> updates = Sets.newHashSet(HOURLY); + ArrayList<FieldSchema> partCols = Lists.newArrayList(getDatePartition(), factPartColumns.get(0)); + + Map<String, String> storageProps = new HashMap<>(); + storageProps.put(getStoragetableStartTimesKey(), "now -30 days"); + storageProps.put(getStoragetableEndTimesKey(), "now +10 days"); + + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, partCols, + datePartKeySingleton); + s1.getTblProps().putAll(storageProps); + + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, updates); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); + + CubeFactTable cubeFactWithParts = new CubeFactTable(CUBE_NAME, factNameSkipPart, factColumns, updatePeriods); + // create cube fact + client.createCubeFactTable(CUBE_NAME, factNameSkipPart, factColumns, + updatePeriods, 0L, null, storageTables); + + assertTrue(client.tableExists(factNameSkipPart)); + Table cubeTbl = client.getHiveTable(factNameSkipPart); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); + CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); + assertTrue(cubeFactWithParts.equals(cubeFact2)); + + // Assert for storage tables + for (String entry : storageTables.keySet()) { + String storageTableName = getFactOrDimtableStorageTableName(factNameSkipPart, entry); + assertTrue(client.tableExists(storageTableName)); + } + + Map<String, String> partSpec = getHashMap(factPartColumns.get(0).getName(), "APAC"); + Map<String, Date> timePartsNow = getHashMap(getDatePartitionKey(), NOW); + Map<String, Date> timePartsBeforeTwoMonths = getHashMap(getDatePartitionKey(), TWO_MONTHS_BACK); + + // test partition + List<StoragePartitionDesc> storageDescs = new ArrayList<>(); + StoragePartitionDesc sPartSpecNow = + new StoragePartitionDesc(cubeFactWithParts.getName(), timePartsNow, partSpec, HOURLY); + StoragePartitionDesc sPartSpecTwoMonthsBack = + new StoragePartitionDesc(cubeFactWithParts.getName(), timePartsBeforeTwoMonths, partSpec, HOURLY); + storageDescs.add(sPartSpecNow); + storageDescs.add(sPartSpecTwoMonthsBack); + + client.getTimelines(factNameSkipPart, null, null, null); + client.addPartitions(storageDescs, c1); + assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); + assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, + timePartsBeforeTwoMonths, partSpec)); + } + + @Test(priority = 2) public void testCubeFactWithPartsAndTimedParts() throws Exception { List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); String factNameWithPart = "testFactPartAndTimedParts"; http://git-wip-us.apache.org/repos/asf/lens/blob/d8121172/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 4bc3f0a..a541e9b 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -924,6 +924,15 @@ public class TestMetastoreService extends LensJerseyTest { private XStorageTableDesc createStorageTableDesc(String name, final String[] timePartColNames) { XStorageTableDesc xs1 = cubeObjectFactory.createXStorageTableDesc(); + XProperties props = cubeObjectFactory.createXProperties(); + XProperty propStartTime = cubeObjectFactory.createXProperty(); + propStartTime.setName(MetastoreUtil.getStoragetableStartTimesKey()); + propStartTime.setValue("now -10 days"); + XProperty propEndTime = cubeObjectFactory.createXProperty(); + propEndTime.setName(MetastoreUtil.getStoragetableEndTimesKey()); + propEndTime.setValue("now +5 days"); + props.getProperty().add(propStartTime); + props.getProperty().add(propEndTime); xs1.setCollectionDelimiter(","); xs1.setEscapeChar("\\"); xs1.setFieldDelimiter("\t"); @@ -932,7 +941,7 @@ public class TestMetastoreService extends LensJerseyTest { xs1.setTableLocation(new Path(new File("target").getAbsolutePath(), name).toString()); xs1.setExternal(true); xs1.setPartCols(new XColumns()); - xs1.setTableParameters(new XProperties()); + xs1.setTableParameters(props); xs1.setSerdeParameters(new XProperties()); for (String timePartColName : timePartColNames) { @@ -2033,6 +2042,53 @@ public class TestMetastoreService extends LensJerseyTest { dropDatabase(dbName, mediaType); } } + @SuppressWarnings("deprecation") + @Test(dataProvider = "mediaTypeData") + public void testSkipFactStoragePartitions(MediaType mediaType) throws Exception { + + final String table = "testSkipFactStoragePartitions"; + final String DB = dbPFX + "testSkipFactStoragePartitions_DB" + mediaType.getSubtype(); + String prevDb = getCurrentDatabase(mediaType); + createDatabase(DB, mediaType); + setCurrentDatabase(DB, mediaType); + createStorage("S1", mediaType); + createStorage("S2", mediaType); + + try { + final Date partDate = new Date(); + final XCube cube = createTestCube("testCube"); + XFactTable f = createFactTable(table); + f.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY")); + f.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY")); + f.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "HOURLY")); + + APIResult result = target() + .path("metastore") + .path("facts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity( + new GenericEntity<JAXBElement<XFactTable>>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + APIResult.class); + assertSuccess(result); + + APIResult partAddResult; + // skip partitons if it starts before storage start date + // Add two partitions one before storage start time and other one after start time + // Add partition status will return partial + XPartitionList partList = new XPartitionList(); + partList.getPartition().add(createPartition(table, DateUtils.addHours(partDate, 1))); + partList.getPartition().add(createPartition(table, DateUtils.addHours(partDate, -300))); + partAddResult = target().path("metastore/facts/").path(table).path("storages/S2/partitions") + .queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(new GenericEntity<JAXBElement<XPartitionList>>( + cubeObjectFactory.createXPartitionList(partList)) { + }, mediaType), APIResult.class); + assertEquals(partAddResult.getStatus(), Status.PARTIAL); + } finally { + setCurrentDatabase(prevDb, mediaType); + dropDatabase(DB, mediaType); + } + } @SuppressWarnings("deprecation")
