Repository: lens Updated Branches: refs/heads/master 1b475f2ea -> 71cf9ffd0
LENS-719 : Fix setting start time for a fact's storage/update period Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/71cf9ffd Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/71cf9ffd Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/71cf9ffd Branch: refs/heads/master Commit: 71cf9ffd0a63b289dd7777a09d541d0932d6feb9 Parents: 1b475f2 Author: Rajat Khandelwal <pro...@apache.org> Authored: Fri Jan 8 14:55:58 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Fri Jan 8 14:55:58 2016 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/CubeMetastoreClient.java | 31 ++++++++++++++++++++ .../lens/cube/parse/StorageTableResolver.java | 28 +----------------- .../cube/metadata/TestCubeMetastoreClient.java | 21 +++++++++++++ 3 files changed, 53 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/71cf9ffd/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index e7550ca..ae0fb90 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -19,6 +19,7 @@ package org.apache.lens.cube.metadata; +import static org.apache.lens.cube.metadata.DateUtil.resolveDate; import static org.apache.lens.cube.metadata.MetastoreUtil.*; import java.text.ParseException; @@ -2162,4 +2163,34 @@ public class CubeMetastoreClient { throw new HiveException(dimTableName + " is not a dimension table"); } } + public boolean isStorageTableCandidateForRange(String storageTableName, Date fromDate, Date toDate) throws + HiveException, LensException { + Date now = new Date(); + String startProperty = getTable(storageTableName).getProperty(getStoragetableStartTimesKey()); + if (StringUtils.isNotBlank(startProperty)) { + for (String timeStr : startProperty.split("\\s*,\\s*")) { + if (fromDate.before(resolveDate(timeStr, now))) { + log.info("from date {} is before validity start time: {}, hence discarding {}", + fromDate, timeStr, storageTableName); + return false; + } + } + } + String endProperty = getTable(storageTableName).getProperty(getStoragetableEndTimesKey()); + if (StringUtils.isNotBlank(endProperty)) { + for (String timeStr : endProperty.split("\\s*,\\s*")) { + if (toDate.after(resolveDate(timeStr, now))) { + log.info("to date {} is after validity end time: {}, hence discarding {}", + toDate, timeStr, storageTableName); + return false; + } + } + } + return true; + } + public boolean isStorageTableCandidateForRange(String storageTableName, String fromDate, String toDate) throws + HiveException, LensException { + Date now = new Date(); + return isStorageTableCandidateForRange(storageTableName, resolveDate(fromDate, now), resolveDate(toDate, now)); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/71cf9ffd/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 14def15..de5f95e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -565,7 +565,7 @@ class StorageTableResolver implements ContextRewriter { Iterator<String> it = storageTbls.iterator(); while (it.hasNext()) { String storageTableName = it.next(); - if (!isStorageTableCandidateForRange(storageTableName, fromDate, toDate)) { + if (!client.isStorageTableCandidateForRange(storageTableName, fromDate, toDate)) { skipStorageCauses.put(storageTableName, new SkipStorageCause(RANGE_NOT_ANSWERABLE)); it.remove(); } else if (!client.partColExists(storageTableName, partCol)) { @@ -685,32 +685,6 @@ class StorageTableResolver implements ContextRewriter { updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions); } - private boolean isStorageTableCandidateForRange(String storageTableName, Date fromDate, Date toDate) throws - HiveException, LensException { - Date now = new Date(); - String startProperty = client.getTable(storageTableName).getProperty(getStoragetableStartTimesKey()); - if (StringUtils.isNotBlank(startProperty)) { - for (String timeStr : startProperty.split("\\s*,\\s*")) { - if (toDate.before(DateUtil.resolveDate(timeStr, now))) { - log.info("from date {} is before validity start time: {}, hence discarding {}", - toDate, timeStr, storageTableName); - return false; - } - } - } - String endProperty = client.getTable(storageTableName).getProperty(getStoragetableEndTimesKey()); - if (StringUtils.isNotBlank(endProperty)) { - for (String timeStr : endProperty.split("\\s*,\\s*")) { - if (fromDate.after(DateUtil.resolveDate(timeStr, now))) { - log.info("to date {} is after validity end time: {}, hence discarding {}", - fromDate, timeStr, storageTableName); - return false; - } - } - } - return true; - } - private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, FactPartition part, Set<String> storageTableNames) throws LensException, HiveException, ParseException { for (String storageTableName : storageTableNames) { http://git-wip-us.apache.org/repos/asf/lens/blob/71cf9ffd/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 0fef13f..1638825 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -773,6 +773,10 @@ public class TestCubeMetastoreClient { StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, datePartSingleton, datePartKeySingleton); + + s1.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "2015, now-10 days"); + s1.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now - 1 day"); + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, Sets.newHashSet(HOURLY, DAILY)); Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); @@ -794,6 +798,9 @@ public class TestCubeMetastoreClient { String storageTableName = getFactOrDimtableStorageTableName(factName, entry); assertTrue(client.tableExists(storageTableName)); } + String storageTable = getFactOrDimtableStorageTableName(factName, c1); + assertRangeValidityForStorageTable(storageTable); + Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0, "non_existing_part_col", 0); // test error on adding invalid partition @@ -847,6 +854,20 @@ public class TestCubeMetastoreClient { assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } + private void assertRangeValidityForStorageTable(String storageTable) throws HiveException, LensException { + Object[][] testCases = new Object[][] { + {"now - 15 days", "now - 11 days", false}, + {"now - 15 days", "now - 1 hour", false}, + {"now - 9 days", "now - 1 hour", false}, + {"now - 3 hour", "now - 1 hour", false}, + {"now - 9 days", "now - 2 days", true}, + }; + for(Object[] testCase: testCases) { + assertEquals(client.isStorageTableCandidateForRange(storageTable, testCase[0].toString(), testCase[1].toString()), + testCase[2]); + } + } + @Test(priority = 2) public void testAlterCubeFact() throws Exception { String factName = "test_alter_fact";
