Repository: lens Updated Branches: refs/heads/lens-1381 4af769ee3 -> 975fa2c2b
http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 2a9be16..cf937a8 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -24,6 +24,7 @@ import static org.apache.lens.cube.metadata.UpdatePeriod.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; import static org.apache.lens.cube.parse.CubeTestSetup.*; + import static org.testng.Assert.*; import java.text.DateFormat; @@ -51,6 +52,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; @@ -102,14 +104,13 @@ public class TestCubeRewriter extends TestQueryRewrite { System.out.println("expected " + expected); compareQueries(rewrittenQuery.toHQL(), expected); - //TODO union : Fact names are different. Check after MaXCoveringFactResolver. //test with msr2 on different fact -// rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " + timeRangeString, conf); -// expected = "select SUM((testCube.msr2)) as `sum(msr2)` from TestQueryRewrite.c0_testFact testcube" -// + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; -// System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); -// System.out.println("expected " + expected); -// compareQueries(rewrittenQuery.toHQL(), expected); + rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " + timeRangeString, conf); + expected = "select SUM((testCube.msr2)) as `sum(msr2)` from TestQueryRewrite.c2_testfact testcube" + + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; + System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); + System.out.println("expected " + expected); + compareQueries(rewrittenQuery.toHQL(), expected); //from date 6 days back timeRangeString = getTimeRangeString(DAILY, -6, 0, qFmt); @@ -143,7 +144,6 @@ public class TestCubeRewriter extends TestQueryRewrite { // assertNotNull(rewrittenQuery.getNonExistingParts()); } - //TODO union: Verify after MaxCoveringFactResolver changes. @Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); @@ -189,16 +189,14 @@ public class TestCubeRewriter extends TestQueryRewrite { assertEquals(th.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) th; PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage(); - //TODO union : check the error code. Its coming as "Columns [msr2] are not present in any table" - //TODO union : Need to check partition resolution flow in StorageTableResolver. -// int endIndex = MISSING_PARTITIONS.errorFormat.length() - 3; -// assertEquals( -// pruneCauses.getBrief().substring(0, endIndex), -// MISSING_PARTITIONS.errorFormat.substring(0, endIndex) -// ); -// assertEquals(pruneCauses.getDetails().get("testfact").size(), 1); -// assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(), -// MISSING_PARTITIONS); + int endIndex = MISSING_PARTITIONS.errorFormat.length() - 3; + assertEquals( + pruneCauses.getBrief().substring(0, endIndex), + MISSING_PARTITIONS.errorFormat.substring(0, endIndex) + ); + assertEquals(pruneCauses.getDetails().get("c1_testfact").size(), 1); + assertEquals(pruneCauses.getDetails().get("c1_testfact").iterator().next().getCause(), + MISSING_PARTITIONS); } @Test @@ -209,9 +207,10 @@ public class TestCubeRewriter extends TestQueryRewrite { getExpectedQuery(DERIVED_CUBE_NAME, "select sum(derivedCube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(DERIVED_CUBE_NAME, "C2_testfact")); compareQueries(rewrittenQuery.toHQL(), expected); - System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - //TODO union: Check this in a better way. -// assertNotNull(rewrittenQuery.getNonExistingParts()); + + System.out.println("Non existing parts:" + ((StorageCandidate) rewrittenQuery.getCandidates().iterator().next()) + .getNonExistingPartitions()); + assertNotNull(((StorageCandidate) rewrittenQuery.getCandidates().iterator().next()).getNonExistingPartitions()); LensException th = getLensExceptionInRewrite( "select SUM(msr4) from derivedCube where " + TWO_DAYS_RANGE, getConf()); @@ -414,8 +413,8 @@ public class TestCubeRewriter extends TestQueryRewrite { "select cubestate.name, cubestate.countryid, msr2 from" + " testCube" + " where cubestate.countryid = 5 and " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select cubestate.name as `name`, " + - "cubestate.countryid as `countryid`, sum(testcube.msr2) as `msr2`" + " FROM ", + getExpectedQuery(TEST_CUBE_NAME, "select cubestate.name as `name`, " + + "cubestate.countryid as `countryid`, sum(testcube.msr2) as `msr2`" + " FROM ", " JOIN " + getDbName() + "c3_statetable_partitioned cubestate ON" + " testCube.stateid = cubestate.id and cubestate.dt = 'latest'", "cubestate.countryid=5", @@ -448,54 +447,51 @@ public class TestCubeRewriter extends TestQueryRewrite { getConfWithStorages("C2")); compareQueries(hqlQuery, expected); - //TODO union : Wrong fact selected. Verify after MaxCoveringFactResolver changes. // q2 -// hqlQuery = -// rewrite("select statedim.name, SUM(msr2) from" + " testCube" + " join citydim on testCube.cityid = citydim.id" -// + " left outer join statedim on statedim.id = citydim.stateid" -// + " right outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); -// expected = -// getExpectedQuery(TEST_CUBE_NAME, -// "select statedim.name as `name`," + " sum(testcube.msr2) as `SUM(msr2)` FROM ", "INNER JOIN " + getDbName() -// + "c1_citytable citydim ON testCube.cityid = citydim.id and citydim.dt='latest' LEFT OUTER JOIN " -// + getDbName() -// + "c1_statetable statedim" + " ON statedim.id = citydim.stateid AND " -// + "(statedim.dt = 'latest') RIGHT OUTER JOIN " + getDbName() + "c1_ziptable" -// + " zipdim ON citydim.zipcode = zipdim.code and zipdim.dt='latest'", null, " group by" + " statedim.name ", -// null, -// getWhereForHourly2days(TEST_CUBE_NAME, "C1_testfact2")); -// compareQueries(hqlQuery, expected); - - //TODO union : Wrong fact selected. Verify after MaxCoveringFactResolver changes. + hqlQuery = + rewrite("select statedim.name, SUM(msr2) from" + " testCube" + " join citydim on testCube.cityid = citydim.id" + + " left outer join statedim on statedim.id = citydim.stateid" + + " right outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); + expected = + getExpectedQuery(TEST_CUBE_NAME, + "select statedim.name as `name`," + " sum(testcube.msr2) as `SUM(msr2)` FROM ", "INNER JOIN " + getDbName() + + "c1_citytable citydim ON testCube.cityid = citydim.id and citydim.dt='latest' LEFT OUTER JOIN " + + getDbName() + + "c1_statetable statedim" + " ON statedim.id = citydim.stateid AND " + + "(statedim.dt = 'latest') RIGHT OUTER JOIN " + getDbName() + "c1_ziptable" + + " zipdim ON citydim.zipcode = zipdim.code and zipdim.dt='latest'", null, " group by" + " statedim.name ", + null, + getWhereForHourly2days(TEST_CUBE_NAME, "C1_testfact2")); + compareQueries(hqlQuery, expected); + // q3 -// hqlQuery = -// rewrite("select st.name, SUM(msr2) from" + " testCube TC" + " join citydim CT on TC.cityid = CT.id" -// + " left outer join statedim ST on ST.id = CT.stateid" -// + " right outer join zipdim ZT on CT.zipcode = ZT.code" + " where " + TWO_DAYS_RANGE, getConf()); -// expected = -// getExpectedQuery("tc", "select st.name as `name`," + " sum(tc.msr2) as `sum(msr2)` FROM ", -// " INNER JOIN " + getDbName() -// + "c1_citytable ct ON" + " tc.cityid = ct.id and ct.dt='latest' LEFT OUTER JOIN " -// + getDbName() + "c1_statetable st" -// + " ON st.id = ct.stateid and (st.dt = 'latest') " + "RIGHT OUTER JOIN " + getDbName() + "c1_ziptable" -// + " zt ON ct.zipcode = zt.code and zt.dt='latest'", null, " group by" + " st.name ", null, -// getWhereForHourly2days("tc", "C1_testfact2")); -// compareQueries(hqlQuery, expected); - - //TODO union : Wrong fact selected. Verify after MaxCoveringFactResolver changes. + hqlQuery = + rewrite("select st.name, SUM(msr2) from" + " testCube TC" + " join citydim CT on TC.cityid = CT.id" + + " left outer join statedim ST on ST.id = CT.stateid" + + " right outer join zipdim ZT on CT.zipcode = ZT.code" + " where " + TWO_DAYS_RANGE, getConf()); + expected = + getExpectedQuery("tc", "select st.name as `name`," + " sum(tc.msr2) as `sum(msr2)` FROM ", + " INNER JOIN " + getDbName() + + "c1_citytable ct ON" + " tc.cityid = ct.id and ct.dt='latest' LEFT OUTER JOIN " + + getDbName() + "c1_statetable st" + + " ON st.id = ct.stateid and (st.dt = 'latest') " + "RIGHT OUTER JOIN " + getDbName() + "c1_ziptable" + + " zt ON ct.zipcode = zt.code and zt.dt='latest'", null, " group by" + " st.name ", null, + getWhereForHourly2days("tc", "C1_testfact2")); + compareQueries(hqlQuery, expected); + // q4 -// hqlQuery = -// rewrite("select citydim.name, SUM(msr2) from" + " testCube" -// + " left outer join citydim on testCube.cityid = citydim.id" -// + " left outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); -// expected = -// getExpectedQuery(TEST_CUBE_NAME, "select citydim.name as `name`," + " sum(testcube.msr2) as `sum(msr2)`FROM ", -// " LEFT OUTER JOIN " -// + getDbName() + "c1_citytable citydim ON" + " testCube.cityid = citydim.id and (citydim.dt = 'latest') " -// + " LEFT OUTER JOIN " + getDbName() + "c1_ziptable" + " zipdim ON citydim.zipcode = zipdim.code AND " -// + "(zipdim.dt = 'latest')", null, " group by" + " citydim.name ", null, -// getWhereForHourly2days(TEST_CUBE_NAME, "C1_testfact2")); -// compareQueries(hqlQuery, expected); + hqlQuery = + rewrite("select citydim.name, SUM(msr2) from" + " testCube" + + " left outer join citydim on testCube.cityid = citydim.id" + + " left outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); + expected = + getExpectedQuery(TEST_CUBE_NAME, "select citydim.name as `name`," + " sum(testcube.msr2) as `sum(msr2)`FROM ", + " LEFT OUTER JOIN " + + getDbName() + "c1_citytable citydim ON" + " testCube.cityid = citydim.id and (citydim.dt = 'latest') " + + " LEFT OUTER JOIN " + getDbName() + "c1_ziptable" + " zipdim ON citydim.zipcode = zipdim.code AND " + + "(zipdim.dt = 'latest')", null, " group by" + " citydim.name ", null, + getWhereForHourly2days(TEST_CUBE_NAME, "C1_testfact2")); + compareQueries(hqlQuery, expected); hqlQuery = rewrite("select SUM(msr2) from testCube" + " join countrydim on testCube.countryid = countrydim.id" + " where " @@ -634,8 +630,8 @@ public class TestCubeRewriter extends TestQueryRewrite { Configuration conf = getConf(); conf.set(DRIVER_SUPPORTED_STORAGES, "C2"); - String hqlQuery = - rewrite("select name, SUM(msr2) from" + " testCube join citydim on testCube.cityid = citydim.id where " + String hqlQuery = + rewrite("select name, SUM(msr2) from" + " testCube join citydim on testCube.cityid = citydim.id where " + TWO_DAYS_RANGE, conf); String expected = getExpectedQuery(TEST_CUBE_NAME, "select citydim.name as `name`, sum(testcube.msr2) as `sum(msr2)` FROM " @@ -955,7 +951,6 @@ public class TestCubeRewriter extends TestQueryRewrite { /* The test is to check no failure on partial data when the flag FAIL_QUERY_ON_PARTIAL_DATA is not set */ - // TODO union : check after MaxCoveringFactResolver @Test public void testQueryWithMeasureWithDataCompletenessTagWithNoFailureOnPartialData() throws ParseException, LensException { @@ -968,24 +963,22 @@ public class TestCubeRewriter extends TestQueryRewrite { compareQueries(hqlQuery, expected); } - // TODO union : check after MaxCoveringFactResolver @Test public void testQueryWithMeasureWithDataCompletenessPresentInMultipleFacts() throws ParseException, - LensException { + LensException { /*In this query a measure is used which is present in two facts with different %completeness. While resolving the facts, the fact with the higher dataCompletenessFactor gets picked up.*/ Configuration conf = getConf(); conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); String hqlQuery = rewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf); String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) as `sum(msr9)` FROM ", null, null, - getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); + getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); compareQueries(hqlQuery, expected); } - // TODO union : check after MaxCoveringFactResolver - @Test + @Test public void testCubeWhereQueryWithMeasureWithDataCompletenessAndFailIfPartialDataFlagSet() throws ParseException, - LensException { + LensException { /*In this query a measure is used for which dataCompletenessTag is set and the flag FAIL_QUERY_ON_PARTIAL_DATA is set. The partitions for the queried range are present but some of the them have incomplete data. So, the query throws NO_CANDIDATE_FACT_AVAILABLE Exception*/ @@ -993,15 +986,16 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); - LensException e = getLensExceptionInRewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf); + LensException e = getLensExceptionInRewrite("select SUM(msr9) from basecube where " + + TWO_DAYS_RANGE, conf); assertEquals(e.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage(); /*Since the Flag FAIL_QUERY_ON_PARTIAL_DATA is set, and thhe queried fact has incomplete data, hence, we expect the prune cause to be INCOMPLETE_PARTITION. The below check is to validate this.*/ assertEquals(pruneCauses.getBrief().substring(0, INCOMPLETE_PARTITION.errorFormat.length() - 3), - INCOMPLETE_PARTITION.errorFormat.substring(0, - INCOMPLETE_PARTITION.errorFormat.length() - 3), pruneCauses.getBrief()); + INCOMPLETE_PARTITION.errorFormat.substring(0, + INCOMPLETE_PARTITION.errorFormat.length() - 3), pruneCauses.getBrief()); } @Test @@ -1021,7 +1015,8 @@ public class TestCubeRewriter extends TestQueryRewrite { MISSING_PARTITIONS.errorFormat.length() - 3), pruneCauses.getBrief()); Set<String> expectedSet = - Sets.newTreeSet(Arrays.asList("summary1", "summary2", "testfact2_raw", "summary3", "testfact")); + Sets.newTreeSet(Arrays.asList("c1_testfact2_raw", "c1_summary3", "c1_summary2", + "c1_summary1", "c2_testfact", "c1_testfact")); boolean missingPartitionCause = false; for (String key : pruneCauses.getDetails().keySet()) { Set<String> actualKeySet = Sets.newTreeSet(Splitter.on(',').split(key)); @@ -1033,12 +1028,13 @@ public class TestCubeRewriter extends TestQueryRewrite { } assertTrue(missingPartitionCause, MISSING_PARTITIONS + " error does not occur for facttables set " + expectedSet + " Details :" + pruneCauses.getDetails()); - assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(), - NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); - assertEquals(pruneCauses.getDetails().get("testfact2").iterator().next().getCause(), + assertEquals(pruneCauses.getDetails().get("c1_testfact2").iterator().next().getCause(), MISSING_PARTITIONS); - assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(), - NO_CANDIDATE_STORAGES); + /* + assertEquals(pruneCauses.getDetails().get("c4_testfact,c3_testfact,c3_testfact2_raw,c4_testfact2," + + "c99_cheapfact,c5_testfact").iterator().next().getCause(), + UNSUPPORTED_STORAGE); + CandidateTablePruneCause cheapFactPruneCauses = pruneCauses.getDetails().get("cheapfact").iterator().next(); assertEquals(cheapFactPruneCauses.getDimStoragePruningCauses().get("c0"), CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); @@ -1046,6 +1042,7 @@ public class TestCubeRewriter extends TestQueryRewrite { CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE); assertEquals(pruneCauses.getDetails().get("summary4").iterator().next().getCause(), TIMEDIM_NOT_SUPPORTED); assertTrue(pruneCauses.getDetails().get("summary4").iterator().next().getUnsupportedTimeDims().contains("d_time")); + */ } @Test @@ -1063,18 +1060,19 @@ public class TestCubeRewriter extends TestQueryRewrite { @Test public void testNoCandidateDimAvailableExceptionCompare() throws Exception { - //Max cause COLUMN_NOT_FOUND, Ordinal 9 + //Max cause COLUMN_NOT_FOUND, Ordinal 2 PruneCauses<CubeDimensionTable> pr1 = new PruneCauses<CubeDimensionTable>(); pr1.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), - CandidateTablePruneCause.columnNotFound("test1", "test2", "test3")); + CandidateTablePruneCause.columnNotFound( + CandidateTablePruneCause.CandidateTablePruneCode.COLUMN_NOT_FOUND, "test1", "test2", "test3")); NoCandidateDimAvailableException ne1 = new NoCandidateDimAvailableException(pr1); - //Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 6 + //Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 14 PruneCauses<CubeDimensionTable> pr2 = new PruneCauses<CubeDimensionTable>(); pr2.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), CandidateTablePruneCause.expressionNotEvaluable("testexp1", "testexp2")); NoCandidateDimAvailableException ne2 = new NoCandidateDimAvailableException(pr2); - assertEquals(ne1.compareTo(ne2), 3); + assertEquals(ne1.compareTo(ne2), -12); } @Test @@ -1261,8 +1259,8 @@ public class TestCubeRewriter extends TestQueryRewrite { " testcube.cityid > 100 ", " group by testcube.cityid having" + " sum(testCube.msr2) < 1000", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")), getExpectedQuery(TEST_CUBE_NAME, "SELECT testCube.cityid as `cityid`, sum(testCube.msr2) as `msr2`" + " FROM ", - " testcube.cityid > 100 ", " group by testcube.cityid having" - + " sum(testCube.msr2) < 1000 order by testCube.cityid asc", + " testcube.cityid > 100 ", " group by testCube.cityid having" + + " sum(testCube.msr2) < 1000 order by cityid asc", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")), }; Configuration conf = getConf(); @@ -1319,8 +1317,8 @@ public class TestCubeRewriter extends TestQueryRewrite { getConf()); expected = getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube,dim2 as `dim2`, " - + "count(testcube.msr4) as `count(msr4)`, sum(testcube.msr2) as `sum(msr2)`, " + - "max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2", + + "count(testcube.msr4) as `count(msr4)`, sum(testcube.msr2) as `sum(msr2)`, " + + "max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2", getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "it", "C2_summary2"), null); compareQueries(hqlQuery, expected); @@ -1390,8 +1388,6 @@ public class TestCubeRewriter extends TestQueryRewrite { compareQueries(hqlQuery, expected); } - // TODO union : Uncomment below test after deleting CandidateFact - /* @Test public void testLookAhead() throws Exception { @@ -1400,15 +1396,15 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); CubeQueryContext ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); - //assertEquals(ctx.candidateFacts.size(), 1); - //CandidateFact candidateFact = ctx.candidateFacts.iterator().next(); - Set<FactPartition> partsQueried = new TreeSet<>(candidateFact.getPartsQueried()); + assertEquals(ctx.getCandidates().size(), 1); + Candidate candidate = ctx.getCandidates().iterator().next(); + Set<FactPartition> partsQueried = new TreeSet<>(((StorageCandidate)candidate).getParticipatingPartitions()); Date ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -2)); Date nextDay = DateUtils.addDays(ceilDay, 1); Date nextToNextDay = DateUtils.addDays(nextDay, 1); HashSet<String> storageTables = Sets.newHashSet(); - for (String storageTable : candidateFact.getStorageTables()) { - storageTables.add(storageTable.split("\\.")[1]); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) { + storageTables.add(sc.getName()); } TreeSet<FactPartition> expectedPartsQueried = Sets.newTreeSet(); for (TimePartition p : Iterables.concat( @@ -1429,11 +1425,11 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_PT_PARTS_PFX, 3); ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); - partsQueried = new TreeSet<>(ctx.candidateFacts.iterator().next().getPartsQueried()); + partsQueried = new TreeSet<>(((StorageCandidate)ctx.getCandidates().iterator().next()) + .getParticipatingPartitions()); // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 assertEquals(partsQueried, expectedPartsQueried); } - */ @Test public void testCubeQueryWithMultipleRanges() throws Exception { @@ -1495,7 +1491,6 @@ public class TestCubeRewriter extends TestQueryRewrite { compareQueries(hqlQuery, expected); } - //TODO union : Wrong fact selected. Verify after MaxCoveringFactResolver changes. @Test public void testJoinWithMultipleAliases() throws Exception { String cubeQl = @@ -1570,7 +1565,6 @@ public class TestCubeRewriter extends TestQueryRewrite { } } - //TODO union: Verify after MaxCoveringFactResolver changes. @Test public void testTimeDimensionAndPartCol() throws Exception { // Test if time dimension is replaced with partition column http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java index a3bb77c..fb803a2 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java @@ -191,38 +191,44 @@ public class TestDenormalizationResolver extends TestQueryRewrite { "select dim2big2, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE, tconf); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; PruneCauses.BriefAndDetailedError error = ne.getJsonMessage(); - Assert.assertEquals(error.getBrief(), CandidateTablePruneCode.NO_CANDIDATE_STORAGES.errorFormat); + Assert.assertEquals(error.getBrief(), CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat); HashMap<String, List<CandidateTablePruneCause>> details = error.getDetails(); for (Map.Entry<String, List<CandidateTablePruneCause>> entry : details.entrySet()) { - if (entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound("dim2big2")))) { + if (entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound( + CandidateTablePruneCode.COLUMN_NOT_FOUND, "dim2big2")))) { Set<String> expectedKeySet = - Sets.newTreeSet(Splitter.on(',').split("summary1,cheapfact,testfactmonthly,testfact2,testfact")); + Sets.newTreeSet(Splitter.on(',').split("c1_summary1, c1_testfact,c1_testfact2")); Assert.assertTrue(expectedKeySet.equals(Sets.newTreeSet(Splitter.on(',').split(entry.getKey())))); } if (entry.getValue().equals( Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE)))) { Set<String> expectedKeySet = - Sets.newTreeSet(Splitter.on(',').split("summary2,testfact2_raw,summary3")); + Sets.newTreeSet(Splitter.on(',').split("c2_summary2, c2_summary3, c1_testfact2_raw, " + + "c3_testfact2_raw, c1_summary3,c1_summary2")); Assert.assertTrue(expectedKeySet.equals(Sets.newTreeSet(Splitter.on(',').split(entry.getKey())))); } - if (entry.getKey().equals("testfact_continuous")) { + if (entry.getKey().equals("c0_testfact_continuous")) { Assert.assertTrue(entry.getValue().equals( - Arrays.asList(CandidateTablePruneCause.columnNotFound("msr2", "msr3"))) - || entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr2")))); + Arrays.asList(CandidateTablePruneCause.columnNotFound(CandidateTablePruneCode.COLUMN_NOT_FOUND, + "msr2", "msr3"))) + || entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound( + CandidateTablePruneCode.COLUMN_NOT_FOUND, "msr3", "msr2")))); } - if (entry.getKey().equals("summary4")) { + if (entry.getKey().equals("c2_summary2, c2_summary3, c2_summary4, c4_testfact, c2_summary1, c3_testfact, " + + "c3_testfact2_raw, c4_testfact2, c99_cheapfact, c5_testfact, c0_cheapfact, " + + "c2_testfact, c2_testfactmonth, c0_testfact")) { List<CandidateTablePruneCause> expectedPruneCauses = Arrays.asList(CandidateTablePruneCause.noCandidateStoragesForDimtable( - new HashMap<String, CandidateTablePruneCode>() { - { - put("C2", CandidateTablePruneCode.UNSUPPORTED_STORAGE); - } - })); + new HashMap<String, CandidateTablePruneCode>() { + { + put("C2", CandidateTablePruneCode.UNSUPPORTED_STORAGE); + } + })); Assert.assertTrue(entry.getValue().equals(expectedPruneCauses)); } } @@ -279,23 +285,21 @@ public class TestDenormalizationResolver extends TestQueryRewrite { "No dimension table has the queried columns " + "for citydim, columns: [name, statename, nocandidatecol]"); } - // TODO union : Fix testcase after deleting CandidateFact - /* @Test public void testCubeQueryWithTwoRefCols() throws Exception { Configuration tConf = new Configuration(conf); tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, ""); CubeQueryContext cubeql = rewriteCtx("select dim2, test_time_dim2 from testcube where " + TWO_DAYS_RANGE, tConf); - Set<String> candidateFacts = new HashSet<String>(); - for (CandidateFact cfact : cubeql.getCandidateFacts()) { - candidateFacts.add(cfact.getName().toLowerCase()); + Set<String> candidates = new HashSet<String>(); + for (Candidate cand : cubeql.getCandidates()) { + candidates.add(cand.toString()); } // testfact contains test_time_dim_day_id, but not dim2 - it should have been removed. - Assert.assertFalse(candidateFacts.contains("testfact")); + Assert.assertFalse(candidates.contains("testfact")); // summary2 contains dim2, but not test_time_dim2 - it should have been removed. - Assert.assertFalse(candidateFacts.contains("summary2")); + Assert.assertFalse(candidates.contains("summary2")); } -*/ + @Test public void testCubeQueryWithHourDimJoin() throws Exception { Configuration tConf = new Configuration(conf); @@ -336,8 +340,8 @@ public class TestDenormalizationResolver extends TestQueryRewrite { + "c1_citytable citydim on basecube.cityid = citydim.id and (citydim.dt = 'latest') " + " join " + getDbName() + "c1_ziptable cityzip on citydim.zipcode = cityzip.code and (cityzip.dt = 'latest')"; String expected = - getExpectedQuery("basecube", "SELECT (cityzip.code) as `code`, (basecube.dim22) as `dim22`, " + - "(basecube.msr11) as `msr11` FROM ", joinExpr, null, null, null, + getExpectedQuery("basecube", "SELECT (cityzip.code) as `code`, (basecube.dim22) as `dim22`, " + + "(basecube.msr11) as `msr11` FROM ", joinExpr, null, null, null, getWhereForHourly2days("basecube", "C1_testfact2_raw_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); } http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java index f93a548..dd18ffd 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java @@ -91,7 +91,8 @@ public class TestExpressionResolver extends TestQueryRewrite { String hqlQuery = rewrite("select msr2 from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ'", + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", null, + " and substr(testCube.dim1, 3) != 'XYZ'", getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -252,10 +253,11 @@ public class TestExpressionResolver extends TestQueryRewrite { + " group by booleancut having msr6 > 100.0 order by booleancut", conf); String expected = getExpectedQuery(cubeName, "SELECT (((testcube.dim1) != 'x') and ((testcube.dim2) != 10)) as `booleancut`, " - + "avg(((testcube.msr1) + (testcube.msr2))) as `avgmsr` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + + "avg(((testcube.msr1) + (testcube.msr2))) as `avgmsr` FROM ", null, + " and substr(testCube.dim1, 3) != 'XYZ' " + " group by testCube.dim1 != 'x' AND testCube.dim2 != 10" + " having (sum(testCube.msr2) + max(testCube.msr3))/ count(testcube.msr4) > 100.0" - + " order by testCube.dim1 != 'x' AND testCube.dim2 != 10 asc", getWhereForHourly2days("C1_testfact2_raw")); + + " order by booleancut asc", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test @@ -287,8 +289,8 @@ public class TestExpressionResolver extends TestQueryRewrite { @Test public void testMultipleExpressionsPickingSecondExpression() throws Exception { String hqlQuery = rewrite("select equalsums from testCube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery(cubeName, "select (max(testCube.msr3) + sum(testCube.msr2))/100 " + - "as `equalsums` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2")); + String expected = getExpectedQuery(cubeName, "select (max(testCube.msr3) + sum(testCube.msr2))/100 " + + "as `equalsums` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); } http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java index 0d7e8ef..f31156a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java @@ -186,8 +186,8 @@ public class TestJoinResolver extends TestQueryRewrite { String query = "select cubecity.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - String expected = getExpectedQuery("testcube", "SELECT (cubecity.name) as `name`, sum((testcube.msr2)) " + - "as `msr2` FROM ", " left outer join " + getDbName() + String expected = getExpectedQuery("testcube", "SELECT (cubecity.name) as `name`, sum((testcube.msr2)) " + + "as `msr2` FROM ", " left outer join " + getDbName() + "c1_citytable cubecity ON testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", null, " group by cubecity.name", null, getWhereForHourly2days("testcube", "c1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -356,9 +356,11 @@ public class TestJoinResolver extends TestQueryRewrite { // Single joinchain with two chains, accessed as refcolumn query = "select cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "SELECT (citystate.capital) as `citystatecapital`, sum((basecube.msr2)) as `sum(msr2)` FROM ", - " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" - + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", + expected = getExpectedQuery("basecube", "SELECT (citystate.capital) as `citystatecapital`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id " + + "and cityState.dt= 'latest'", null, "group by citystate.capital", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -375,7 +377,7 @@ public class TestJoinResolver extends TestQueryRewrite { + "sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", - null, "group by citystate.capital order by citystate.capital asc", + null, "group by citystate.capital order by citystatecapital", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -454,8 +456,8 @@ public class TestJoinResolver extends TestQueryRewrite { query = "select cubeStateCountry.name, cubeCityStateCountry.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "SELECT (cubestatecountry.name) as `name`, (cubecitystatecountry.name) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", - "" + "SELECT (cubestatecountry.name) as `name`, (cubecitystatecountry.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", "" + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and (citydim.dt = 'latest')" + " join " + getDbName() + "c1_statetable statedim_0 on citydim.stateid=statedim_0.id and statedim_0.dt='latest'" @@ -617,8 +619,8 @@ public class TestJoinResolver extends TestQueryRewrite { query = "select cubecity.name, dim4chain.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select cubecity.name as `name`, dim4chain.name as `name`, " + - "dim3chain.id as `testdim3id`, avg(testcube.msr2) as `avg(msr2)`" + expected = getExpectedQuery("testcube", "select cubecity.name as `name`, dim4chain.name as `name`, " + + "dim3chain.id as `testdim3id`, avg(testcube.msr2) as `avg(msr2)`" + "FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java index 27a18f4..f9a5421 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java @@ -47,24 +47,26 @@ public class TestQueryMetrics extends TestQueryRewrite { Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AggregateResolver-ITER-6", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AliasReplacer-ITER-1", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-11", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse" + + ".CandidateCoveringSetsResolver-ITER-13", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-12", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-5", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ColumnResolver-ITER-0", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-16", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-17", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-3", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-17", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-18", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-2", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.FieldValidator-ITER-8", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.GroupbyResolver-ITER-7", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.JoinResolver-ITER-9", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-19", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestDimensionResolver-ITER-20", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-18", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-14", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-12", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-13", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-15", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimeRangeChecker-ITER-10", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.JoinResolver-ITER-10", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-20", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestDimensionResolver-ITER-21", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-19", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-15", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-14", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-16", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-9", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimeRangeChecker-ITER-11", "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimerangeResolver-ITER-4") ), reg.getGauges().keySet().toString()); } http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java index a14296c..76ea77d 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java @@ -63,7 +63,6 @@ public class TestRewriterPlan extends TestQueryRewrite { Assert.assertTrue(plan.getPartitions().get("c2_testfact").size() > 1); } - //TODO union : Wrong fact name picked. Check after MaxCoveringSetResolver changes. @Test public void testPlanExtractionForComplexQuery() throws Exception { // complex query @@ -86,7 +85,6 @@ public class TestRewriterPlan extends TestQueryRewrite { Assert.assertEquals(plan.getPartitions().get("citytable").size(), 1); } - //TODO union : Wrong fact name picked. Check after MaxCoveringSetResolver changes. @Test public void testPlanExtractionForMultipleQueries() throws Exception { // simple query http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java index 280a8c4..1e5facd 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java @@ -21,15 +21,14 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateFactory.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.COLUMN_NOT_FOUND; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE; +import static org.apache.lens.cube.parse.CandidateTablePruneCause. + CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.util.Calendar; -import java.util.GregorianCalendar; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.TimeRange; @@ -74,12 +73,14 @@ public class TestTimeRangeResolver extends TestQueryRewrite { getConf()); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; PruneCauses.BriefAndDetailedError causes = ne.getJsonMessage(); - assertTrue(causes.getBrief().contains("Columns [msr2] are not present in any table")); - assertEquals(causes.getDetails().size(), 2); + assertTrue(causes.getBrief().contains("No storages available for all of these time ranges: " + + "[dt [2016-01-01-00:00:00,000 to 2017-01-01-00:00:00,000)]")); + assertEquals(causes.getDetails().size(), 3); Set<CandidateTablePruneCause.CandidateTablePruneCode> expectedPruneCodes = Sets.newTreeSet(); - expectedPruneCodes.add(FACT_NOT_AVAILABLE_IN_RANGE); expectedPruneCodes.add(COLUMN_NOT_FOUND); + expectedPruneCodes.add(UNSUPPORTED_STORAGE); + expectedPruneCodes.add(STORAGE_NOT_AVAILABLE_IN_RANGE); Set<CandidateTablePruneCause.CandidateTablePruneCode> actualPruneCodes = Sets.newTreeSet(); for (List<CandidateTablePruneCause> cause : causes.getDetails().values()) { assertEquals(cause.size(), 1); @@ -93,13 +94,27 @@ public class TestTimeRangeResolver extends TestQueryRewrite { CubeQueryContext ctx = rewriteCtx("select msr12 from basecube where " + TWO_DAYS_RANGE + " or " + TWO_DAYS_RANGE_BEFORE_4_DAYS, getConf()); - assertEquals(ctx.getFactPruningMsgs().get(ctx.getMetastoreClient().getCubeFact("testfact_deprecated")).size(), 1); - CandidateTablePruneCause pruningMsg = - ctx.getFactPruningMsgs().get(ctx.getMetastoreClient().getCubeFact("testfact_deprecated")).get(0); + List<CandidateTablePruneCause> causes = findPruningMessagesForStorage("c3_testfact_deprecated", + ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE); + + causes = findPruningMessagesForStorage("c4_testfact_deprecated", ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE); + // testfact_deprecated's validity should be in between of both ranges. So both ranges should be in the invalid list // That would prove that parsing of properties has gone through successfully - assertEquals(pruningMsg.getCause(), FACT_NOT_AVAILABLE_IN_RANGE); - assertTrue(pruningMsg.getInvalidRanges().containsAll(ctx.getTimeRanges())); + + causes = findPruningMessagesForStorage("c1_testfact_deprecated", ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), STORAGE_NOT_AVAILABLE_IN_RANGE); + assertTrue(causes.get(0).getInvalidRanges().containsAll(ctx.getTimeRanges())); + + causes = findPruningMessagesForStorage("c2_testfact_deprecated", ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), STORAGE_NOT_AVAILABLE_IN_RANGE); + assertTrue(causes.get(0).getInvalidRanges().containsAll(ctx.getTimeRanges())); } @Test @@ -115,4 +130,21 @@ public class TestTimeRangeResolver extends TestQueryRewrite { assertEquals(timeRange.getFromDate(), from.getTime()); assertEquals(timeRange.getToDate(), dt.toDate()); } + + /** + * + * @param stoargeName storageName_factName + * @param allStoragePruningMsgs + * @return + */ + private static List<CandidateTablePruneCause> findPruningMessagesForStorage(String stoargeName, + PruneCauses<StorageCandidate> allStoragePruningMsgs) { + for (StorageCandidate sc : allStoragePruningMsgs.keySet()) { + if (sc.getName().equals(stoargeName)) { + return allStoragePruningMsgs.get(sc); + } + } + return new ArrayList<CandidateTablePruneCause>(); + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java index 1eb7217..09e09f8 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java @@ -79,7 +79,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { return cal.getTime(); } - //TODO union : Wrong fact table picked. Check after MaxCoveringSetResolver + //TODO union : Revisit Continuous update period. @Test public void testCubeQueryContinuousUpdatePeriod() throws Exception { LensException th = null; @@ -102,10 +102,11 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + twoDaysInRangeClause, conf); Map<String, String> whereClauses = new HashMap<String, String>(); whereClauses.put( - getDbName() + "c1_testfact", + getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -2), getDateWithOffset(DAILY, 0), CONTINUOUS.format())); - String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, whereClauses); + String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", + null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -119,13 +120,13 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { whereClauses = new HashMap<String, String>(); whereClauses.put( - getDbName() + "c1_testfact", + getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -2), getDateWithOffset(DAILY, 0), CONTINUOUS.format()) + " OR" + TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -6), getDateWithOffset(DAILY, 0), CONTINUOUS.format())); - expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses); + expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -133,10 +134,10 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); whereClauses = new HashMap<String, String>(); - whereClauses.put(getDbName() + "c1_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, + whereClauses.put(getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getUptoHour(TWODAYS_BACK), getUptoHour(NOW), TestTimeRangeWriter.DB_FORMAT)); - expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses); + expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); } http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java index 935c739..931f789 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java @@ -107,8 +107,8 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite { + "sum(0.0) as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact2"; expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else " - + "(basecube.union_join_ctx_cityid) end as `alias2`, sum(0.0) as `alias3`, " + - "sum((basecube.union_join_ctx_msr2)) as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact3"; + + "(basecube.union_join_ctx_cityid) end as `alias2`, sum(0.0) as `alias3`, " + + "sum((basecube.union_join_ctx_msr2)) as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact3"; String outerGroupBy = "GROUP BY (basecube.alias0), (basecube.alias1), (basecube.alias2)"; compareContains(outerSelect, rewrittenQuery); compareContains(expectedInnerSelect1, rewrittenQuery); @@ -120,28 +120,25 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite { colsSelected = " union_join_ctx_cityid as `city id`, union_join_ctx_cityname, sum(union_join_ctx_msr1), " + "sum(union_join_ctx_msr2), union_join_ctx_non_zero_msr2_sum, union_join_ctx_msr1_greater_than_100, " + "sum(union_join_ctx_msr1) + 10 "; - //colsSelected = " union_join_ctx_cityid as `city id`, union_join_ctx_msr1_greater_than_100, union_join_ctx_non_zero_msr2_sum "; whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); outerSelect = "SELECT (basecube.alias0) as `city id`, (basecube.alias1) as `union_join_ctx_cityname`, " + "sum((basecube.alias2)) as `sum(union_join_ctx_msr1)`, sum((basecube.alias3)) " + "as `sum(union_join_ctx_msr2)`, sum((basecube.alias4)) as `union_join_ctx_non_zero_msr2_sum`, " - + "case when (sum((basecube.alias5)) > 100) then \"high\" else \"low\" end " - + "as `union_join_ctx_msr1_greater_than_100`, (sum((basecube.alias6)) + 10) " + + "case when (sum((basecube.alias2)) > 100) then \"high\" else \"low\" end as " + + "`union_join_ctx_msr1_greater_than_100`, (sum((basecube.alias2)) + 10) " + "as `(sum(union_join_ctx_msr1) + 10)` FROM "; expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, " + "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, " - + "sum(0.0) as `alias3`, sum(0.0) as `alias4`, sum((basecube.union_join_ctx_msr1)) as `alias5`, " - + "sum((basecube.union_join_ctx_msr1)) as `alias6`"; + + "sum(0.0) as `alias3`, sum(0.0) as `alias4` FROM"; expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, " + "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, " - + "sum(0.0) as `alias3`, sum(0.0) as `alias4`, sum((basecube.union_join_ctx_msr1)) as `alias5`, " - + "sum((basecube.union_join_ctx_msr1)) as `alias6`"; - expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, " - + "(cubecityjoinunionctx.name) as `alias1`, sum(0.0) as `alias2`, sum((basecube.union_join_ctx_msr2)) " - + "as `alias3`, sum(case when ((basecube.union_join_ctx_msr2) > 0) then (basecube.union_join_ctx_msr2) " - + "else 0 end) as `alias4`, sum(0.0) as `alias5`, sum(0.0) as `alias6`"; + + "sum(0.0) as `alias3`, sum(0.0) as `alias4` FROM"; + expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + + "as `alias1`, sum(0.0) as `alias2`, sum((basecube.union_join_ctx_msr2)) as `alias3`, " + + "sum(case when ((basecube.union_join_ctx_msr2) > 0) then (basecube.union_join_ctx_msr2) else 0 end) " + + "as `alias4` FROM"; String innerGroupBy = "GROUP BY (basecube.union_join_ctx_cityid), (cubecityjoinunionctx.name)"; outerGroupBy = "GROUP BY (basecube.alias0), (basecube.alias1)"; http://git-wip-us.apache.org/repos/asf/lens/blob/975fa2c2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 42282e9..c984a05 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -334,7 +334,6 @@ public class TestUnionQueries extends TestQueryRewrite { } } - //TODO union : Revisit after MaxCoveringFactResolver @Test public void testCubeWhereQueryWithMultipleTables() throws Exception { Configuration conf = getConf();
