[ASTERIXDB-1972][COMP][RT][TX] index-only plan - user model changes: no - storage format changes: no - interface changes: IAccessMethod, ILSMIndexOperationContext, IIndexAccessor
Details: - Implement an index-only plan - Add a SET option that disables the index-only plan Change-Id: Ifd5c9ab1cf2e4bedb7d8db582441919875e74d51 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1866 Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Taewoo Kim <wangs...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c3c23574 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c3c23574 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c3c23574 Branch: refs/heads/master Commit: c3c23574327d8c950820ea3e6539f4a1ade77fb3 Parents: 028537d Author: Taewoo Kim <wangs...@yahoo.com> Authored: Thu Feb 15 21:34:32 2018 -0800 Committer: Taewoo Kim <wangs...@gmail.com> Committed: Fri Feb 16 11:03:38 2018 -0800 ---------------------------------------------------------------------- .../physical/BTreeSearchPOperator.java | 3 +- .../physical/RTreeSearchPOperator.java | 8 +- .../asterix/optimizer/base/RuleCollections.java | 3 +- .../am/AbstractIntroduceAccessMethodRule.java | 183 +- .../rules/am/AccessMethodAnalysisContext.java | 64 +- .../optimizer/rules/am/AccessMethodUtils.java | 1858 +++++++++++++++++- .../optimizer/rules/am/BTreeAccessMethod.java | 338 +++- .../optimizer/rules/am/IAccessMethod.java | 25 +- .../rules/am/IntroduceJoinAccessMethodRule.java | 111 +- .../am/IntroduceLSMComponentFilterRule.java | 42 +- ...IntroducePrimaryIndexForAggregationRule.java | 2 +- .../am/IntroduceSelectAccessMethodRule.java | 143 +- .../rules/am/InvertedIndexAccessMethod.java | 125 +- .../rules/am/OptimizableOperatorSubTree.java | 67 +- .../optimizer/rules/am/RTreeAccessMethod.java | 270 ++- .../data/nontagged/customerData2.json | 6 + .../apache/asterix/api/common/APIFramework.java | 3 +- ...-pidx-with-join-btree-sidx_01-index-only.aql | 72 + ...rjoin-probe-pidx-with-join-btree-sidx_01.aql | 5 +- ...to-secondary-indexonly-plan-equi-join_01.aql | 69 + .../secondary-equi-join_04.sqlpp | 44 + .../secondary-equi-join_05.sqlpp | 42 + .../secondary-equi-join_06.sqlpp | 43 + ...y-indexonly-plan-to-primary-equi-join_01.aql | 71 + ...to-secondary-indexonly-plan-equi-join_01.aql | 72 + ...to-secondary-indexonly-plan-equi-join_01.aql | 74 + .../secondary-self-equi-join-index-only.aql | 65 + .../secondary-self-equi-join.aql | 2 +- ...ondary-composite-index-indexonly-plan-01.aql | 64 + ...dexonly-plan-01-disable-indexonly-plan.sqlpp | 67 + ...tree-secondary-index-indexonly-plan-01.sqlpp | 64 + .../btree-secondary-index-indexonly-plan-02.aql | 65 + .../btree-secondary-index-indexonly-plan-03.aql | 65 + .../btree-secondary-index-indexonly-plan-04.aql | 67 + .../btree-secondary-index-indexonly-plan-05.aql | 67 + .../btree-secondary-index-indexonly-plan-06.aql | 67 + .../btree-secondary-index-indexonly-plan-07.aql | 67 + .../btree-secondary-index-indexonly-plan-08.aql | 67 + .../btree-secondary-index-indexonly-plan-09.aql | 50 + ...-pidx-with-join-rtree-sidx_01-index-only.aql | 75 + ...rjoin-probe-pidx-with-join-rtree-sidx_01.aql | 2 +- .../nonpure/keep-datetime-local-index-only.aql | 74 + .../queries/nonpure/keep-datetime-local.aql | 2 +- ...orders-composite-index-search-index-only.aql | 46 + .../queries/orders-composite-index-search.aql | 18 +- .../queries/orders-index-search-index-only.aql | 53 + .../orders-index-search-open-index-only.aql | 51 + .../queries/orders-index-search-open.aql | 20 +- .../optimizerts/queries/orders-index-search.aql | 20 +- .../rtree-index-join/issue730-index-only.aql | 60 + .../queries/rtree-index-join/issue730.aql | 6 +- ...-pidx-with-join-rtree-sidx_01-index-only.aql | 72 + ...rjoin-probe-pidx-with-join-rtree-sidx_01.aql | 5 +- .../spatial-intersect-point_04.aql | 49 + .../spatial-intersect-point_05.aql | 49 + .../spatial-self-intersect-point-index-only.aql | 64 + .../spatial-self-intersect-point.aql | 2 +- ...dexonly-plan-01-disable-indexonly-plan.sqlpp | 114 ++ ...tree-secondary-index-indexonly-plan-01.sqlpp | 111 ++ .../rtree-secondary-index-indexonly-plan-02.aql | 111 ++ .../rtree-secondary-index-indexonly-plan-03.aql | 111 ++ .../rtree-secondary-index-indexonly-plan-04.aql | 114 ++ .../rtree-secondary-index-indexonly-plan-05.aql | 114 ++ .../rtree-secondary-index-indexonly-plan-06.aql | 114 ++ .../rtree-secondary-index-indexonly-plan-07.aql | 114 ++ .../rtree-secondary-index-indexonly-plan-08.aql | 114 ++ .../rtree-secondary-index-indexonly-plan-09.aql | 112 ++ .../rtree-secondary-index-index-only.aql | 51 + .../rtree-secondary-index-open-index-only.aql | 51 + .../queries/rtree-secondary-index-open.aql | 3 +- .../queries/rtree-secondary-index.aql | 5 +- .../udfs/query-ASTERIXDB-1019-index-only.aql | 75 + .../queries/udfs/query-ASTERIXDB-1019.aql | 2 + ...pidx-with-join-btree-sidx_01-index-only.plan | 50 + ...o-secondary-indexonly-plan-equi-join_01.plan | 45 + .../secondary-equi-join_04.plan | 8 +- .../secondary-equi-join_05.plan | 2 +- .../secondary-equi-join_06.plan | 8 +- ...-indexonly-plan-to-primary-equi-join_01.plan | 36 + ...o-secondary-indexonly-plan-equi-join_01.plan | 81 + ...o-secondary-indexonly-plan-equi-join_01.plan | 65 + .../secondary-self-equi-join-index-only.plan | 47 + ...ndary-composite-index-indexonly-plan-01.plan | 31 + ...ndexonly-plan-01-disable-indexonly-plan.plan | 19 + ...btree-secondary-index-indexonly-plan-01.plan | 30 + ...btree-secondary-index-indexonly-plan-02.plan | 33 + ...btree-secondary-index-indexonly-plan-03.plan | 31 + ...btree-secondary-index-indexonly-plan-04.plan | 32 + ...btree-secondary-index-indexonly-plan-05.plan | 35 + ...btree-secondary-index-indexonly-plan-06.plan | 33 + ...btree-secondary-index-indexonly-plan-07.plan | 35 + ...btree-secondary-index-indexonly-plan-08.plan | 33 + ...btree-secondary-index-indexonly-plan-09.plan | 53 + ...pidx-with-join-rtree-sidx_01-index-only.plan | 41 + .../nonpure/keep-datetime-local-index-only.plan | 58 + ...rders-composite-index-search-index-only.plan | 30 + .../results/orders-index-search-index-only.plan | 29 + .../orders-index-search-open-index-only.plan | 29 + .../rtree-index-join/issue730-index-only.plan | 62 + ...pidx-with-join-rtree-sidx_01-index-only.plan | 62 + .../spatial-intersect-point_04.plan | 47 + .../spatial-intersect-point_05.plan | 47 + ...spatial-self-intersect-point-index-only.plan | 57 + ...ndexonly-plan-01-disable-indexonly-plan.plan | 22 + ...rtree-secondary-index-indexonly-plan-01.plan | 39 + ...rtree-secondary-index-indexonly-plan-02.plan | 33 + ...rtree-secondary-index-indexonly-plan-03.plan | 37 + ...rtree-secondary-index-indexonly-plan-04.plan | 38 + ...rtree-secondary-index-indexonly-plan-05.plan | 35 + ...rtree-secondary-index-indexonly-plan-06.plan | 39 + ...rtree-secondary-index-indexonly-plan-07.plan | 35 + ...rtree-secondary-index-indexonly-plan-08.plan | 33 + ...rtree-secondary-index-indexonly-plan-09.plan | 35 + .../rtree-secondary-index-index-only.plan | 47 + .../rtree-secondary-index-open-index-only.plan | 47 + .../udfs/query-ASTERIXDB-1019-index-only.plan | 145 ++ ...ad-with-secondary-btree-index-only.1.ddl.aql | 38 + ...with-secondary-btree-index-only.2.update.aql | 22 + ...ad-with-secondary-btree-index-only.3.ddl.aql | 21 + ...with-secondary-btree-index-only.4.update.aql | 18 + ...-with-secondary-btree-index-only.5.query.aql | 24 + ...ondary-indexonly-plan-equi-join_01.1.ddl.aql | 64 + ...ary-indexonly-plan-equi-join_01.2.update.aql | 36 + ...dary-indexonly-plan-equi-join_01.3.query.aql | 34 + .../btree-secondary-equi-join_04.1.ddl.aql | 52 + .../btree-secondary-equi-join_04.2.update.aql | 29 + .../btree-secondary-equi-join_04.3.ddl.aql | 24 + .../btree-secondary-equi-join_04.4.query.aql | 26 + ...xonly-plan-to-primary-equi-join_01.1.ddl.aql | 63 + ...ly-plan-to-primary-equi-join_01.2.update.aql | 29 + ...nly-plan-to-primary-equi-join_01.3.query.aql | 27 + ...ondary-indexonly-plan-equi-join_01.1.ddl.aql | 65 + ...ary-indexonly-plan-equi-join_01.2.update.aql | 29 + ...dary-indexonly-plan-equi-join_01.3.query.aql | 27 + ...xonly-plan-to-primary-equi-join_01.1.ddl.aql | 66 + ...ly-plan-to-primary-equi-join_01.2.update.aql | 29 + ...nly-plan-to-primary-equi-join_01.3.query.aql | 27 + ...ondary-indexonly-plan-equi-join_01.1.ddl.aql | 67 + ...ary-indexonly-plan-equi-join_01.2.update.aql | 29 + ...dary-indexonly-plan-equi-join_01.3.query.aql | 27 + .../rtree-spatial-intersect-point.1.ddl.aql | 45 - .../rtree-spatial-intersect-point.2.update.aql | 35 - .../rtree-spatial-intersect-point.3.ddl.aql | 29 - .../rtree-spatial-intersect-point.4.query.aql | 32 - .../rtree-spatial-intersect-point_01.1.ddl.aql | 45 + ...tree-spatial-intersect-point_01.2.update.aql | 35 + .../rtree-spatial-intersect-point_01.3.ddl.aql | 29 + ...rtree-spatial-intersect-point_01.4.query.aql | 32 + .../rtree-spatial-intersect-point_02.1.ddl.aql | 46 + ...tree-spatial-intersect-point_02.2.update.aql | 29 + .../rtree-spatial-intersect-point_02.3.ddl.aql | 30 + ...rtree-spatial-intersect-point_02.4.query.aql | 33 + .../rtree-spatial-intersect-point_03.1.ddl.aql | 47 + ...tree-spatial-intersect-point_03.2.update.aql | 29 + .../rtree-spatial-intersect-point_03.3.ddl.aql | 24 + ...rtree-spatial-intersect-point_03.4.query.aql | 26 + .../rtree-spatial-intersect-point_04.1.ddl.aql | 47 + ...tree-spatial-intersect-point_04.2.update.aql | 29 + .../rtree-spatial-intersect-point_04.3.ddl.aql | 24 + ...rtree-spatial-intersect-point_04.4.query.aql | 26 + .../rtree-spatial-intersect-point_05.1.ddl.aql | 47 + ...tree-spatial-intersect-point_05.2.update.aql | 29 + .../rtree-spatial-intersect-point_05.3.ddl.aql | 24 + ...rtree-spatial-intersect-point_05.4.query.aql | 26 + .../probe-pidx-with-join-btree-pidx1.1.ddl.aql | 56 + ...robe-pidx-with-join-btree-pidx1.2.update.aql | 30 + ...probe-pidx-with-join-btree-pidx1.3.query.aql | 40 + ...robe-pidx-with-join-btree-sidx2.2.update.aql | 4 +- ...probe-pidx-with-join-btree-sidx2.3.query.aql | 6 +- ...th-join-btree-sidx3-indexonly-plan.1.ddl.aql | 56 + ...join-btree-sidx3-indexonly-plan.2.update.aql | 24 + ...-join-btree-sidx3-indexonly-plan.3.query.aql | 32 + ...dexonly-plan-with-join-btree-pidx1.1.ddl.aql | 57 + ...only-plan-with-join-btree-pidx1.2.update.aql | 24 + ...xonly-plan-with-join-btree-pidx1.3.query.aql | 32 + ...th-join-btree-sidx1-indexonly-plan.1.ddl.aql | 60 + ...join-btree-sidx1-indexonly-plan.2.update.aql | 24 + ...-join-btree-sidx1-indexonly-plan.3.query.aql | 32 + ...dexonly-plan-with-join-btree-pidx1.1.ddl.aql | 57 + ...only-plan-with-join-btree-pidx1.2.update.aql | 24 + ...xonly-plan-with-join-btree-pidx1.3.query.aql | 32 + ...th-join-btree-sidx1-indexonly-plan.1.ddl.aql | 59 + ...join-btree-sidx1-indexonly-plan.2.update.aql | 24 + ...-join-btree-sidx1-indexonly-plan.3.query.aql | 32 + ...-composite-index-indexonly-plan-01.1.ddl.aql | 62 + ...mposite-index-indexonly-plan-01.2.update.aql | 25 + ...omposite-index-indexonly-plan-01.3.query.aql | 25 + ...-composite-index-indexonly-plan-02.1.ddl.aql | 62 + ...mposite-index-indexonly-plan-02.2.update.aql | 24 + ...omposite-index-indexonly-plan-03.3.query.aql | 24 + ...-composite-index-indexonly-plan-03.1.ddl.aql | 62 + ...mposite-index-indexonly-plan-03.2.update.aql | 25 + ...omposite-index-indexonly-plan-03.3.query.aql | 24 + ...-secondary-index-indexonly-plan-01.1.ddl.aql | 60 + ...condary-index-indexonly-plan-01.10.query.aql | 28 + ...condary-index-indexonly-plan-01.2.update.aql | 25 + ...econdary-index-indexonly-plan-01.3.query.aql | 25 + ...econdary-index-indexonly-plan-01.4.query.aql | 25 + ...econdary-index-indexonly-plan-01.5.query.aql | 25 + ...econdary-index-indexonly-plan-01.6.query.aql | 28 + ...econdary-index-indexonly-plan-01.7.query.aql | 28 + ...econdary-index-indexonly-plan-01.8.query.aql | 28 + ...econdary-index-indexonly-plan-01.9.query.aql | 27 + ...ondary-index-non-indexonly-plan-01.1.ddl.aql | 56 + ...ary-index-non-indexonly-plan-01.2.update.aql | 25 + ...dary-index-non-indexonly-plan-01.3.query.aql | 26 + ...-secondary-index-indexonly-plan-01.1.ddl.aql | 107 + ...condary-index-indexonly-plan-01.10.query.aql | 28 + ...condary-index-indexonly-plan-01.11.query.aql | 27 + ...condary-index-indexonly-plan-01.2.update.aql | 29 + ...econdary-index-indexonly-plan-01.3.query.aql | 24 + ...econdary-index-indexonly-plan-01.4.query.aql | 26 + ...econdary-index-indexonly-plan-01.5.query.aql | 26 + ...econdary-index-indexonly-plan-01.6.query.aql | 28 + ...econdary-index-indexonly-plan-01.7.query.aql | 28 + ...econdary-index-indexonly-plan-01.8.query.aql | 28 + ...econdary-index-indexonly-plan-01.9.query.aql | 28 + ...ondary-index-non-indexonly-plan-01.1.ddl.aql | 56 + ...ary-index-non-indexonly-plan-01.2.update.aql | 25 + ...dary-index-non-indexonly-plan-01.3.query.aql | 27 + ...ondary-index-non-indexonly-plan-02.1.ddl.aql | 57 + ...ary-index-non-indexonly-plan-02.2.update.aql | 25 + ...dary-index-non-indexonly-plan-02.3.query.aql | 28 + .../orderby_limit_02.3.query.aql | 2 +- ...-with-secondary-btree-index-only.1.ddl.sqlpp | 40 + ...th-secondary-btree-index-only.2.update.sqlpp | 24 + ...-with-secondary-btree-index-only.3.ddl.sqlpp | 24 + ...th-secondary-btree-index-only.4.update.sqlpp | 19 + ...ith-secondary-btree-index-only.5.query.sqlpp | 26 + ...dary-indexonly-plan-equi-join_01.1.ddl.sqlpp | 64 + ...y-indexonly-plan-equi-join_01.2.update.sqlpp | 36 + ...ry-indexonly-plan-equi-join_01.3.query.sqlpp | 32 + .../btree-secondary-equi-join_04.1.ddl.sqlpp | 52 + .../btree-secondary-equi-join_04.2.update.sqlpp | 29 + .../btree-secondary-equi-join_04.3.ddl.sqlpp | 24 + .../btree-secondary-equi-join_04.4.query.sqlpp | 25 + ...nly-plan-to-primary-equi-join_01.1.ddl.sqlpp | 63 + ...-plan-to-primary-equi-join_01.2.update.sqlpp | 29 + ...y-plan-to-primary-equi-join_01.3.query.sqlpp | 26 + ...dary-indexonly-plan-equi-join_01.1.ddl.sqlpp | 65 + ...y-indexonly-plan-equi-join_01.2.update.sqlpp | 29 + ...ry-indexonly-plan-equi-join_01.3.query.sqlpp | 26 + ...nly-plan-to-primary-equi-join_01.1.ddl.sqlpp | 66 + ...-plan-to-primary-equi-join_01.2.update.sqlpp | 29 + ...y-plan-to-primary-equi-join_01.3.query.sqlpp | 26 + ...dary-indexonly-plan-equi-join_01.1.ddl.sqlpp | 67 + ...y-indexonly-plan-equi-join_01.2.update.sqlpp | 29 + ...ry-indexonly-plan-equi-join_01.3.query.sqlpp | 26 + .../rtree-spatial-intersect-point.1.ddl.sqlpp | 48 - ...rtree-spatial-intersect-point.2.update.sqlpp | 32 - .../rtree-spatial-intersect-point.3.ddl.sqlpp | 30 - .../rtree-spatial-intersect-point.4.query.sqlpp | 34 - ...rtree-spatial-intersect-point_01.1.ddl.sqlpp | 48 + ...ee-spatial-intersect-point_01.2.update.sqlpp | 32 + ...rtree-spatial-intersect-point_01.3.ddl.sqlpp | 30 + ...ree-spatial-intersect-point_01.4.query.sqlpp | 34 + ...rtree-spatial-intersect-point_02.1.ddl.sqlpp | 46 + ...ee-spatial-intersect-point_02.2.update.sqlpp | 29 + ...rtree-spatial-intersect-point_02.3.ddl.sqlpp | 30 + ...ree-spatial-intersect-point_02.4.query.sqlpp | 34 + ...rtree-spatial-intersect-point_03.1.ddl.sqlpp | 47 + ...ee-spatial-intersect-point_03.2.update.sqlpp | 29 + ...rtree-spatial-intersect-point_03.3.ddl.sqlpp | 24 + ...ree-spatial-intersect-point_03.4.query.sqlpp | 27 + ...rtree-spatial-intersect-point_04.1.ddl.sqlpp | 47 + ...ee-spatial-intersect-point_04.2.update.sqlpp | 29 + ...rtree-spatial-intersect-point_04.3.ddl.sqlpp | 24 + ...ree-spatial-intersect-point_04.4.query.sqlpp | 27 + ...rtree-spatial-intersect-point_05.1.ddl.sqlpp | 47 + ...ee-spatial-intersect-point_05.2.update.sqlpp | 29 + ...rtree-spatial-intersect-point_05.3.ddl.sqlpp | 24 + ...ree-spatial-intersect-point_05.4.query.sqlpp | 27 + ...-join-btree-sidx3-indexonly-plan.1.ddl.sqlpp | 56 + ...in-btree-sidx3-indexonly-plan.2.update.sqlpp | 24 + ...oin-btree-sidx3-indexonly-plan.3.query.sqlpp | 29 + ...xonly-plan-with-join-btree-pidx1.1.ddl.sqlpp | 57 + ...ly-plan-with-join-btree-pidx1.2.update.sqlpp | 24 + ...nly-plan-with-join-btree-pidx1.3.query.sqlpp | 30 + ...-join-btree-sidx1-indexonly-plan.1.ddl.sqlpp | 60 + ...in-btree-sidx1-indexonly-plan.2.update.sqlpp | 24 + ...oin-btree-sidx1-indexonly-plan.3.query.sqlpp | 29 + ...xonly-plan-with-join-btree-pidx1.1.ddl.sqlpp | 57 + ...ly-plan-with-join-btree-pidx1.2.update.sqlpp | 24 + ...nly-plan-with-join-btree-pidx1.3.query.sqlpp | 30 + ...-join-btree-sidx1-indexonly-plan.1.ddl.sqlpp | 59 + ...in-btree-sidx1-indexonly-plan.2.update.sqlpp | 24 + ...oin-btree-sidx1-indexonly-plan.3.query.sqlpp | 29 + ...omposite-index-indexonly-plan-01.1.ddl.sqlpp | 60 + ...osite-index-indexonly-plan-01.2.update.sqlpp | 25 + ...posite-index-indexonly-plan-01.3.query.sqlpp | 25 + ...omposite-index-indexonly-plan-02.1.ddl.sqlpp | 60 + ...osite-index-indexonly-plan-02.2.update.sqlpp | 24 + ...posite-index-indexonly-plan-02.3.query.sqlpp | 24 + ...omposite-index-indexonly-plan-03.1.ddl.sqlpp | 60 + ...osite-index-indexonly-plan-03.2.update.sqlpp | 25 + ...posite-index-indexonly-plan-03.3.query.sqlpp | 25 + ...econdary-index-indexonly-plan-01.1.ddl.sqlpp | 60 + ...ndary-index-indexonly-plan-01.10.query.sqlpp | 27 + ...ndary-index-indexonly-plan-01.2.update.sqlpp | 25 + ...ondary-index-indexonly-plan-01.3.query.sqlpp | 25 + ...ondary-index-indexonly-plan-01.4.query.sqlpp | 25 + ...ondary-index-indexonly-plan-01.5.query.sqlpp | 25 + ...ondary-index-indexonly-plan-01.6.query.sqlpp | 26 + ...ondary-index-indexonly-plan-01.7.query.sqlpp | 27 + ...ondary-index-indexonly-plan-01.8.query.sqlpp | 26 + ...ondary-index-indexonly-plan-01.9.query.sqlpp | 26 + ...dary-index-non-indexonly-plan-01.1.ddl.sqlpp | 56 + ...y-index-non-indexonly-plan-01.2.update.sqlpp | 25 + ...ry-index-non-indexonly-plan-01.3.query.sqlpp | 25 + ...econdary-index-indexonly-plan-01.1.ddl.sqlpp | 107 + ...ndary-index-indexonly-plan-01.10.query.sqlpp | 27 + ...ndary-index-indexonly-plan-01.11.query.sqlpp | 26 + ...ndary-index-indexonly-plan-01.2.update.sqlpp | 29 + ...ondary-index-indexonly-plan-01.3.query.sqlpp | 26 + ...ondary-index-indexonly-plan-01.4.query.sqlpp | 26 + ...ondary-index-indexonly-plan-01.5.query.sqlpp | 26 + ...ondary-index-indexonly-plan-01.6.query.sqlpp | 27 + ...ondary-index-indexonly-plan-01.7.query.sqlpp | 27 + ...ondary-index-indexonly-plan-01.8.query.sqlpp | 27 + ...ondary-index-indexonly-plan-01.9.query.sqlpp | 27 + ...dary-index-non-indexonly-plan-01.1.ddl.sqlpp | 56 + ...y-index-non-indexonly-plan-01.2.update.sqlpp | 25 + ...ry-index-non-indexonly-plan-01.3.query.sqlpp | 27 + ...dary-index-non-indexonly-plan-02.1.ddl.sqlpp | 57 + ...y-index-non-indexonly-plan-02.2.update.sqlpp | 25 + ...ry-index-non-indexonly-plan-02.3.query.sqlpp | 27 + .../load-with-secondary-btree-index-only.1.adm | 15 + ...-secondary-indexonly-plan-equi-join_01.1.adm | 1 + .../btree-secondary-equi-join_04.1.adm | 5 + ...indexonly-plan-to-primary-equi-join_01.1.adm | 1 + ...-secondary-indexonly-plan-equi-join_01.1.adm | 1 + ...indexonly-plan-to-primary-equi-join_01.1.adm | 1 + ...-secondary-indexonly-plan-equi-join_01.1.adm | 1 + .../rtree-spatial-intersect-point.1.adm | 44 - .../rtree-spatial-intersect-point_01.1.adm | 44 + .../rtree-spatial-intersect-point_02.1.adm | 65 + .../rtree-spatial-intersect-point_03.1.adm | 65 + .../rtree-spatial-intersect-point_04.1.adm | 20 + .../rtree-spatial-intersect-point_05.1.adm | 204 ++ .../probe-pidx-with-join-btree-pidx1.1.adm | 1 + ...x-with-join-rtree-sidx3-indexonly-plan.1.adm | 1 + ...e-indexonly-plan-with-join-btree-pidx1.1.adm | 1 + ...n-with-join-btree-sidx1-indexonly-plan.1.adm | 1 + ...n-indexonly-plan-with-join-btree-pidx1.1.adm | 1 + ...n-with-join-btree-sidx1-indexonly-plan.1.adm | 1 + ...dary-composite-index-indexonly-plan-01.1.adm | 2 + ...dary-composite-index-indexonly-plan-02.1.adm | 2 + ...dary-composite-index-indexonly-plan-03.1.adm | 2 + ...ree-secondary-index-indexonly-plan-01.10.adm | 1 + ...tree-secondary-index-indexonly-plan-01.3.adm | 2 + ...tree-secondary-index-indexonly-plan-01.4.adm | 2 + ...tree-secondary-index-indexonly-plan-01.5.adm | 2 + ...tree-secondary-index-indexonly-plan-01.6.adm | 1 + ...tree-secondary-index-indexonly-plan-01.7.adm | 1 + ...tree-secondary-index-indexonly-plan-01.8.adm | 1 + ...tree-secondary-index-indexonly-plan-01.9.adm | 1 + ...-secondary-index-non-indexonly-plan-01.1.adm | 1 + ...ree-secondary-index-indexonly-plan-01.10.adm | 1 + ...ree-secondary-index-indexonly-plan-01.11.adm | 30 + ...tree-secondary-index-indexonly-plan-01.3.adm | 1 + ...tree-secondary-index-indexonly-plan-01.4.adm | 6 + ...tree-secondary-index-indexonly-plan-01.5.adm | 6 + ...tree-secondary-index-indexonly-plan-01.6.adm | 1 + ...tree-secondary-index-indexonly-plan-01.7.adm | 1 + ...tree-secondary-index-indexonly-plan-01.8.adm | 1 + ...tree-secondary-index-indexonly-plan-01.9.adm | 1 + ...-secondary-index-non-indexonly-plan-01.1.adm | 1 + ...-secondary-index-non-indexonly-plan-02.1.adm | 1 + .../orderby_limit_02/orderby_limit_02.1.adm | 4 +- .../src/test/resources/runtimets/testsuite.xml | 124 +- .../resources/runtimets/testsuite_sqlpp.xml | 126 +- .../asterix/common/exceptions/ErrorCode.java | 6 + .../main/resources/asx_errormsg/en.properties | 6 + .../markdown/sqlpp/appendix_2_parameters.md | 18 + .../metadata/declared/DatasetDataSource.java | 2 +- .../metadata/declared/MetadataProvider.java | 60 +- .../asterix/metadata/entities/Dataset.java | 38 +- .../serde/SerializerDeserializerUtil.java | 21 + .../om/types/hierachy/ATypeHierarchy.java | 54 +- .../DoubleToFloatTypeConvertComputer.java | 8 +- .../DoubleToInt16TypeConvertComputer.java | 7 +- .../DoubleToInt32TypeConvertComputer.java | 7 +- .../DoubleToInt64TypeConvertComputer.java | 7 +- .../DoubleToInt8TypeConvertComputer.java | 7 +- .../FloatToDoubleTypeConvertComputer.java | 10 +- .../FloatToInt16TypeConvertComputer.java | 7 +- .../FloatToInt32TypeConvertComputer.java | 7 +- .../FloatToInt64TypeConvertComputer.java | 7 +- .../FloatToInt8TypeConvertComputer.java | 7 +- .../om/types/hierachy/ITypeConvertComputer.java | 4 +- .../IntegerToDoubleTypeConvertComputer.java | 4 +- .../IntegerToFloatTypeConvertComputer.java | 4 +- .../IntegerToInt16TypeConvertComputer.java | 4 +- .../IntegerToInt32TypeConvertComputer.java | 4 +- .../IntegerToInt64TypeConvertComputer.java | 4 +- .../IntegerToInt8TypeConvertComputer.java | 4 +- ...xInstanctSearchOperationCallbackFactory.java | 68 + ...daryIndexInstantSearchOperationCallback.java | 72 + .../algebricks/common/utils/Quadruple.java | 84 + .../logical/AbstractUnnestMapOperator.java | 27 +- .../logical/LeftOuterUnnestMapOperator.java | 3 +- .../LogicalOperatorPrettyPrintVisitor.java | 2 +- .../LogicalOperatorPrettyPrintVisitorJson.java | 3 +- .../algebra/util/OperatorPropertiesUtil.java | 20 +- .../rules/ExtractCommonOperatorsRule.java | 3 +- .../rewriter/rules/IntroduceProjectsRule.java | 33 +- .../RemoveUnusedAssignAndAggregateRule.java | 5 +- .../hyracks/api/exceptions/ErrorCode.java | 2 + .../src/main/resources/errormsg/en.properties | 2 + .../dataflow/BTreeSearchOperatorDescriptor.java | 27 +- .../BTreeSearchOperatorNodePushable.java | 16 +- .../storage/am/common/api/ILSMIndexCursor.java | 10 + .../IndexSearchOperatorNodePushable.java | 44 +- .../btree/impls/LSMBTreePointSearchCursor.java | 11 +- .../btree/impls/LSMBTreeRangeSearchCursor.java | 88 +- .../lsm/btree/impls/LSMBTreeSearchCursor.java | 5 + .../impls/LSMBTreeWithBuddyAbstractCursor.java | 6 + .../lsm/common/impls/LSMIndexSearchCursor.java | 5 + .../impls/LSMInvertedIndexSearchCursor.java | 6 + .../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 10 +- .../lsm/rtree/impls/LSMRTreeSearchCursor.java | 8 + ...LSMRTreeWithAntiMatterTuplesFlushCursor.java | 5 + ...SMRTreeWithAntiMatterTuplesSearchCursor.java | 12 +- .../dataflow/RTreeSearchOperatorDescriptor.java | 20 +- .../RTreeSearchOperatorNodePushable.java | 16 +- 425 files changed, 16222 insertions(+), 1023 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java index 11b86cb..beb47a8 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java @@ -125,7 +125,8 @@ public class BTreeSearchPOperator extends IndexSearchPOperator { Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime( builder.getJobSpec(), opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainMissing, dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(), - jobGenParams.isHighKeyInclusive(), propagateFilter, minFilterFieldIndexes, maxFilterFieldIndexes); + jobGenParams.isHighKeyInclusive(), propagateFilter, minFilterFieldIndexes, maxFilterFieldIndexes, + unnestMap.getGenerateCallBackProceedResultVar()); builder.contributeHyracksOperator(unnestMap, btreeSearch.first); builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java index 733e62f..949be23 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java @@ -98,10 +98,10 @@ public class RTreeSearchPOperator extends IndexSearchPOperator { // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples. retainNull = true; } - Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch = - mp.buildRtreeRuntime(builder.getJobSpec(), outputVars, opSchema, typeEnv, context, - jobGenParams.getRetainInput(), retainNull, dataset, jobGenParams.getIndexName(), keyIndexes, - propagateIndexFilter, minFilterFieldIndexes, maxFilterFieldIndexes); + Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch = mp.buildRtreeRuntime( + builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainNull, + dataset, jobGenParams.getIndexName(), keyIndexes, propagateIndexFilter, minFilterFieldIndexes, + maxFilterFieldIndexes, unnestMap.getGenerateCallBackProceedResultVar()); builder.contributeHyracksOperator(unnestMap, rtreeSearch.first); builder.contributeAlgebricksPartitionConstraint(rtreeSearch.first, rtreeSearch.second); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java index bd67ed4..cd217ab 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java @@ -332,7 +332,6 @@ public final class RuleCollections { physicalRewritesAllLevels.add(new RemoveSortInFeedIngestionRule()); physicalRewritesAllLevels.add(new RemoveUnnecessarySortMergeExchange()); physicalRewritesAllLevels.add(new PushProjectDownRule()); - physicalRewritesAllLevels.add(new InsertProjectBeforeUnionRule()); physicalRewritesAllLevels.add(new IntroduceMaterializationForInsertWithSelfScanRule()); physicalRewritesAllLevels.add(new InlineSingleReferenceVariablesRule()); physicalRewritesAllLevels.add(new RemoveUnusedAssignAndAggregateRule()); @@ -361,6 +360,8 @@ public final class RuleCollections { public static final List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() { List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<>(); + prepareForJobGenRewrites.add(new InsertProjectBeforeUnionRule()); + prepareForJobGenRewrites.add(new SetAlgebricksPhysicalOperatorsRule()); prepareForJobGenRewrites .add(new IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled)); prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index e0c6249..f4178d7 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer; import org.apache.asterix.metadata.declared.MetadataProvider; @@ -44,6 +45,7 @@ import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.asterix.om.utils.ConstantExpressionUtil; +import org.apache.asterix.optimizer.base.AnalysisUtil; import org.apache.asterix.optimizer.rules.am.OptimizableOperatorSubTree.DataSourceType; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -63,6 +65,9 @@ import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFun import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext; @@ -75,6 +80,9 @@ import com.google.common.collect.ImmutableSet; * methods. */ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRewriteRule { + // When this option is set to true before executing a query, we don't apply the index-only plan. + public final static String NO_INDEX_ONLY_PLAN_OPTION = "noindexonly"; + public final static boolean NO_INDEX_ONLY_PLAN_OPTION_DEFAULT_VALUE = false; protected MetadataProvider metadataProvider; @@ -91,12 +99,12 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew protected static void registerAccessMethod(IAccessMethod accessMethod, Map<FunctionIdentifier, List<IAccessMethod>> accessMethods) { - List<FunctionIdentifier> funcs = accessMethod.getOptimizableFunctions(); - for (FunctionIdentifier funcIdent : funcs) { - List<IAccessMethod> l = accessMethods.get(funcIdent); + List<Pair<FunctionIdentifier, Boolean>> funcs = accessMethod.getOptimizableFunctions(); + for (Pair<FunctionIdentifier, Boolean> funcIdent : funcs) { + List<IAccessMethod> l = accessMethods.get(funcIdent.first); if (l == null) { - l = new ArrayList<>(); - accessMethods.put(funcIdent, l); + l = new ArrayList<IAccessMethod>(); + accessMethods.put(funcIdent.first, l); } l.add(accessMethod); } @@ -550,16 +558,13 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew int optFuncExprIndex = 0; List<Index> datasetIndexes = new ArrayList<>(); LogicalVariable datasetMetaVar = null; - LogicalVariable datasetRecordVar = null; - if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN) { + if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN + && subTree.getDataSourceType() != DataSourceType.INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP) { datasetIndexes = metadataProvider.getDatasetIndexes(subTree.getDataset().getDataverseName(), subTree.getDataset().getDatasetName()); List<LogicalVariable> datasetVars = subTree.getDataSourceVariables(); if (subTree.getDataset().hasMetaPart()) { datasetMetaVar = datasetVars.get(datasetVars.size() - 1); - datasetRecordVar = datasetVars.get(datasetVars.size() - 2); - } else { - datasetRecordVar = datasetVars.get(datasetVars.size() - 1); } } for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) { @@ -568,11 +573,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew .size(); assignOrUnnestIndex++) { AbstractLogicalOperator op = subTree.getAssignsAndUnnests().get(assignOrUnnestIndex); if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) { - analyzeAssignOp((AssignOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetRecordVar, - datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx); + analyzeAssignOp((AssignOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetMetaVar, + context, datasetIndexes, optFuncExprIndex, analysisCtx); } else { - analyzeUnnestOp((UnnestOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetRecordVar, - datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx); + analyzeUnnestOp((UnnestOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetMetaVar, + context, datasetIndexes, optFuncExprIndex, analysisCtx); } } @@ -602,9 +607,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew } private void analyzeUnnestOp(UnnestOperator unnestOp, IOptimizableFuncExpr optFuncExpr, - OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable datasetRecordVar, - LogicalVariable datasetMetaVar, IOptimizationContext context, List<Index> datasetIndexes, - int optFuncExprIndex, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException { + OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable datasetMetaVar, + IOptimizationContext context, List<Index> datasetIndexes, int optFuncExprIndex, + AccessMethodAnalysisContext analysisCtx) throws AlgebricksException { LogicalVariable var = unnestOp.getVariable(); int funcVarIndex = optFuncExpr.findLogicalVar(var); // No matching var in optFuncExpr. @@ -621,7 +626,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew } else { fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0, subTree.getRecordType(), funcVarIndex, optFuncExpr.getFuncExpr().getArguments().get(funcVarIndex).getValue(), - datasetRecordVar, subTree.getMetaRecordType(), datasetMetaVar); + subTree.getMetaRecordType(), datasetMetaVar); if (fieldName.isEmpty()) { return; } @@ -641,9 +646,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew } private void analyzeAssignOp(AssignOperator assignOp, IOptimizableFuncExpr optFuncExpr, - OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable datasetRecordVar, - LogicalVariable datasetMetaVar, IOptimizationContext context, List<Index> datasetIndexes, - int optFuncExprIndex, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException { + OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable datasetMetaVar, + IOptimizationContext context, List<Index> datasetIndexes, int optFuncExprIndex, + AccessMethodAnalysisContext analysisCtx) throws AlgebricksException { List<LogicalVariable> varList = assignOp.getVariables(); for (int varIndex = 0; varIndex < varList.size(); varIndex++) { LogicalVariable var = varList.get(varIndex); @@ -659,8 +664,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew List<String> fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex, subTree.getRecordType(), optVarIndex, - optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue(), datasetRecordVar, - subTree.getMetaRecordType(), datasetMetaVar); + optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue(), subTree.getMetaRecordType(), + datasetMetaVar); IAType fieldType = (IAType) context.getOutputTypeEnvironment(assignOp).getVarType(var); // Set the fieldName in the corresponding matched @@ -742,14 +747,14 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew /** * Returns the field name corresponding to the assigned variable at - * varIndex. Returns null if the expr at varIndex does not yield to a field + * varIndex. Returns Collections.emptyList() if the expr at varIndex does not yield to a field * access function after following a set of allowed functions. * * @throws AlgebricksException */ protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex, ARecordType recordType, int funcVarIndex, - ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, LogicalVariable metaVar) + ILogicalExpression parentFuncExpr, ARecordType metaType, LogicalVariable metaVar) throws AlgebricksException { // Get expression corresponding to opVar at varIndex. AbstractLogicalExpression expr = null; @@ -818,7 +823,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew if (isFieldAccess) { LogicalVariable sourceVar = ((VariableReferenceExpression) funcExpr.getArguments().get(0).getValue()).getVariableReference(); - optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr); + if (optFuncExpr != null) { + optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr); + } int[] assignAndExpressionIndexes = null; //go forward through nested assigns until you find the relevant one @@ -856,7 +863,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew //Recursive call on nested assign List<String> parentFieldNames = getFieldNameFromSubTree(optFuncExpr, subTree, assignAndExpressionIndexes[0], assignAndExpressionIndexes[1], recordType, funcVarIndex, - parentFuncExpr, recordVar, metaType, metaVar); + parentFuncExpr, metaType, metaVar); if (parentFieldNames.isEmpty()) { //Nested assign was not a field access. @@ -869,7 +876,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew ? ((ARecordType) metaType.getSubFieldType(parentFieldNames)).getFieldNames()[fieldIndex] : ((ARecordType) recordType.getSubFieldType(parentFieldNames)).getFieldNames()[fieldIndex]; } - optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex)); + if (optFuncExpr != null) { + optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex)); + } //add fieldName to the nested fieldName, return if (nestedAccessFieldName != null) { for (int i = 0; i < nestedAccessFieldName.size(); i++) { @@ -881,7 +890,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew return (parentFieldNames); } - optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex)); + if (optFuncExpr != null) { + optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex)); + } //no nested assign, we are at the lowest level. if (isByName) { if (nestedAccessFieldName != null) { @@ -898,7 +909,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew return Collections.emptyList(); } // We use a part of the field in edit distance computation - if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.EDIT_DISTANCE_CHECK) { + if (optFuncExpr != null + && optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.EDIT_DISTANCE_CHECK) { optFuncExpr.setPartialField(true); } // We expect the function's argument to be a variable, otherwise we @@ -918,10 +930,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew List<LogicalVariable> varList = assignOp.getVariables(); for (int varIndex = 0; varIndex < varList.size(); varIndex++) { LogicalVariable var = varList.get(varIndex); - if (var.equals(curVar)) { + if (var.equals(curVar) && optFuncExpr != null) { optFuncExpr.setSourceVar(funcVarIndex, var); return getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex, recordType, - funcVarIndex, childFuncExpr, recordVar, metaType, metaVar); + funcVarIndex, childFuncExpr, metaType, metaVar); } } } else { @@ -929,10 +941,115 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew LogicalVariable var = unnestOp.getVariable(); if (var.equals(curVar)) { getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0, recordType, funcVarIndex, - childFuncExpr, recordVar, metaType, metaVar); + childFuncExpr, metaType, metaVar); } } } return Collections.emptyList(); } + + /** + * Finds the field name of each variable in the ASSIGN or UNNEST operators of the sub-tree. + */ + protected void fillFieldNamesInTheSubTree(OptimizableOperatorSubTree subTree) throws AlgebricksException { + LogicalVariable datasetMetaVar = null; + if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN + && subTree.getDataSourceType() != DataSourceType.INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP) { + List<LogicalVariable> datasetVars = subTree.getDataSourceVariables(); + if (subTree.getDataset().hasMetaPart()) { + datasetMetaVar = datasetVars.get(datasetVars.size() - 1); + } + } + for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.getAssignsAndUnnests() + .size(); assignOrUnnestIndex++) { + AbstractLogicalOperator op = subTree.getAssignsAndUnnests().get(assignOrUnnestIndex); + if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) { + AssignOperator assignOp = (AssignOperator) op; + List<LogicalVariable> varList = assignOp.getVariables(); + for (int varIndex = 0; varIndex < varList.size(); varIndex++) { + LogicalVariable var = varList.get(varIndex); + // funcVarIndex is not required. Thus, we set it to -1. + // optFuncExpr and parentFuncExpr are not required, too. Thus, we set them to null. + List<String> fieldName = getFieldNameFromSubTree(null, subTree, assignOrUnnestIndex, varIndex, + subTree.getRecordType(), -1, null, subTree.getMetaRecordType(), datasetMetaVar); + if (fieldName != null && !fieldName.isEmpty()) { + subTree.getVarsToFieldNameMap().put(var, fieldName); + } + } + } else if (op.getOperatorTag() == LogicalOperatorTag.UNNEST) { + UnnestOperator unnestOp = (UnnestOperator) op; + LogicalVariable var = unnestOp.getVariable(); + List<String> fieldName = null; + if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN) { + // funcVarIndex is not required. Thus, we set it to -1. + // optFuncExpr and parentFuncExpr are not required, too. Thus, we set them to null. + fieldName = getFieldNameFromSubTree(null, subTree, assignOrUnnestIndex, 0, subTree.getRecordType(), + -1, null, subTree.getMetaRecordType(), datasetMetaVar); + if (fieldName != null && !fieldName.isEmpty()) { + subTree.getVarsToFieldNameMap().put(var, fieldName); + } + } + } else { + // unnestmap or left-outer-unnestmap? + LeftOuterUnnestMapOperator leftOuterUnnestMapOp = null; + UnnestMapOperator unnestMapOp = null; + List<LogicalVariable> varList = null; + + if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) { + unnestMapOp = (UnnestMapOperator) op; + varList = unnestMapOp.getVariables(); + } else if (op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) { + leftOuterUnnestMapOp = (LeftOuterUnnestMapOperator) op; + varList = leftOuterUnnestMapOp.getVariables(); + } else { + continue; + } + + for (int varIndex = 0; varIndex < varList.size(); varIndex++) { + LogicalVariable var = varList.get(varIndex); + // funcVarIndex is not required. Thus, we set it to -1. + // optFuncExpr and parentFuncExpr are not required, too. Thus, we set them to null. + List<String> fieldName = getFieldNameFromSubTree(null, subTree, assignOrUnnestIndex, varIndex, + subTree.getRecordType(), -1, null, subTree.getMetaRecordType(), datasetMetaVar); + if (fieldName != null && !fieldName.isEmpty()) { + subTree.getVarsToFieldNameMap().put(var, fieldName); + } + } + } + } + + // DatasourceScan? + if (subTree.hasDataSourceScan()) { + List<LogicalVariable> primaryKeyVarList = new ArrayList<>(); + + if (subTree.getDataset().getDatasetType() == DatasetType.INTERNAL) { + subTree.getPrimaryKeyVars(null, primaryKeyVarList); + + Index primaryIndex = getPrimaryIndexFromDataSourceScanOp(subTree.getDataSourceRef().getValue()); + + for (int i = 0; i < primaryKeyVarList.size(); i++) { + subTree.getVarsToFieldNameMap().put(primaryKeyVarList.get(i), + primaryIndex.getKeyFieldNames().get(i)); + } + } + + } + + } + + /** + * Fetches the associated primary index from the given DATASOURCESCAN operator. + */ + protected Index getPrimaryIndexFromDataSourceScanOp(ILogicalOperator dataSourceScanOp) throws AlgebricksException { + if (dataSourceScanOp.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) { + return null; + } + Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo((DataSourceScanOperator) dataSourceScanOp); + String dataverseName = datasetInfo.first; + String datasetName = datasetInfo.second; + + Index idxUsedInUnnestMap = metadataProvider.getIndex(dataverseName, datasetName, datasetName); + return idxUsedInUnnestMap; + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java index 16ee6d1..c186687 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java @@ -28,6 +28,7 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.common.utils.Quadruple; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; @@ -52,7 +53,40 @@ public class AccessMethodAnalysisContext { // variables for resetting null placeholder for left-outer-join private Mutable<ILogicalOperator> lojGroupbyOpRef = null; - private ScalarFunctionCallExpression lojIsNullFuncInGroupBy = null; + private ScalarFunctionCallExpression lojIsMissingFuncInGroupBy = null; + + // For a secondary index, if we use only PK and secondary key field in a plan, it is an index-only plan. + // Contains information about index-only plan + // + // 1. isIndexOnlyPlan - index-only plan possible? + // This option is the primary option. If this is false, then regardless of the following variables, + // An index-only plan will not be constructed. If this is true, then we use the following variables to + // construct an index-only plan. + // + // 2. secondaryKeyFieldUsedAfterSelectOrJoinOp - secondary key field usage after the select or join operator? + // If the secondary key field is used after SELECT or JOIN operator (e.g., returning the field), + // then we need to keep secondary keys from the secondary index search. + // + // 3. requireVerificationAfterSIdxSearch - + // whether a verification (especially for R-Tree case) is required after the secondary index search? + // For an R-Tree index, if the given query shape is not RECTANGLE or POINT, + // we need to add the original SELECT operator to filter out the false positive results. + // (e.g., spatial-intersect($o.pointfield, create-circle(create-point(30.0,70.0), 5.0)) ) + // + // Also, for a B-Tree composite index, we need to apply SELECT operators in the right path + // to remove any false positive results from the secondary composite index search. + // + // Lastly, if there is an index-nested-loop-join and the join contains more conditions + // other than joining fields, then those conditions need to be applied to filter out + // false positive results in the right path (isntantTryLock success path). + // (e.g., where $a.authors /*+ indexnl */ = $b.authors and $a.id = $b.id) + // For more details, refer to AccessMethodUtils.createPrimaryIndexUnnestMap() method. + // + // 4. doesSIdxSearchCoverAllPredicates - can the given index cover all search predicates? + // In other words, all search predicates are about the given secondary index? + // + private Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo = + new Quadruple<>(false, false, false, false); public void addIndexExpr(Dataset dataset, Index index, Integer exprIndex, Integer varIndex) { List<Pair<Integer, Integer>> exprs = getIndexExprsFromIndexExprsAndVars(index); @@ -108,20 +142,36 @@ public class AccessMethodAnalysisContext { return lojGroupbyOpRef; } - public void setLOJIsNullFuncInGroupBy(ScalarFunctionCallExpression isNullFunc) { - lojIsNullFuncInGroupBy = isNullFunc; + public void setLOJIsMissingFuncInGroupBy(ScalarFunctionCallExpression isMissingFunc) { + lojIsMissingFuncInGroupBy = isMissingFunc; } - public ScalarFunctionCallExpression getLOJIsNullFuncInGroupBy() { - return lojIsNullFuncInGroupBy; + public ScalarFunctionCallExpression getLOJIsMissingFuncInGroupBy() { + return lojIsMissingFuncInGroupBy; } public Dataset getDatasetFromIndexDatasetMap(Index idx) { - return indexDatasetMap.get(idx); + return getIndexDatasetMap().get(idx); } public void putDatasetIntoIndexDatasetMap(Index idx, Dataset ds) { - indexDatasetMap.put(idx, ds); + getIndexDatasetMap().put(idx, ds); + } + + public void setIndexOnlyPlanInfo(Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo) { + this.indexOnlyPlanInfo = indexOnlyPlanInfo; + } + + public Quadruple<Boolean, Boolean, Boolean, Boolean> getIndexOnlyPlanInfo() { + return this.indexOnlyPlanInfo; + } + + public Map<Index, Dataset> getIndexDatasetMap() { + return indexDatasetMap; + } + + public void setIndexDatasetMap(Map<Index, Dataset> indexDatasetMap) { + this.indexDatasetMap = indexDatasetMap; } }