abdullah alamoudi has submitted this change and it was merged. Change subject: ASTERIXDB-1333 Fix Feed Ingestion for Dataset with Filters ......................................................................
ASTERIXDB-1333 Fix Feed Ingestion for Dataset with Filters This change fixes a bug in the compilation of feed connection when the dataset has filters. The bug is caused by not setting the filter expressions during initial plan generation. Change-Id: Ic323c9a74504921b984f68790bff2d3a7140b85b Reviewed-on: https://asterix-gerrit.ics.uci.edu/716 Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Jenkins <[email protected]> --- M asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java A asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql A asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql A asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql A asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm M asterix-app/src/test/resources/runtimets/testsuite.xml 7 files changed, 140 insertions(+), 1 deletion(-) Approvals: Murtadha Hubail: Looks good to me, approved Jenkins: Verified diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 28de56b..4739b71 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -398,9 +398,10 @@ break; } case SUBSCRIBE_FEED: { - ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign)); + insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); leafOperator = new SinkOperator(); leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp)); break; diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql new file mode 100644 index 0000000..260dd1c --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test feed ingestion on a filtered dataset + * Expected Res : Success + * Date : 14th Mar 2016 + */ + +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type FacebookMessageType as closed { + message-id: int64, + author-id: int64, + in-response-to: int64?, + sender-location: point?, + message: string, + send-time: datetime +} + +create dataset FacebookMessages(FacebookMessageType) +primary key message-id with filter on send-time; + +create feed MessageFeed using push_localfs( +("path"="asterix_nc1://data/fbm-with-send-time.adm"), +("format"="adm"), +("type-name"="FacebookMessageType")); diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql new file mode 100644 index 0000000..36564a3 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test feed ingestion on a filtered dataset + * Expected Res : Success + * Date : 14th Mar 2016 + */ + +use dataverse test; +set wait-for-completion-feed "true"; +connect feed MessageFeed to dataset FacebookMessages; diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql new file mode 100644 index 0000000..daf4882 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test feed ingestion on a filtered dataset + * Expected Res : Success + * Date : 14th Mar 2016 + */ + +use dataverse test; + +for $m in dataset('FacebookMessages') +where $m.send-time > datetime("2012-08-20T10:10:00") +return $m; \ No newline at end of file diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql new file mode 100644 index 0000000..cc50807 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test feed ingestion on a filtered dataset + * Expected Res : Success + * Date : 14th Mar 2016 + */ + +drop dataverse test; diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm new file mode 100644 index 0000000..7a4df99 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm @@ -0,0 +1,7 @@ +{ "message-id": 11, "author-id": 1, "in-response-to": 1, "sender-location": point("38.97,77.49"), "message": " can't stand at&t its plan is terrible", "send-time": datetime("2012-11-20T10:10:00.000Z") } +{ "message-id": 12, "author-id": 10, "in-response-to": 6, "sender-location": point("42.26,77.76"), "message": " can't stand t-mobile its voicemail-service is OMG:(", "send-time": datetime("2012-12-20T10:10:00.000Z") } +{ "message-id": 14, "author-id": 9, "in-response-to": 12, "sender-location": point("41.33,85.28"), "message": " love at&t its 3G is good:)", "send-time": datetime("2013-09-20T10:10:00.000Z") } +{ "message-id": 13, "author-id": 10, "in-response-to": 4, "sender-location": point("42.77,78.92"), "message": " dislike iphone the voice-command is bad:(", "send-time": datetime("2013-08-20T10:10:00.000Z") } +{ "message-id": 15, "author-id": 7, "in-response-to": 11, "sender-location": point("44.47,67.11"), "message": " like iphone the voicemail-service is awesome", "send-time": datetime("2014-01-20T10:10:00.000Z") } +{ "message-id": 9, "author-id": 3, "in-response-to": 12, "sender-location": point("34.45,96.48"), "message": " love verizon its wireless is good", "send-time": datetime("2012-09-20T10:10:00.000Z") } +{ "message-id": 10, "author-id": 1, "in-response-to": 12, "sender-location": point("42.5,70.01"), "message": " can't stand motorola the touch-screen is terrible", "send-time": datetime("2012-10-20T10:10:00.000Z") } diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml index b46b75b..d50a0a6 100644 --- a/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -36,6 +36,11 @@ </compilation-unit> </test-case> --> <test-case FilePath="feeds"> + <compilation-unit name="feed-with-filtered-dataset"> + <output-dir compare="Text">feed-with-filtered-dataset</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="feeds"> <compilation-unit name="feed-push-socket"> <output-dir compare="Text">feed-push-socket</output-dir> </compilation-unit> -- To view, visit https://asterix-gerrit.ics.uci.edu/716 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic323c9a74504921b984f68790bff2d3a7140b85b Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
