Xikui Wang has submitted this change and it was merged. Change subject: [ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser ......................................................................
[ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser - user model changes: no - storage format changes: no - interface changes: no Details: The generic record used in current http-feed returns length of "-1" which will cause the JSON parser to fail. For compatibility consideration, we should use CharArrayRecord instead. Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e Reviewed-on: https://asterix-gerrit.ics.uci.edu/3026 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Wail Alkowaileet <wael....@gmail.com> --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java 10 files changed, 156 insertions(+), 3 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Wail Alkowaileet: Looks good to me, approved diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp new file mode 100644 index 0000000..94a773c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use test; + +create type WorrisomeNotificationType as { + dataverseName: string, + channelName: string +}; +create type WorrisomeNotificationStoreType as { + id: uuid +}; +create feed WorrisomeTweetsFeed with { + "adapter-name" : "http_adapter", + "addresses" : "asterix_nc1:10001", + "address-type" : "NC", + "type-name" : "WorrisomeNotificationType", + "format" : "json" +}; + +create dataset WorrisomeNotifications(WorrisomeNotificationStoreType) primary key id autogenerated; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp new file mode 100644 index 0000000..cc180a3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +set `wait-for-completion-feed` "false"; + +connect feed WorrisomeTweetsFeed to dataset WorrisomeNotifications; +start feed WorrisomeTweetsFeed; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http new file mode 100644 index 0000000..2169163 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1:10001 / +--body={ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"}{ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp new file mode 100644 index 0000000..90c3f8e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +stop feed WorrisomeTweetsFeed; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp new file mode 100644 index 0000000..65d6faf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +select value count(t) from WorrisomeNotifications as t; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp new file mode 100644 index 0000000..c17ef97 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +drop dataverse test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index e52449c..72a96a6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -9984,6 +9984,11 @@ <output-dir compare="Text">http_feed</output-dir> </compilation-unit> </test-case> + <test-case FilePath="feeds"> + <compilation-unit name="http_feed_json"> + <output-dir compare="Text">http_feed_json</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="meta"> <test-case FilePath="meta"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java index 2979641..b466614 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java @@ -27,7 +27,7 @@ import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.api.IRecordReader; import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; -import org.apache.asterix.external.input.record.GenericRecord; +import org.apache.asterix.external.input.record.CharArrayRecord; import org.apache.asterix.external.util.FeedLogManager; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; @@ -49,7 +49,7 @@ private static final String DEFAULT_ENTRY_POINT = "/"; private static final int DEFAULT_QUEUE_SIZE = 128; private LinkedBlockingQueue<String> inputQ; - private GenericRecord<char[]> record; + private CharArrayRecord record; private boolean closed = false; private WebManager webManager; private HttpServer webServer; @@ -57,7 +57,7 @@ public HttpServerRecordReader(int port, String entryPoint, int queueSize, HttpServerConfig httpServerConfig) throws Exception { this.inputQ = new LinkedBlockingQueue<>(queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE); - this.record = new GenericRecord<>(); + this.record = new CharArrayRecord(0); webManager = new WebManager(); webServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), port, httpServerConfig); webServer.addServlet(new HttpFeedServlet(webServer.ctx(), -- To view, visit https://asterix-gerrit.ics.uci.edu/3026 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang <xkk...@gmail.com> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Till Westmann <ti...@apache.org> Gerrit-Reviewer: Wail Alkowaileet <wael....@gmail.com> Gerrit-Reviewer: Xikui Wang <xkk...@gmail.com>
