Repository: asterixdb Updated Branches: refs/heads/master 0dec33a96 -> 19e9d8290
[ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser - user model changes: no - storage format changes: no - interface changes: no Details: The generic record used in current http-feed returns length of "-1" which will cause the JSON parser to fail. For compatibility consideration, we should use CharArrayRecord instead. Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e Reviewed-on: https://asterix-gerrit.ics.uci.edu/3026 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Wail Alkowaileet <wael....@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/19e9d829 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/19e9d829 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/19e9d829 Branch: refs/heads/master Commit: 19e9d8290d64d4a29c1de78df3eaf8ee32553eed Parents: 0dec33a Author: Xikui Wang <xkk...@gmail.com> Authored: Thu Nov 15 13:55:02 2018 -0800 Committer: Xikui Wang <xkk...@gmail.com> Committed: Thu Nov 15 22:08:19 2018 -0800 ---------------------------------------------------------------------- .../http_feed_json/http_feed_json.1.ddl.sqlpp | 38 ++++++++++++++++++++ .../http_feed_json.2.update.sqlpp | 24 +++++++++++++ .../http_feed_json/http_feed_json.3.post.http | 20 +++++++++++ .../http_feed_json.4.update.sqlpp | 21 +++++++++++ .../http_feed_json/http_feed_json.5.query.sqlpp | 22 ++++++++++++ .../http_feed_json/http_feed_json.6.ddl.sqlpp | 21 +++++++++++ .../feeds/http_feed_json/http_feed.1.adm | 1 + .../feeds/http_feed_json/http_feed.2.adm | 1 + .../resources/runtimets/testsuite_sqlpp.xml | 5 +++ .../reader/http/HttpServerRecordReader.java | 6 ++-- 10 files changed, 156 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp new file mode 100644 index 0000000..94a773c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use test; + +create type WorrisomeNotificationType as { + dataverseName: string, + channelName: string +}; +create type WorrisomeNotificationStoreType as { + id: uuid +}; +create feed WorrisomeTweetsFeed with { + "adapter-name" : "http_adapter", + "addresses" : "asterix_nc1:10001", + "address-type" : "NC", + "type-name" : "WorrisomeNotificationType", + "format" : "json" +}; + +create dataset WorrisomeNotifications(WorrisomeNotificationStoreType) primary key id autogenerated; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp new file mode 100644 index 0000000..cc180a3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +set `wait-for-completion-feed` "false"; + +connect feed WorrisomeTweetsFeed to dataset WorrisomeNotifications; +start feed WorrisomeTweetsFeed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http new file mode 100644 index 0000000..2169163 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1:10001 / +--body={ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"}{ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp new file mode 100644 index 0000000..90c3f8e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +stop feed WorrisomeTweetsFeed; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp new file mode 100644 index 0000000..65d6faf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +select value count(t) from WorrisomeNotifications as t; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp new file mode 100644 index 0000000..c17ef97 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +drop dataverse test; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index e52449c..72a96a6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -9984,6 +9984,11 @@ <output-dir compare="Text">http_feed</output-dir> </compilation-unit> </test-case> + <test-case FilePath="feeds"> + <compilation-unit name="http_feed_json"> + <output-dir compare="Text">http_feed_json</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="meta"> <test-case FilePath="meta"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19e9d829/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java index 2979641..b466614 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java @@ -27,7 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.api.IRecordReader; import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; -import org.apache.asterix.external.input.record.GenericRecord; +import org.apache.asterix.external.input.record.CharArrayRecord; import org.apache.asterix.external.util.FeedLogManager; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; @@ -49,7 +49,7 @@ public class HttpServerRecordReader implements IRecordReader<char[]> { private static final String DEFAULT_ENTRY_POINT = "/"; private static final int DEFAULT_QUEUE_SIZE = 128; private LinkedBlockingQueue<String> inputQ; - private GenericRecord<char[]> record; + private CharArrayRecord record; private boolean closed = false; private WebManager webManager; private HttpServer webServer; @@ -57,7 +57,7 @@ public class HttpServerRecordReader implements IRecordReader<char[]> { public HttpServerRecordReader(int port, String entryPoint, int queueSize, HttpServerConfig httpServerConfig) throws Exception { this.inputQ = new LinkedBlockingQueue<>(queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE); - this.record = new GenericRecord<>(); + this.record = new CharArrayRecord(0); webManager = new WebManager(); webServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), port, httpServerConfig); webServer.addServlet(new HttpFeedServlet(webServer.ctx(),