abdullah alamoudi has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1290
Change subject: Move Twitter Firehose Datasource to Test Source Folder ...................................................................... Move Twitter Firehose Datasource to Test Source Folder Change-Id: Iefe2130707012b8ce60f5dfac96635a1a515a076 --- M asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql M asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql M asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java R asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java R asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java R asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java R asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java M asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java 13 files changed, 25 insertions(+), 27 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/90/1290/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql index 70ea8d6..f3f8f7d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql @@ -54,12 +54,12 @@ primary key id; create feed SyntheticTweetFeed -using twitter_firehose( +using stream( ("duration"="5"), ("tps"="50"), ("type-name"="TweetMessageType"), ("format"="adm"), -("reader-stream"="twitter_firehose"), +("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"), ("tput-duration"="5"), ("dataverse-dataset"="feeds:SyntheticTweets"), ("mode"="controlled")); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql index 658487b..c339563 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql @@ -58,8 +58,9 @@ create index ngram_index on SyntheticTweets(message_text) type ngram(3); create feed SyntheticTweetFeed -using twitter_firehose +using stream (("duration"="5"), +("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"), ("tps"="50"), ("type-name"="TweetMessageType"), ("tput-duration"="5"), diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql index 6714850..59385c4 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql @@ -56,12 +56,12 @@ create index message_text on SyntheticTweets(message_text) type btree; create feed SyntheticTweetFeed -using twitter_firehose +using stream (("duration"="5"), ("tps"="50"), ("tput-duration"="5"), ("type-name"="TweetMessageType"), ("dataverse-dataset"="feeds:SyntheticTweets"), ("format"="adm"), -("reader-stream"="twitter_firehose"), +("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"), ("mode"="controlled")); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp index e3f3ae5..1f24192 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp @@ -45,12 +45,12 @@ create dataset SyntheticTweets(TweetMessageType) primary key id; -create primary feed SyntheticTweetFeed using twitter_firehose( +create primary feed SyntheticTweetFeed using stream( (`duration`=`5`), (`tps`=`50`), (`type-name`=`TweetMessageType`), (`format`=`adm`), -(`reader-stream`=`twitter_firehose`), +(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`), (`tput-duration`=`5`), (`dataverse-dataset`=`feeds:SyntheticTweets`), (`mode`=`controlled`)); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp index a98b745..6311b8b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp @@ -45,9 +45,10 @@ create dataset SyntheticTweets(TweetMessageType) primary key id; create index ngram_index on SyntheticTweets (message_text) type ngram (3); -create primary feed SyntheticTweetFeed using twitter_firehose ( +create primary feed SyntheticTweetFeed using stream ( (`duration`=`5`), (`tps`=`50`), +(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`), (`type-name`=`TweetMessageType`), (`tput-duration`=`5`), (`dataverse-dataset`=`feeds:SyntheticTweets`), diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp index 1b1c780..71a0ca2 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp @@ -45,7 +45,7 @@ } create dataset SyntheticTweets(TweetMessageType) primary key id; create index message_text on SyntheticTweets (message_text) type btree; -create primary feed SyntheticTweetFeed using twitter_firehose ((`duration`=`5`), +create primary feed SyntheticTweetFeed using stream ((`duration`=`5`), (`tps`=`50`),(`tput-duration`=`5`),(`type-name`=`TweetMessageType`), (`dataverse-dataset`=`feeds:SyntheticTweets`),(`format`=`adm`), -(`reader-stream`=`twitter_firehose`),(`mode`=`controlled`)); +(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`),(`mode`=`controlled`)); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java index ad11171..7ab6430 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java @@ -33,7 +33,6 @@ import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory; import org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory; import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory; -import org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamFactory; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataUtils; @@ -73,9 +72,6 @@ case ExternalDataConstants.STREAM_SOCKET_CLIENT: streamSourceFactory = new SocketServerInputStreamFactory(); break; - case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER: - streamSourceFactory = new TwitterFirehoseStreamFactory(); - break; default: try { streamSourceFactory = (IInputStreamFactory) Class.forName(streamSource).newInstance(); @@ -102,8 +98,6 @@ case ExternalDataConstants.READER_PUSH_TWITTER: case ExternalDataConstants.READER_PULL_TWITTER: return new TwitterRecordReaderFactory(); - case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER: - return new StreamRecordReaderFactory(new TwitterFirehoseStreamFactory()); case ExternalDataConstants.ALIAS_SOCKET_ADAPTER: case ExternalDataConstants.SOCKET: return new StreamRecordReaderFactory(new SocketServerInputStreamFactory()); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index 7d27e45..e795730 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -174,7 +174,6 @@ public static final String ALIAS_LOCALFS_PUSH_ADAPTER = "push_localfs"; public static final String ALIAS_HDFS_ADAPTER = "hdfs"; public static final String ALIAS_SOCKET_ADAPTER = "socket_adapter"; - public static final String ALIAS_TWITTER_FIREHOSE_ADAPTER = "twitter_firehose"; public static final String ALIAS_SOCKET_CLIENT_ADAPTER = "socket_client"; public static final String ALIAS_RSS_ADAPTER = "rss"; public static final String ALIAS_FILE_FEED_ADAPTER = "file_feed"; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java similarity index 99% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java rename to asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java index 59c9a0f..f61c7d0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +package org.apache.asterix.external.generator; import java.nio.CharBuffer; import java.util.ArrayList; @@ -25,6 +25,9 @@ import java.util.List; import java.util.Random; +import org.apache.asterix.external.util.Datatypes; +import org.apache.asterix.external.util.Datatypes.Tweet; + public class DataGenerator { private RandomDateGenerator randDateGen; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java similarity index 93% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java rename to asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java index b86c170..88ca4cc 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +package org.apache.asterix.external.generator; import java.io.IOException; import java.io.OutputStream; @@ -27,15 +27,15 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.asterix.external.util.DataGenerator.InitializationInfo; -import org.apache.asterix.external.util.DataGenerator.TweetMessage; -import org.apache.asterix.external.util.DataGenerator.TweetMessageIterator; +import org.apache.asterix.external.generator.DataGenerator.InitializationInfo; +import org.apache.asterix.external.generator.DataGenerator.TweetMessage; +import org.apache.asterix.external.generator.DataGenerator.TweetMessageIterator; public class TweetGenerator { private static final Logger LOGGER = Logger.getLogger(TweetGenerator.class.getName()); public static final String KEY_DURATION = "duration"; - public static final String KEY_TPS = "tps"; + public static final String KEY_TPS = "tps"; public static final String KEY_VERBOSE = "verbose"; public static final String KEY_FIELDS = "fields"; public static final int INFINITY = 0; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java similarity index 98% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java rename to asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java index e2afd7b..dd98691 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java @@ -29,7 +29,7 @@ import java.util.logging.Logger; import org.apache.asterix.external.api.AsterixInputStream; -import org.apache.asterix.external.util.TweetGenerator; +import org.apache.asterix.external.generator.TweetGenerator; import org.apache.hyracks.api.context.IHyracksTaskContext; public class TwitterFirehoseInputStream extends AsterixInputStream { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java similarity index 96% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java rename to asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java index abe67fd..77b8df1 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.input.stream.factory; +package org.apache.asterix.external.input.stream; import java.io.IOException; import java.util.ArrayList; @@ -67,7 +67,7 @@ count = Integer.parseInt(ingestionCardinalityParam); } - List<String> chosenLocations = new ArrayList<String>(); + List<String> chosenLocations = new ArrayList<>(); String[] availableLocations = locations != null ? locations : ClusterStateManager.INSTANCE.getParticipantNodes().toArray(new String[] {}); for (int i = 0, k = 0; i < count; i++, k = (k + 1) % availableLocations.length) { diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java index 93ffe59..763a4ae 100644 --- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java +++ b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Random; -import org.apache.asterix.external.util.DataGenerator; +import org.apache.asterix.external.generator.DataGenerator; public class DataGeneratorForSpatialIndexEvaluation { -- To view, visit https://asterix-gerrit.ics.uci.edu/1290 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iefe2130707012b8ce60f5dfac96635a1a515a076 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <bamou...@gmail.com>