abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/737
Change subject: Fix Asterix Installer Test Cases
......................................................................
Fix Asterix Installer Test Cases
Many tests started failing in installer after being
disabled for a long time. This change fixes the following:
1. asterix-installer: external library tests.
2. asterix-installer: recovery: query-after-restart.
Change-Id: Iefad5ac3ff754693025312f1e2fa01b08832514c
---
M .gitignore
M asterix-app/pom.xml
M
asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
M
asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser/classad-parser.2.lib.aql
M
asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser2/classad-parser2.2.lib.aql
M
asterix-app/src/test/resources/runtimets/queries/external-library/getCapital/getCapital.2.lib.aql
M
asterix-app/src/test/resources/runtimets/queries/external-library/typed_adapter/typed_adapter.2.lib.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
C
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
M
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-parser/feed-with-external-parser.2.lib.aql
M
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
M
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
A
asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/LookupAdapter.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
M
asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
M asterix-installer/pom.xml
M
asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
M
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
A
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
R
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
M
asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
M
asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
M
asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
M
asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
M
asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
39 files changed, 400 insertions(+), 97 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/37/737/1
diff --git a/.gitignore b/.gitignore
index d802def..1b129d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@
asterix-app/runtime_parserts/
asterix-installer/ittest/
asterix-installer/repliationtest/
+asterix-installer/src/test/resources/clusterts/asterix-installer-0.8.9-SNAPSHOT-binary-assembly/
build
asterix_logs
bin/
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 00e924f..c1bfa22 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -64,48 +64,48 @@
</executions>
</plugin>
<plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-external-library</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
-
<outputDirectory>src/test/resources/externallib</outputDirectory>
- <overwrite>true</overwrite>
- <resources>
- <resource>
- <directory>../asterix-external-data/target</directory>
- <includes>
- <include>testlib-zip-binary-assembly.zip</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- <execution>
- <id>copy-beer-csv</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>data/csv</outputDirectory>
- <overwrite>true</overwrite>
- <resources>
- <resource>
-
<directory>../asterix-external-data/src/test/resources</directory>
- <includes>
- <include>beer.csv</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
-</plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-external-library</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>target/resources/externallib</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+
<directory>../asterix-external-data/target</directory>
+ <includes>
+
<include>testlib-zip-binary-assembly.zip</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-beer-csv</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/data/csv</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+
<directory>../asterix-external-data/src/test/resources</directory>
+ <includes>
+ <include>beer.csv</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<dependencies>
diff --git
a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 6b54fbc..b8dcefb 100644
---
a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++
b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -26,7 +26,6 @@
import java.io.InputStreamReader;
import java.rmi.RemoteException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@@ -862,7 +861,7 @@
if (subType.isOpen()) {
isOpen = true;
break;
- };
+ } ;
}
}
if (fieldExpr.second == null) {
@@ -1949,7 +1948,7 @@
private JobSpecification rewriteCompileQuery(AqlMetadataProvider
metadataProvider, Query query,
ICompiledDmlStatement stmt)
- throws AsterixException, RemoteException,
AlgebricksException, JSONException, ACIDException {
+ throws AsterixException, RemoteException, AlgebricksException,
JSONException, ACIDException {
//Query Rewriting (happens under the same ongoing metadata transaction)
Pair<Query, Integer> reWrittenQuery =
apiFramework.reWriteQuery(declaredFunctions, metadataProvider, query,
@@ -2254,7 +2253,7 @@
*/
private Triple<FeedConnectionRequest, Boolean, List<IFeedJoint>>
getFeedConnectionRequest(String dataverse,
Feed feed, String dataset, FeedPolicyEntity feedPolicy,
MetadataTransactionContext mdTxnCtx)
- throws MetadataException {
+ throws MetadataException {
IFeedJoint sourceFeedJoint = null;
FeedConnectionRequest request = null;
List<String> functionsToApply = new ArrayList<String>();
@@ -2924,7 +2923,7 @@
private void prepareRunExternalRuntime(AqlMetadataProvider
metadataProvider, IHyracksClientConnection hcc,
RunStatement pregelixStmt, String dataverseNameFrom, String
dataverseNameTo, String datasetNameFrom,
String datasetNameTo, MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException, AsterixException, Exception {
+ throws AlgebricksException, AsterixException, Exception {
//Validates the source/sink dataverses and datasets.
Dataset fromDataset = metadataProvider.findDataset(dataverseNameFrom,
datasetNameFrom);
if (fromDataset == null) {
diff --git
a/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser/classad-parser.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser/classad-parser.2.lib.aql
index 0290611..9a7e516 100644
---
a/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser/classad-parser.2.lib.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser/classad-parser.2.lib.aql
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib
src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser2/classad-parser2.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser2/classad-parser2.2.lib.aql
index 0290611..9a7e516 100644
---
a/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser2/classad-parser2.2.lib.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/external-library/classad-parser2/classad-parser2.2.lib.aql
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib
src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/external-library/getCapital/getCapital.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/external-library/getCapital/getCapital.2.lib.aql
index 0290611..9a7e516 100644
---
a/asterix-app/src/test/resources/runtimets/queries/external-library/getCapital/getCapital.2.lib.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/external-library/getCapital/getCapital.2.lib.aql
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib
src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/external-library/typed_adapter/typed_adapter.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/external-library/typed_adapter/typed_adapter.2.lib.aql
index 0290611..9a7e516 100644
---
a/asterix-app/src/test/resources/runtimets/queries/external-library/typed_adapter/typed_adapter.2.lib.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/external-library/typed_adapter/typed_adapter.2.lib.aql
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib
src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
new file mode 100644
index 0000000..5c30ac5
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local
fs.
+ Associate with the feed an external user-defined function.
The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+
+drop dataverse externallibtest if exists;
+create dataverse externallibtest;
+use dataverse externallibtest;
+
+create type TweetInputType as closed {
+ id: string,
+ username : string,
+ location : string,
+ text : string,
+ timestamp : string
+}
+
+create type TweetOutputType as closed {
+ id: string,
+ username : string,
+ location : string,
+ text : string,
+ timestamp : string,
+ topics : {{string}}
+}
+
+create feed TweetFeed
+using localfs
+(("type-name"="TweetInputType"),
+("path"="asterix_nc1://data/twitter/obamatweets.adm"),
+("format"="adm"))
+apply function testlib#parseTweet;
+
+create dataset TweetsFeedIngest(TweetOutputType)
+primary key id;
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
new file mode 100644
index 0000000..e4c5bd8
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
new file mode 100644
index 0000000..d5a6f58
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed dataset that uses the feed simulator adapter.
+ The feed simulator simulates feed from a file in the local
fs.
+ Associate with the feed an external user-defined function.
The UDF
+ finds topics in each tweet. A topic is identified by a #.
+ Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+set wait-for-completion-feed "true";
+
+connect feed TweetFeed to dataset TweetsFeedIngest;
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
new file mode 100644
index 0000000..18bbbbc
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a socket feed with a client that pushes
+ * 10 records. The feed is connected to a dataset that is then
+ * queried for the data.
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+10000
\ No newline at end of file
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
similarity index 100%
copy from
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
copy to
asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
new file mode 100644
index 0000000..16ada23
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed which uses an external parser to parse data
from files
+ * The files have duplicates and long records
+ * Expected Res : Success
+ * Date : Feb, 09, 2016
+ */
+drop dataverse externallibtest if exists;
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
new file mode 100644
index 0000000..86af80f
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+uninstall externallibtest testlib
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-parser/feed-with-external-parser.2.lib.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-parser/feed-with-external-parser.2.lib.aql
index 0290611..9a7e516 100644
---
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-parser/feed-with-external-parser.2.lib.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-parser/feed-with-external-parser.2.lib.aql
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib
src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
+install externallibtest testlib
target/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file
diff --git
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
index ec4a712..de9446a 100644
---
a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
@@ -46,7 +46,7 @@
("parser"="record-with-metadata"),
("type-name"="DocumentType"),
("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
("format"="csv"),
("delimiter"=","),
("record-format"="adm"),
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
index 718d2a6..26d4b09 100644
---
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
@@ -41,7 +41,7 @@
("parser"="record-with-metadata"),
("type-name"="DocumentType"),
("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
("format"="csv"),
("delimiter"=","),
("record-format"="adm"),
diff --git
a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
new file mode 100644
index 0000000..1291213
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
@@ -0,0 +1,12 @@
+{ "id": "nc1:1", "username": "BronsonMike", "location": "", "text":
"@GottaLaff @reutersus Christie and obama just foul weather friends",
"timestamp": "Thu Dec 06 16:53:06 PST 2012", "topics": {{ }} }
+{ "id": "nc1:100", "username": "KidrauhlProuds", "location": "", "text": "RT
@01Direclieber: A filha do Michael Jackson uma Belieber,a filha do Eminem e
uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e
Belieber", "timestamp": "Thu Dec 06 16:53:16 PST 2012", "topics": {{ }} }
+{ "id": "nc1:102", "username": "jaysauce82", "location": "", "text": "Not
voting for President Obama #BadDecision", "timestamp": "Thu Dec 06 16:53:16 PST
2012", "topics": {{ "#BadDecision" }} }
+{ "id": "nc1:104", "username": "princeofsupras", "location": "", "text": "RT
@01Direclieber: A filha do Michael Jackson e uma Belieber,a filha do Eminem e
uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e
Belieber", "timestamp": "Thu Dec 06 16:53:15 PST 2012", "topics": {{ }} }
+{ "id": "nc1:106", "username": "GulfDogs", "location": "", "text": "Obama
Admin Knew Libyan Terrorists Had US-Provided Weaponsteaparty #tcot #ccot
#NewGuards #BreitbartArmy #patriotwttp://t.co/vJxzrQUE", "timestamp": "Thu Dec
06 16:53:14 PST 2012", "topics": {{ "#tcot", "#ccot", "#NewGuards",
"#BreitbartArmy", "#patriotwttp://t.co/vJxzrQUE" }} }
+{ "id": "nc1:108", "username": "Laugzpz", "location": "", "text":
"@AlfredoJalife Maestro Obama se hace de la vista gorda, es un acuerdo de
siempre creo yo.", "timestamp": "Thu Dec 06 16:53:14 PST 2012", "topics": {{
}} }
+{ "id": "nc1:11", "username": "magarika", "location": "", "text": "RT
@ken24xavier: Obama tells SOROS - our plan is ALMOST finished
http://t.co/WvzK0GtU";, "timestamp": "Thu Dec 06 16:53:05 PST 2012", "topics":
{{ }} }
+{ "id": "nc1:111", "username": "ToucanMall", "location": "", "text": "RT
@WorldWar3Watch: Michelle Obama Gets More Grammy Nominations Than Justin ...
#Obama #WW3 http://t.co/0Wv2GKij";, "timestamp": "Thu Dec 06 16:53:13 PST 2012",
"topics": {{ "#Obama", "#WW3" }} }
+{ "id": "nc1:113", "username": "ToucanMall", "location": "", "text": "RT
@ObamaPalooza: Tiffany Shared What $2,000 Meant to Her ... and the President
Stopped by to Talk About It http://t.co/sgT7lsNV #Obama", "timestamp": "Thu Dec
06 16:53:12 PST 2012", "topics": {{ "#Obama" }} }
+{ "id": "nc1:115", "username": "thewildpitch", "location": "", "text": "RT
@RevkahJC: Dennis Miller: Obama Should Just Say He Wants To Tax Successful
People http://t.co/Ihlemy9Y";, "timestamp": "Thu Dec 06 16:53:11 PST 2012",
"topics": {{ }} }
+{ "id": "nc1:117", "username": "Rnugent24", "location": "", "text": "RT
@ConservativeQuo: unemployment is above 8% again. I wonder how long it will
take for Obama to start blaming Bush? 3-2-1 #tcot #antiobama", "timestamp":
"Thu Dec 06 16:53:10 PST 2012", "topics": {{ "#tcot", "#antiobama" }} }
+{ "id": "nc1:119", "username": "ToucanMall", "location": "", "text": "RT
@Newitrsdotcom: I hope #Obama will win re-election... Other four years without
meaningless #wars", "timestamp": "Thu Dec 06 16:53:09 PST 2012", "topics": {{
"#Obama", "#wars" }} }
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml
b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 9dcd883..8fabe54 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -30,6 +30,11 @@
QueryFileExtension=".aql">
<test-group name="feeds">
<test-case FilePath="feeds">
+ <compilation-unit name="feed-with-external-function">
+ <output-dir
compare="Text">feed-with-external-function</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
<compilation-unit name="change-feed-with-meta-pk-in-meta">
<output-dir
compare="Text">change-feed-with-meta-pk-in-meta</output-dir>
</compilation-unit>
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
index 99654d0..915c343 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
@@ -28,9 +28,11 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.log4j.Logger;
public class RecordDataFlowController<T> extends AbstractDataFlowController {
+ private static final Logger LOGGER =
Logger.getLogger(RecordDataFlowController.class.getName());
protected final IRecordDataParser<T> dataParser;
protected final IRecordReader<? extends T> recordReader;
protected final int numOfTupleFields;
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/LookupAdapter.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/LookupAdapter.java
index 69cd063..40051ae 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/LookupAdapter.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/LookupAdapter.java
@@ -57,7 +57,7 @@
public LookupAdapter(IRecordDataParser<T> dataParser,
ILookupRecordReader<? extends T> recordReader,
RecordDescriptor inRecDesc, RecordIdReader ridReader, boolean
propagateInput, boolean retainNull,
INullWriterFactory iNullWriterFactory, IHyracksTaskContext ctx,
IFrameWriter writer)
- throws HyracksDataException {
+ throws HyracksDataException {
this.dataParser = dataParser;
this.recordReader = recordReader;
this.propagateInput = propagateInput;
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
index b36fb6a..67e6295 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
@@ -26,13 +26,13 @@
import org.apache.asterix.external.feed.api.IExceptionHandler;
import org.apache.asterix.external.feed.api.IFeedMetricCollector;
-import org.apache.asterix.external.feed.api.IFrameEventCallback;
-import org.apache.asterix.external.feed.api.IFramePostProcessor;
-import org.apache.asterix.external.feed.api.IFramePreprocessor;
import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
import org.apache.asterix.external.feed.dataflow.DataBucket;
import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
@@ -170,7 +170,7 @@
this.timer.scheduleAtFixedRate(monitorInputQueueLengthTask, 0,
INPUT_QUEUE_MEASURE_FREQUENCY);
}
- if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+ if (reportInflowRate || reportOutflowRate) {
this.logInflowOutflowRateTask = new LogInputOutputRateTask(this,
logInflowOutflowRate, reportInflowRate,
reportOutflowRate);
this.timer.scheduleAtFixedRate(logInflowOutflowRateTask, 0,
LOG_INPUT_OUTPUT_RATE_FREQUENCY);
@@ -188,7 +188,7 @@
if (processingRateTask != null) {
processingRateTask.cancel();
}
- if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+ if (reportInflowRate || reportOutflowRate) {
metricCollector.removeReportSender(inflowReportSenderId);
metricCollector.removeReportSender(outflowReportSenderId);
logInflowOutflowRateTask.cancel();
@@ -244,7 +244,7 @@
}
public void sendReport(ByteBuffer frame) {
- if ((logInflowOutflowRate || reportInflowRate) &&
!(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
+ if ((reportInflowRate) &&
!(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
|| inputHandler.getMode().equals(Mode.PROCESS_SPILL))) {
inflowFta.reset(frame);
metricCollector.sendReport(inflowReportSenderId,
inflowFta.getTupleCount());
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
index 86c6bca..b8629c1 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBufferTimerTasks.java
@@ -31,10 +31,10 @@
import org.apache.asterix.common.config.AsterixFeedProperties;
import org.apache.asterix.external.feed.api.IFeedManager;
import org.apache.asterix.external.feed.api.IFeedMessageService;
-import org.apache.asterix.external.feed.api.IFrameEventCallback;
import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
import org.apache.asterix.external.feed.management.FeedConnectionId;
@@ -141,7 +141,8 @@
private final IFeedMessageService messageService;
private final FeedReportMessage message;
- public LogInputOutputRateTask(MonitoredBuffer mBuffer, boolean log,
boolean reportInflow, boolean reportOutflow) {
+ public LogInputOutputRateTask(MonitoredBuffer mBuffer, boolean log,
boolean reportInflow,
+ boolean reportOutflow) {
this.mBuffer = mBuffer;
this.log = log;
this.reportInflow = reportInflow;
@@ -284,8 +285,8 @@
}
} else {
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inflow Rate (" + inflowRate + ")
exceeds Processing Rate" + " (" + procRate
- + ")");
+ LOGGER.info(
+ "Inflow Rate (" + inflowRate + ") exceeds
Processing Rate" + " (" + procRate + ")");
}
}
}
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java
index 2cab3a7..ed563cc 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java
@@ -64,7 +64,7 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext
ctx,
final IRecordDescriptorProvider recordDescProvider, final int
partition, int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
// Create a file index accessor to be used for files lookup operations
// Note that all file index accessors will use partition 0 since we
only have 1 files index per NC
final ExternalFileIndexAccessor snapshotAccessor = new
ExternalFileIndexAccessor(
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
index 66b520f..210a94d 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
@@ -95,9 +95,14 @@
private ByteBuffer message =
ByteBuffer.allocate(MessagingFrameTupleAppender.MAX_MESSAGE_SIZE);
+ private final FeedMetaOperatorDescriptor opDesc;
+
+ private final IRecordDescriptorProvider recordDescProvider;
+
public FeedMetaComputeNodePushable(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider,
int partition, int nPartitions, IOperatorDescriptor coreOperator,
FeedConnectionId feedConnectionId,
- Map<String, String> feedPolicyProperties, String operationId)
throws HyracksDataException {
+ Map<String, String> feedPolicyProperties, String operationId,
+ FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) throws
HyracksDataException {
this.ctx = ctx;
this.coreOperator =
(AbstractUnaryInputUnaryOutputOperatorNodePushable) ((IActivity) coreOperator)
.createPushRuntime(ctx, recordDescProvider, partition,
nPartitions);
@@ -108,6 +113,8 @@
this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext)
ctx.getJobletContext().getApplicationContext()
.getApplicationObject()).getFeedManager();
ctx.setSharedObject(message);
+ this.opDesc = feedMetaOperatorDescriptor;
+ this.recordDescProvider = recordDescProvider;
}
@Override
@@ -129,7 +136,7 @@
}
private void initializeNewFeedRuntime(FeedRuntimeId runtimeId) throws
Exception {
- this.fta = new FrameTupleAccessor(recordDesc);
+ this.fta = new
FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(),
0));
this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId,
runtimeId, coreOperator,
policyEnforcer.getFeedPolicyAccessor(),
policyEnforcer.getFeedPolicyAccessor().bufferingEnabled(), fta,
recordDesc, feedManager, nPartitions);
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
index 354636d..6591795 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
@@ -90,12 +90,12 @@
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext
ctx,
final IRecordDescriptorProvider recordDescProvider, final int
partition, final int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
IOperatorNodePushable nodePushable = null;
switch (runtimeType) {
case COMPUTE:
nodePushable = new FeedMetaComputeNodePushable(ctx,
recordDescProvider, partition, nPartitions,
- coreOperator, feedConnectionId, feedPolicyProperties,
operandId);
+ coreOperator, feedConnectionId, feedPolicyProperties,
operandId, this);
break;
case STORE:
nodePushable = new FeedMetaStoreNodePushable(ctx,
recordDescProvider, partition, nPartitions,
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index c9fc5fc..5ab41af 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -45,16 +45,17 @@
return dataverseName + File.separator + feedName;
}
- public static FileSplit[] splitsForAdapter(String dataverseName, String
feedName, String nodeName, int partition) {
+ public static FileSplit splitsForAdapter(String dataverseName, String
feedName, int partition,
+ ClusterPartition[] nodePartitions) {
File relPathFile = new File(prepareDataverseFeedName(dataverseName,
feedName));
String storageDirName =
AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
- ClusterPartition nodePartition =
AsterixClusterProperties.INSTANCE.getNodePartitions(nodeName)[0];
+ ClusterPartition nodePartition = nodePartitions[0];
String storagePartitionPath =
StoragePathUtil.prepareStoragePartitionPath(storageDirName,
nodePartition.getPartitionId());
// format: 'storage dir name'/partition_#/dataverse/feed/adapter_#
File f = new File(storagePartitionPath + File.separator + relPathFile
+ File.separator
+ StoragePathUtil.ADAPTER_INSTANCE_PREFIX + partition);
- return new FileSplit[] {
StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f) };
+ return StoragePathUtil.getFileSplitForClusterPartition(nodePartition,
f);
}
public static FileSplit[] splitsForAdapter(String dataverseName, String
feedName,
@@ -93,6 +94,12 @@
feedLogFileSplits[partition].getIODeviceId(),
ctx.getIOManager()).getFile());
}
+ public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx,
FileSplit feedLogFileSplit)
+ throws HyracksDataException {
+ return new
FeedLogManager(FeedUtils.getAbsoluteFileRef(feedLogFileSplit.getLocalFile().getFile().getPath(),
+ feedLogFileSplit.getIODeviceId(),
ctx.getIOManager()).getFile());
+ }
+
public static void processFeedMessage(ByteBuffer input, ByteBuffer
message, FrameTupleAccessor fta) {
// read the message and reduce the number of tuples
fta.reset(input);
diff --git
a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index 9082a72..7da6389 100644
---
a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++
b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -22,6 +22,8 @@
import java.io.InputStream;
import java.util.Map;
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.config.IAsterixPropertiesProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
@@ -36,6 +38,7 @@
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.std.file.ITupleParser;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -74,12 +77,19 @@
ADMDataParser parser;
ITupleForwarder forwarder;
ArrayTupleBuilder tb;
+ IAsterixPropertiesProvider propertiesProvider =
(IAsterixPropertiesProvider) ((NodeControllerService) ctx
+
.getJobletContext().getApplicationContext().getControllerService()).getApplicationContext()
+ .getApplicationObject();
+ ClusterPartition[] nodePartitions =
propertiesProvider.getMetadataProperties().getNodePartitions()
+ .get(nodeId);
try {
parser = new ADMDataParser(outputType, true);
- forwarder = DataflowUtils.getTupleForwarder(configuration,
- FeedUtils.getFeedLogManager(ctx, partition,
-
FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
-
ExternalDataUtils.getFeedName(configuration), nodeId, partition)));
+ forwarder = DataflowUtils
+ .getTupleForwarder(configuration,
+ FeedUtils.getFeedLogManager(ctx,
+
FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
+
ExternalDataUtils.getFeedName(configuration), partition,
+ nodePartitions)));
tb = new ArrayTupleBuilder(1);
} catch (Exception e) {
throw new HyracksDataException(e);
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 712d350..832d0a9 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -138,8 +138,57 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-external-library</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>target/resources/externallib</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+
<directory>../asterix-external-data/target</directory>
+ <includes>
+
<include>testlib-zip-binary-assembly.zip</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-test-data</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/data</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+
<directory>../asterix-external-data/src/test/resources</directory>
+ <includes>
+ <include>beer.csv</include>
+ </includes>
+ </resource>
+ <resource>
+
<directory>../asterix-app/data/external-parser</directory>
+ <includes>
+ <include>jobads.new</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
- </build>
+ </build>
<profiles>
diff --git
a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
index 1da01c3..506e31f 100644
---
a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
+++
b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -35,8 +35,8 @@
private static final String LIBRARY_DATAVERSE = "externallibtest";
private static final String PATH_BASE =
"src/test/resources/integrationts/library";
private static final String PATH_ACTUAL = "ittest/";
- private static final String LIBRARY_PATH = "asterix-external-data" +
File.separator + "target" + File.separator
- + "testlib-zip-binary-assembly.zip";
+ private static final String LIBRARY_PATH = "target" + File.separator +
"resources" + File.separator + "externallib"
+ + File.separator + "testlib-zip-binary-assembly.zip";
private static final Logger LOGGER =
Logger.getLogger(AsterixExternalLibraryIT.class.getName());
private static List<TestCaseContext> testCaseCollection;
private final TestExecutor testExecutor = new TestExecutor();
@@ -46,8 +46,8 @@
try {
AsterixInstallerIntegrationUtil.init();
File asterixInstallerProjectDir = new
File(System.getProperty("user.dir"));
- String asterixExternalLibraryPath =
asterixInstallerProjectDir.getParentFile().getAbsolutePath()
- + File.separator + LIBRARY_PATH;
+ String asterixExternalLibraryPath =
asterixInstallerProjectDir.getAbsolutePath() + File.separator
+ + LIBRARY_PATH;
LOGGER.info("Installing library :" + LIBRARY_NAME + " located at "
+ asterixExternalLibraryPath
+ " in dataverse " + LIBRARY_DATAVERSE);
AsterixInstallerIntegrationUtil.installLibrary(LIBRARY_NAME,
LIBRARY_DATAVERSE, asterixExternalLibraryPath);
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
index f9a6eda..dbe3cfa 100644
---
a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
+++
b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
@@ -48,8 +48,7 @@
using localfs
(("type-name"="TweetInputType"),
("path"="asterix_nc1://../../../../../../asterix-app/data/twitter/obamatweets.adm"),
-("format"="adm"),
-("tuple-interval"="10"))
+("format"="adm"))
apply function testlib#parseTweet;
create dataset TweetsFeedIngest(TweetOutputType)
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
new file mode 100644
index 0000000..18bbbbc
--- /dev/null
+++
b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a socket feed with a client that pushes
+ * 10 records. The feed is connected to a dataset that is then
+ * queried for the data.
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+10000
\ No newline at end of file
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
similarity index 100%
rename from
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
rename to
asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
index d523247..09db37c 100644
---
a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
+++
b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
@@ -24,10 +24,12 @@
use dataverse externallibtest;
create type Classad as open {
- GlobalJobId: int64
+ GlobalJobId: string
}
create external dataset Condor(Classad) using localfs(
-("path"="asterix_nc1://data/external-parser/jobads.new"),
-("format"="adm"),
+("path"="asterix_nc1://../../../../data/jobads.new"),
+("format"="semi-structured"),
+("record-start"="["),
+("record-end"="]"),
("parser"="testlib#org.apache.asterix.external.library.ClassAdParserFactory"));
diff --git
a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
index ca2d832..26f980f 100644
---
a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
+++
b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
@@ -24,4 +24,5 @@
use dataverse externallibtest;
for $x in dataset Condor
+order by $x.GlobalJobId
return $x;
diff --git
a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
index d4e5619..2c4af14 100644
---
a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
+++
b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
@@ -1,2 +1,2 @@
-{ "DataverseName": "Metadata", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr
25 11:17:56 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "externallibtest", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr
25 11:18:12 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr
25 11:17:56 PDT 2013", "PendingOp": 0i32 }
+{ "DataverseName": "externallibtest", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr
25 11:18:12 PDT 2013", "PendingOp": 0i32 }
diff --git
a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
index 36b836c..3ee4a1e 100644
---
a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
+++
b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
@@ -1,4 +1,2 @@
-[ { "id": 3, "name": "Samuel", "age": 22 }
-, { "id": 10, "name": "David", "age": 22 }
- ]
-
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 78c6587..8561bf5 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -214,12 +214,15 @@
Map<ConnectorDescriptorId, ConnectorDescriptorId> connectorMapping =
new HashMap<ConnectorDescriptorId, ConnectorDescriptorId>();
for (Entry<ConnectorDescriptorId, IConnectorDescriptor> entry :
spec.getConnectorMap().entrySet()) {
IConnectorDescriptor connDesc = entry.getValue();
+ ConnectorDescriptorId newConnId;
if (connDesc instanceof MToNPartitioningConnectorDescriptor) {
MToNPartitioningConnectorDescriptor m2nConn =
(MToNPartitioningConnectorDescriptor) connDesc;
connDesc = new
MToNPartitioningWithMessageConnectorDescriptor(altered,
m2nConn.getTuplePartitionComputerFactory());
+ newConnId = connDesc.getConnectorId();
+ } else {
+ newConnId = altered.createConnectorDescriptor(connDesc);
}
- ConnectorDescriptorId newConnId =
altered.createConnectorDescriptor(connDesc);
connectorMapping.put(entry.getKey(), newConnId);
}
@@ -466,7 +469,7 @@
@SuppressWarnings("rawtypes")
public static Triple<IAdapterFactory, RecordDescriptor,
IDataSourceAdapter.AdapterType> getPrimaryFeedFactoryAndOutput(
Feed feed, FeedPolicyAccessor policyAccessor,
MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException {
+ throws AlgebricksException {
// This method needs to be re-visited
String adapterName = null;
DatasourceAdapter adapterEntity = null;
@@ -606,7 +609,7 @@
public static String getSecondaryFeedOutput(Feed feed, FeedPolicyAccessor
policyAccessor,
MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException, MetadataException,
RemoteException, ACIDException {
+ throws AlgebricksException, MetadataException, RemoteException,
ACIDException {
String outputType = null;
String primaryFeedName = feed.getSourceFeedName();
Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx,
feed.getDataverseName(), primaryFeedName);
diff --git
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index 33b7f30..b76e829 100644
---
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -45,8 +45,9 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider
runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) {
+ int partition, int ioDeviceNum) {
FileReference file = new FileReference(new File(filePath));
+
LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file,
runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits,
cmpFactories, bloomFilterKeyFields,
runtimeContextProvider.getBloomFilterFalsePositiveRate(),
--
To view, visit https://asterix-gerrit.ics.uci.edu/737
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iefad5ac3ff754693025312f1e2fa01b08832514c
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
