from:"amoudi"

asterixdb git commit: Revert "[NO ISSUE][LOG] Include destination IP in thread name on push back"

2018-08-31 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 129d401bf -> 8bf93fa70


Revert "[NO ISSUE][LOG] Include destination IP in thread name on push back"

This reverts commit 2498d28f0584d6ab002625417622be429f5c8131.

Change-Id: I221071a8a2851bde839e67d7f0380816c249366d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2941
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8bf93fa7
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8bf93fa7
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8bf93fa7

Branch: refs/heads/master
Commit: 8bf93fa70aaf44deced9737ef71001366a35b242
Parents: 129d401
Author: abdullah alamoudi 
Authored: Thu Aug 30 13:57:44 2018 -0700
Committer: abdullah alamoudi 
Committed: Fri Aug 31 00:05:58 2018 -0700

--
 .../hyracks/comm/channels/NetworkOutputChannel.java | 12 +---
 .../net/protocols/muxdemux/ChannelControlBlock.java |  5 -
 .../hyracks/net/protocols/muxdemux/ChannelSet.java  |  4 
 .../net/protocols/muxdemux/MultiplexedConnection.java   |  5 -
 4 files changed, 1 insertion(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8bf93fa7/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
 
b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
index 261e7c4..334fb5c 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
@@ -18,11 +18,9 @@
  */
 package org.apache.hyracks.comm.channels;
 
-import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.ArrayDeque;
 import java.util.Deque;
-import java.util.Objects;
 
 import org.apache.hyracks.api.comm.IBufferAcceptor;
 import org.apache.hyracks.api.comm.IFrameWriter;
@@ -77,15 +75,7 @@ public class NetworkOutputChannel implements IFrameWriter {
 break;
 }
 try {
-InetSocketAddress remoteAddress = 
ccb.getRemoteAddress();
-String nameBefore = Thread.currentThread().getName();
-try {
-Thread.currentThread()
-.setName(nameBefore + ":SendingTo(" + 
Objects.toString(remoteAddress) + ")");
-wait();
-} finally {
-Thread.currentThread().setName(nameBefore);
-}
+wait();
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw HyracksDataException.create(e);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8bf93fa7/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
 
b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
index f7ef2aa..998acfb 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -19,7 +19,6 @@
 package org.apache.hyracks.net.protocols.muxdemux;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.nio.channels.SocketChannel;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -164,8 +163,4 @@ public class ChannelControlBlock implements 
IChannelControlBlock {
 + " remoteClose: " + remoteClose + " remoteCloseAck:" + 
remoteCloseAck + " readCredits: "
 + ri.getCredits() + " writeCredits: " + wi.getCredits() + "]";
 }
-
-public InetSocketAddress getRemoteAddress() {
-return cSet.getMultiplexedConnection().getRemoteAddress();
-}
 }

asterixdb git commit: [NO ISSUE][LOG] Include destination IP in thread name on push back

2018-08-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master b944080fc -> 2498d28f0


[NO ISSUE][LOG] Include destination IP in thread name on push back

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- In certain cases, we noticed that threads of certain jobs are
  waiting on the network boundaries to send tuples.
- In that case and in a big cluster, it is difficult to tell which
  node is pushing back.
- To make this easy, we include the IP of the receiving node
  in the thread name.

Change-Id: I81b48cb7a79de0a37f2171a1bd3b82141f89b49d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2929
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2498d28f
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2498d28f
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2498d28f

Branch: refs/heads/master
Commit: 2498d28f0584d6ab002625417622be429f5c8131
Parents: b944080
Author: Abdullah Alamoudi 
Authored: Mon Aug 27 13:42:21 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Aug 27 18:48:53 2018 -0700

--
 .../hyracks/comm/channels/NetworkOutputChannel.java | 12 +++-
 .../net/protocols/muxdemux/ChannelControlBlock.java |  5 +
 .../hyracks/net/protocols/muxdemux/ChannelSet.java  |  4 
 .../net/protocols/muxdemux/MultiplexedConnection.java   |  5 +
 4 files changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2498d28f/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
 
b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
index 334fb5c..261e7c4 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/NetworkOutputChannel.java
@@ -18,9 +18,11 @@
  */
 package org.apache.hyracks.comm.channels;
 
+import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.ArrayDeque;
 import java.util.Deque;
+import java.util.Objects;
 
 import org.apache.hyracks.api.comm.IBufferAcceptor;
 import org.apache.hyracks.api.comm.IFrameWriter;
@@ -75,7 +77,15 @@ public class NetworkOutputChannel implements IFrameWriter {
 break;
 }
 try {
-wait();
+InetSocketAddress remoteAddress = 
ccb.getRemoteAddress();
+String nameBefore = Thread.currentThread().getName();
+try {
+Thread.currentThread()
+.setName(nameBefore + ":SendingTo(" + 
Objects.toString(remoteAddress) + ")");
+wait();
+} finally {
+Thread.currentThread().setName(nameBefore);
+}
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw HyracksDataException.create(e);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2498d28f/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
 
b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
index 998acfb..f7ef2aa 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -19,6 +19,7 @@
 package org.apache.hyracks.net.protocols.muxdemux;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.nio.channels.SocketChannel;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -163,4 +164,8 @@ public class ChannelControlBlock implements 
IChannelControlBlock {
 + " remoteClose: " + remoteClose + " remoteCloseAck:" + 
remoteCloseAck + " readCredits: "
 + ri.getCredits() + " writeCredits: "

asterixdb git commit: [NO ISSUE][COMP] Fix TypeComputeUtils.getResultType

2018-08-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 262a52025 -> b944080fc


[NO ISSUE][COMP] Fix TypeComputeUtils.getResultType

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Fix TypeComputeUtils.getResultType to either return
  unknowable or missable type.

Change-Id: Id881f2a5ad8134a3e86b2b08ce1acdc53c4948a2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2928
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Dmitry Lychagin 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b944080f
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b944080f
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b944080f

Branch: refs/heads/master
Commit: b944080fc286b90a37fcc2c63e93caa590b65007
Parents: 262a520
Author: Abdullah Alamoudi 
Authored: Sun Aug 26 13:27:28 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Aug 27 11:01:39 2018 -0700

--
 .../org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b944080f/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java
--
diff --git 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java
 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java
index 806235c..448ab4c 100644
--- 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java
+++ 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputeUtils.java
@@ -169,8 +169,7 @@ public class TypeComputeUtils {
 IAType resultType = type;
 if ((category & NULLABLE) != 0 || (category & NULL) != 0) {
 resultType = AUnionType.createUnknownableType(resultType);
-}
-if ((category & MISSABLE) != 0 || (category & MISSING) != 0) {
+} else if ((category & MISSABLE) != 0 || (category & MISSING) != 0) {
 resultType = AUnionType.createMissableType(resultType);
 }
 return resultType;

asterixdb git commit: [NO ISSUE][RT] Improve debuggability of runtime functions

2018-08-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 90114dfa3 -> 262a52025


[NO ISSUE][RT] Improve debuggability of runtime functions

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- This change introduce util methods to facilitate debugging
  functions runtime. Runtime functions read and write
  values from and into byte arrays.
- To allow easier inspection of such values, we provide
  a method that takes a pointable and produces an adm
  node representing the value pointed to by the pointable.

Change-Id: I33257507e9bdd3be80355218e4bafecd1c71c650
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2926
Reviewed-by: Till Westmann 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/262a5202
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/262a5202
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/262a5202

Branch: refs/heads/master
Commit: 262a5202523de9b220f72a812f989256a83b185b
Parents: 90114df
Author: Abdullah Alamoudi 
Authored: Fri Aug 24 14:44:31 2018 -0700
Committer: Till Westmann 
Committed: Fri Aug 24 16:11:50 2018 -0700

--
 .../external/parser/jackson/ObjectPool.java |  5 +++
 .../SerializerDeserializerProvider.java |  2 +-
 .../pointables/AbstractVisitablePointable.java  |  6 +++
 .../om/pointables/PointableAllocator.java   |  2 +-
 .../apache/asterix/om/utils/AdmNodeUtils.java   | 40 
 5 files changed, 53 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/262a5202/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
index 5ccd9e9..8945e71 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
@@ -59,4 +59,9 @@ public class ObjectPool {
 recycledObjects.add(object);
 }
 }
+
+@Override
+public String toString() {
+return recycledObjects.toString();
+}
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/262a5202/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
--
diff --git 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
index 0cba8b6..356b84c 100644
--- 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
+++ 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
@@ -171,7 +171,7 @@ public class SerializerDeserializerProvider implements 
ISerializerDeserializerPr
 }
 
 @SuppressWarnings("rawtypes")
-private ISerializerDeserializer addTag(final ISerializerDeserializer 
nonTaggedSerde) {
+public static ISerializerDeserializer addTag(final ISerializerDeserializer 
nonTaggedSerde) {
 return new ISerializerDeserializer() {
 
 private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/262a5202/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AbstractVisitablePointable.java
--
diff --git 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AbstractVisitablePointable.java
 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AbstractVisitablePointable.java
index 2ff9357..e9aa99c 100644
--- 
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AbstractVisitablePointable.java
+++ 
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AbstractVisitablePointable.java
@@ -59,4 +59,10 @@ public abstract class AbstractVisitablePointable implements 
IVisitablePointable
 set(ivf.getByteArray(), ivf.getStartOffset(), ivf.getLength());
 }
 
+@Override
+public String toString() {
+return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"data\" 
: "
++ (data == null ? "null" : ("\"" + 
System.identityHashCode(data) + ":" + data.length + "\""))
+

asterixdb-bad git commit: Fix tuple translators with extension datasets

2018-08-15 Thread amoudi

Repository: asterixdb-bad
Updated Branches:
  refs/heads/master ad1c6015e -> d0ccc577d


Fix tuple translators with extension datasets

Change-Id: I39f6707392085416c01e834ac85767f79a64f9d6


Project: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/commit/d0ccc577
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/tree/d0ccc577
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/diff/d0ccc577

Branch: refs/heads/master
Commit: d0ccc577db92bdb86ff42f8889fc826ef4f2f8ac
Parents: ad1c601
Author: Abdullah Alamoudi 
Authored: Wed Aug 15 08:50:05 2018 -0700
Committer: Abdullah Alamoudi 
Committed: Wed Aug 15 08:50:05 2018 -0700

--
 .../bad/metadata/BADMetadataIndexes.java| 26 ++--
 1 file changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb-bad/blob/d0ccc577/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/BADMetadataIndexes.java
--
diff --git 
a/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/BADMetadataIndexes.java
 
b/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/BADMetadataIndexes.java
index b33dcad..fa35fd5 100644
--- 
a/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/BADMetadataIndexes.java
+++ 
b/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/BADMetadataIndexes.java
@@ -32,17 +32,17 @@ public class BADMetadataIndexes {
 
 public static final ExtensionMetadataDatasetId BAD_CHANNEL_INDEX_ID = new 
ExtensionMetadataDatasetId(
 BADMetadataExtension.BAD_METADATA_EXTENSION_ID, 
BADConstants.CHANNEL_EXTENSION_NAME);
-public static final MetadataIndexImmutableProperties PROPERTIES_CHANNEL = 
new MetadataIndexImmutableProperties(
-BADConstants.CHANNEL_EXTENSION_NAME,
-
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID,
-
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID);
+public static final MetadataIndexImmutableProperties PROPERTIES_CHANNEL =
+new 
MetadataIndexImmutableProperties(BADConstants.CHANNEL_EXTENSION_NAME,
+
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID,
+
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID);
 
-public static final ExtensionMetadataDatasetId BAD_BROKER_INDEX_ID = new 
ExtensionMetadataDatasetId(
-BADMetadataExtension.BAD_METADATA_EXTENSION_ID, 
BADConstants.BROKER_KEYWORD);
-public static final MetadataIndexImmutableProperties PROPERTIES_BROKER = 
new MetadataIndexImmutableProperties(
-BADConstants.BROKER_KEYWORD,
-
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID 
+ 1,
-
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID 
+ 1);
+public static final ExtensionMetadataDatasetId BAD_BROKER_INDEX_ID =
+new 
ExtensionMetadataDatasetId(BADMetadataExtension.BAD_METADATA_EXTENSION_ID, 
BADConstants.BROKER_KEYWORD);
+public static final MetadataIndexImmutableProperties PROPERTIES_BROKER =
+new MetadataIndexImmutableProperties(BADConstants.BROKER_KEYWORD,
+
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID 
+ 1,
+
MetadataIndexImmutableProperties.FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID 
+ 1);
 
 public static final ExtensionMetadataDatasetId BAD_PROCEDURE_INDEX_ID = 
new ExtensionMetadataDatasetId(
 BADMetadataExtension.BAD_METADATA_EXTENSION_ID, 
BADConstants.PROCEDURE_KEYWORD);
@@ -61,7 +61,7 @@ public class BADMetadataIndexes {
 
Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
 Arrays.asList(BADConstants.ChannelName)),
 0, BADMetadataRecordTypes.CHANNEL_RECORDTYPE, true, new int[] { 0, 
1 }, BAD_CHANNEL_INDEX_ID,
-new ChannelTupleTranslator(true));
+() -> new ChannelTupleTranslator(true));
 
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public static final ExtensionMetadataDataset BROKER_DATASET = new 
ExtensionMetadataDataset(PROPERTIES_BROKER,
@@ -69,7 +69,7 @@ public class BADMetadataIndexes {
 
Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
 Arrays.asList(BADConstants.BrokerName)),
 0, BADMetadataRecordTypes.BROKER_RECORDTYPE, true, new int[] { 0, 
1 }, BAD_BROKER_INDEX_ID,
-new BrokerTupleTranslator(true));
+() -> new BrokerTupleTranslator(true));
 
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public static final

asterixdb git commit: [NO ISSUE][TXN] Fix extension datasets tuple translators

2018-08-15 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 4af131ada -> 51aef3c22


[NO ISSUE][TXN] Fix extension datasets tuple translators

Change-Id: Ifb8a8342d0f00468e1561d91eb802c52e66b149d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2908
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/51aef3c2
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/51aef3c2
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/51aef3c2

Branch: refs/heads/master
Commit: 51aef3c2283cee81ae4e57295bddb7abf555cb0c
Parents: 4af131a
Author: Abdullah Alamoudi 
Authored: Wed Aug 15 08:28:26 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Aug 15 12:55:32 2018 -0700

--
 .../metadata/api/ExtensionMetadataDataset.java  |  8 +++---
 .../IMetadataEntityTupleTranslatorFactory.java  | 29 
 2 files changed, 33 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/51aef3c2/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
--
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
index d25f488..ebb9cac 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
@@ -29,16 +29,16 @@ public class ExtensionMetadataDataset extends 
MetadataIndex {
 
 private static final long serialVersionUID = 1L;
 private final ExtensionMetadataDatasetId indexId;
-private final transient IMetadataEntityTupleTranslator tupleTranslator;
+private final transient IMetadataEntityTupleTranslatorFactory 
tupleTranslatorFactory;
 
 public ExtensionMetadataDataset(MetadataIndexImmutableProperties 
indexProperties, int numFields, IAType[] keyTypes,
 List> keyNames, int numSecondaryIndexKeys, 
ARecordType payloadType, boolean isPrimaryIndex,
 int[] primaryKeyIndexes, ExtensionMetadataDatasetId indexId,
-IMetadataEntityTupleTranslator tupleTranslator) {
+IMetadataEntityTupleTranslatorFactory tupleTranslatorFactory) {
 super(indexProperties, numFields, keyTypes, keyNames, 
numSecondaryIndexKeys, payloadType, isPrimaryIndex,
 primaryKeyIndexes);
 this.indexId = indexId;
-this.tupleTranslator = tupleTranslator;
+this.tupleTranslatorFactory = tupleTranslatorFactory;
 }
 
 public ExtensionMetadataDatasetId getId() {
@@ -46,6 +46,6 @@ public class ExtensionMetadataDataset extends 
MetadataIndex {
 }
 
 public IMetadataEntityTupleTranslator getTupleTranslator() {
-return tupleTranslator;
+return tupleTranslatorFactory.createTupleTranslator();
 }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/51aef3c2/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslatorFactory.java
--
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslatorFactory.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslatorFactory.java
new file mode 100644
index 000..e472383
--- /dev/null
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslatorFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.api;
+
+import java.io.Serializable;
+
+public interface

asterixdb git commit: [NO ISSUE][RT] Follow IFrameWriter protocol in AbstractOneInputPushRuntime

2018-08-14 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 9c7088ddb -> b2a7381a1


[NO ISSUE][RT] Follow IFrameWriter protocol in AbstractOneInputPushRuntime

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Many implementations of AbstractOneInputPushRuntime didn't
  follow the IFrameWriter protocol causing many unexpected
  runtime exceptions.
- This change ensures that all of the subclasses implement the
  protocol correctly.

Change-Id: I5133007f298366f58b53acc9f48bc553724dd7b5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2884
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b2a7381a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b2a7381a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b2a7381a

Branch: refs/heads/master
Commit: b2a7381a1dd644141affde764bc31beaf7bb482d
Parents: 9c7088d
Author: Abdullah Alamoudi 
Authored: Sun Aug 12 21:42:26 2018 -0700
Committer: abdullah alamoudi 
Committed: Tue Aug 14 09:43:58 2018 -0700

--
 .../management/runtime/CommitRuntime.java   | 27 +---
 .../aggreg/AggregateRuntimeFactory.java | 12 +
 ...actOneInputOneOutputOneFramePushRuntime.java | 22 +++-
 .../base/AbstractOneInputPushRuntime.java   | 19 +-
 .../base/AbstractOneInputSourcePushRuntime.java | 10 
 .../operators/meta/SubplanRuntimeFactory.java   |  3 ++-
 .../sort/InMemorySortRuntimeFactory.java| 24 -
 .../operators/std/AssignRuntimeFactory.java | 18 +
 .../std/EmptyTupleSourceRuntimeFactory.java | 11 +++-
 .../std/NestedTupleSourceRuntimeFactory.java|  5 
 .../std/RunningAggregateRuntimeFactory.java | 18 +
 .../std/StreamLimitRuntimeFactory.java  |  2 +-
 .../std/StreamProjectRuntimeFactory.java|  2 +-
 .../std/StreamSelectRuntimeFactory.java | 12 +
 .../std/StringStreamingRuntimeFactory.java  |  5 +++-
 .../operators/std/UnnestRuntimeFactory.java |  2 +-
 16 files changed, 63 insertions(+), 129 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b2a7381a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/runtime/CommitRuntime.java
--
diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/runtime/CommitRuntime.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/runtime/CommitRuntime.java
index 74ba139..2692cc7 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/runtime/CommitRuntime.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/runtime/CommitRuntime.java
@@ -89,7 +89,7 @@ public class CommitRuntime extends 
AbstractOneInputOneOutputOneFramePushRuntime
 return;
 }
 initAccessAppend(ctx);
-writer.open();
+super.open();
 } catch (ACIDException e) {
 throw HyracksDataException.create(e);
 }
@@ -142,31 +142,6 @@ public class CommitRuntime extends 
AbstractOneInputOneOutputOneFramePushRuntime
 }
 
 @Override
-public void fail() throws HyracksDataException {
-failed = true;
-if (isSink) {
-return;
-}
-writer.fail();
-}
-
-@Override
-public void close() throws HyracksDataException {
-if (isSink) {
-return;
-}
-try {
-flushIfNotFailed();
-} catch (Exception e) {
-writer.fail();
-throw e;
-} finally {
-writer.close();
-}
-appender.reset(frame, true);
-}
-
-@Override
 public void setInputRecordDescriptor(int index, RecordDescriptor 
recordDescriptor) {
 this.inputRecordDesc = recordDescriptor;
 this.tAccess = new FrameTupleAccessor(inputRecordDesc);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b2a7381a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
--
diff --git 
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
 
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
index e99b61b..1f9cb91

asterixdb git commit: [ASTERIXDB-2065][ING] Make stop timeout configurable

2018-08-08 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master aeb6e3bc2 -> 8b3a2e2ce


[ASTERIXDB-2065][ING] Make stop timeout configurable

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Make timeout of a graceful stop of an active job
  configurable.

Change-Id: Id8d0c205a6959967dbce2b7223061ffa2f26059c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2845
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8b3a2e2c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8b3a2e2c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8b3a2e2c

Branch: refs/heads/master
Commit: 8b3a2e2ce5fa3fd26e9770a0ba144b1b4b537f62
Parents: aeb6e3b
Author: Abdullah Alamoudi 
Authored: Wed Aug 8 16:52:53 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Aug 8 18:37:54 2018 -0700

--
 .../asterix/app/active/ActiveEntityEventsListener.java |  7 ++-
 .../asterix/test/active/ActiveEventsListenerTest.java  |  4 +++-
 .../apache/asterix/test/active/TestEventsListener.java |  5 -
 .../api/cluster_state_1/cluster_state_1.1.regexadm |  2 ++
 .../cluster_state_1_full.1.regexadm|  2 ++
 .../cluster_state_1_less.1.regexadm|  2 ++
 .../apache/asterix/common/config/ActiveProperties.java | 13 -
 7 files changed, 27 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8b3a2e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 22c9b5d..39a8402 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -74,10 +74,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 private static final EnumSet TRANSITION_STATES = 
EnumSet.of(ActivityState.RESUMING,
 ActivityState.STARTING, ActivityState.STOPPING, 
ActivityState.RECOVERING, ActivityState.CANCELLING);
 private static final String DEFAULT_ACTIVE_STATS = "{\"Stats\":\"N/A\"}";
-// TODO: Make configurable 
https://issues.apache.org/jira/browse/ASTERIXDB-2065
-protected static final long STOP_MESSAGE_TIMEOUT = 5L;
-protected static final long SUSPEND_MESSAGE_TIMEOUT = 10L;
-protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.MINUTES;
+protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
 protected final IClusterStateManager clusterStateManager;
 protected final ActiveNotificationHandler handler;
 protected final List subscribers = new 
ArrayList<>();
@@ -521,7 +518,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 } else if (state == ActivityState.RUNNING) {
 setState(ActivityState.STOPPING);
 try {
-doStop(metadataProvider, STOP_MESSAGE_TIMEOUT, TIMEOUT_UNIT);
+doStop(metadataProvider, 
appCtx.getActiveProperties().getActiveStopTimeout(), TIMEOUT_UNIT);
 } catch (Exception e) {
 setState(ActivityState.STOPPED);
 LOGGER.log(Level.ERROR, "Failed to stop the entity " + 
entityId, e);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8b3a2e2c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index 6c95958..64520a4 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -37,6 +37,7 @@ import 
org.apache.asterix.app.active.ActiveNotificationHandler;
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.api.IMetadataLockManager;
 import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.config.ActiveProperties;
 import org.apache.asterix.common.context.IStorageComponentProvider;
 import

asterixdb git commit: [NO ISSUE][ING] Set JobId in thread waiting for job to complete

2018-08-07 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 53f4f6f48 -> 1f821f2b2


[NO ISSUE][ING] Set JobId in thread waiting for job to complete

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When stopping an active job, set the job Id in the thread to enable
  debugging if the job gets stuck.

Change-Id: Ic19d11f150aa404522133d4d7227c758dbf0de6a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2850
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/1f821f2b
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/1f821f2b
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/1f821f2b

Branch: refs/heads/master
Commit: 1f821f2b25cc529dc55e566f4ca5bf1cafbf0ce9
Parents: 53f4f6f
Author: Abdullah Alamoudi 
Authored: Mon Aug 6 16:29:04 2018 -0700
Committer: abdullah alamoudi 
Committed: Tue Aug 7 11:56:02 2018 -0700

--
 .../src/main/java/org/apache/asterix/active/ActiveManager.java   | 2 --
 .../apache/asterix/api/http/server/NCQueryServiceServlet.java| 3 ++-
 .../apache/asterix/app/active/ActiveEntityEventsListener.java| 4 
 3 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1f821f2b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index 6373d6c..4adffda 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -172,8 +172,6 @@ public class ActiveManager {
 executor.execute(() -> {
 try {
 stopIfRunning(runtime, content.getTimeout(), 
content.getUnit());
-} catch (Exception e) {
-LOGGER.warn("Failed to stop runtime: {}", runtimeId, e);
 } catch (Throwable th) {
 LOGGER.warn("Failed to stop runtime: {}", runtimeId, th);
 ExitUtil.halt(ExitUtil.EC_UNCAUGHT_THROWABLE);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1f821f2b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 77b8647..362f924 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -45,8 +45,8 @@ import org.apache.asterix.translator.ResultProperties;
 import org.apache.asterix.translator.SessionOutput;
 import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hyracks.api.application.INCServiceContext;
-import org.apache.hyracks.api.result.ResultSetId;
 import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
 import org.apache.hyracks.http.api.IChannelClosedHandler;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.server.HttpServer;
@@ -144,6 +144,7 @@ public class NCQueryServiceServlet extends 
QueryServiceServlet {
 CancelQueryRequest cancelQueryMessage =
 new CancelQueryRequest(nodeId, 
cancelQueryFuture.getFutureId(), clientContextID);
 // TODO(mblow): multicc -- need to send cancellation to the 
correct cc
+LOGGER.info("Cancelling query due to {}", 
exception.getClass().getSimpleName());
 messageBroker.sendMessageToPrimaryCC(cancelQueryMessage);
 if (wait) {
 
cancelQueryFuture.get(ExecuteStatementRequestMessage.DEFAULT_QUERY_CANCELLATION_WAIT_MILLIS,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1f821f2b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index b5432c5..22c9b5d 100644
---

asterixdb git commit: [NO ISSUE][STO] Notify of completion of IO request in finally

2018-08-07 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master cf4f7f7be -> 700bbddf5


[NO ISSUE][STO] Notify of completion of IO request in finally

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Notify completion of a request in a finally clause to ensure
  waiting thread is always notified.

Change-Id: I8b3003b47b6b181856faf82aca6e828ee014527c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2839
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/700bbddf
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/700bbddf
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/700bbddf

Branch: refs/heads/master
Commit: 700bbddf58a5290bbfe6b916a7ab33361544532c
Parents: cf4f7f7
Author: Abdullah Alamoudi 
Authored: Mon Aug 6 13:47:31 2018 -0700
Committer: abdullah alamoudi 
Committed: Tue Aug 7 10:12:13 2018 -0700

--
 .../org/apache/hyracks/control/nc/io/IoRequest.java | 10 ++
 .../hyracks/storage/am/lsm/common/impls/LSMHarness.java | 12 +++-
 2 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/700bbddf/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IoRequest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IoRequest.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IoRequest.java
index 8c81d41..93e38f5 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IoRequest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IoRequest.java
@@ -44,7 +44,7 @@ public class IoRequest implements IAsyncRequest, 
InterruptibleAction {
 private long offset;
 private ByteBuffer data;
 private ByteBuffer[] dataArray;
-private HyracksDataException failure;
+private Throwable failure;
 private int read;
 private int write;
 private long writes;
@@ -133,15 +133,17 @@ public class IoRequest implements IAsyncRequest, 
InterruptibleAction {
 state = State.OPERATION_SUCCEEDED;
 } catch (Throwable th) { // NOSONAR: This method must never throw 
anything
 state = State.OPERATION_FAILED;
-failure = HyracksDataException.create(th);
+failure = th;
+} finally {
+notifyAll();
 }
-notifyAll();
 }
 
 public State getState() {
 return state;
 }
 
+@SuppressWarnings("squid:S899") // Offer failing means we're over capacity 
and this should be garbage collected
 void recycle() {
 reset();
 freeRequests.offer(this);
@@ -165,6 +167,6 @@ public class IoRequest implements IAsyncRequest, 
InterruptibleAction {
 }
 
 public HyracksDataException getFailure() {
-return failure;
+return HyracksDataException.create(failure);
 }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/700bbddf/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 3eea0a9..8e2ff01 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -142,9 +142,7 @@ public class LSMHarness implements ILSMHarness {
 }
 entranceSuccessful = numEntered == components.size();
 } catch (Throwable e) { // NOSONAR: Log and re-throw
-if (LOGGER.isErrorEnabled()) {
-LOGGER.log(Level.ERROR, opType.name() + " failed to enter 
components on " + lsmIndex, e);
-}
+LOGGER.warn("{} failed to enter components on {}", opType.name(), 
lsmIndex, e);
 throw e;
 } finally {
 if (!entranceSuccessful) {
@@ -201,9 +199,7 @@ public class LSMHarness implements ILSMHarness {

asterixdb git commit: [NO ISSUE][STO] Skip flush recovery of empty resources

2018-08-04 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 6cd3127c4 -> 8026b2dc9


[NO ISSUE][STO] Skip flush recovery of empty resources

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Before this change, recovery would throw a NullPointerException
  on recovery of a flush operation on a component without
  update logs.
- Since this can happen, we simply check for the case and skip the
  flush.

Change-Id: Ib01d7513f43830109632760860d34ca3dcddeaee
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2844
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: abdullah alamoudi 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8026b2dc
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8026b2dc
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8026b2dc

Branch: refs/heads/master
Commit: 8026b2dc95b0b48949b192409be97ec40ac3fa2d
Parents: 6cd3127
Author: Abdullah Alamoudi 
Authored: Fri Aug 3 15:46:20 2018 -0700
Committer: abdullah alamoudi 
Committed: Fri Aug 3 23:33:48 2018 -0700

--
 .../apache/asterix/app/nc/RecoveryManager.java  | 30 
 1 file changed, 18 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8026b2dc/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 5e8a5e8..adf9960 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -296,7 +296,7 @@ public class RecoveryManager implements IRecoveryManager, 
ILifeCycleComponent {
 ((INcApplicationContext) 
(serviceCtx.getApplicationContext())).getIndexCheckpointManagerProvider();
 
 Map resourcesMap = 
localResourceRepository.loadAndGetAllResources();
-Map resourceId2MaxLSNMap = new HashMap<>();
+final Map resourceId2MaxLSNMap = new HashMap<>();
 TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, 
false);
 
 ILogRecord logRecord = null;
@@ -399,19 +399,25 @@ public class RecoveryManager implements IRecoveryManager, 
ILifeCycleComponent {
 // we only need to flush open indexes here (opened 
by previous update records)
 // if an index has no ongoing updates, then it's 
memory component must be empty
 // and there is nothing to flush
-for (IndexInfo iInfo : 
dsInfo.getIndexes().values()) {
+for (final IndexInfo iInfo : 
dsInfo.getIndexes().values()) {
 if (iInfo.isOpen() && iInfo.getPartition() == 
partition) {
-maxDiskLastLsn = 
resourceId2MaxLSNMap.get(iInfo.getResourceId());
-index = iInfo.getIndex();
-if (logRecord.getLSN() > maxDiskLastLsn
-&& 
!index.isCurrentMutableComponentEmpty()) {
-// schedule flush
-redoFlush(index, logRecord);
-redoCount++;
+Long maxLsnBeforeFlush = 
resourceId2MaxLSNMap.get(iInfo.getResourceId());
+if (maxLsnBeforeFlush != null) {
+// If there was at least one update to 
the resource.
+// IMPORTANT: Don't remove the check 
above
+// This check is to support indexes 
without transaction logs
+maxDiskLastLsn = maxLsnBeforeFlush;
+index = iInfo.getIndex();
+if (logRecord.getLSN() > maxDiskLastLsn
+&& 
!index.isCurrentMutableComponentEmpty()) {
+// schedule flush
+redoFlush(index, logRecord);
+redoCount++;
+} else {
+// TODO: update checkpoint file?
+}
 } else {
-

asterixdb git commit: [NO ISSUE][ING] Increase timeout for stopping ingestion

2018-08-02 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a7094732b -> 5de12d01c


[NO ISSUE][ING] Increase timeout for stopping ingestion

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- To allow systems under high load enough time to push
  accumulated data to storage, we increased the default
  timeout for stopping active entities to 5 minutes.

Change-Id: I3e590b1c7fdc0eb9be34d82b4b2ac1b2a2eef515
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2841
Reviewed-by: Murtadha Hubail 
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/5de12d01
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/5de12d01
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/5de12d01

Branch: refs/heads/master
Commit: 5de12d01c019e609992845e6fc5a2d0a9b2cb9e9
Parents: a709473
Author: Abdullah Alamoudi 
Authored: Thu Aug 2 17:05:43 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Aug 2 20:13:45 2018 -0700

--
 .../apache/asterix/app/active/ActiveEntityEventsListener.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5de12d01/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 345080c..b5432c5 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -75,8 +75,8 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 ActivityState.STARTING, ActivityState.STOPPING, 
ActivityState.RECOVERING, ActivityState.CANCELLING);
 private static final String DEFAULT_ACTIVE_STATS = "{\"Stats\":\"N/A\"}";
 // TODO: Make configurable 
https://issues.apache.org/jira/browse/ASTERIXDB-2065
-protected static final long STOP_MESSAGE_TIMEOUT = 2L;
-protected static final long SUSPEND_MESSAGE_TIMEOUT = 5L;
+protected static final long STOP_MESSAGE_TIMEOUT = 5L;
+protected static final long SUSPEND_MESSAGE_TIMEOUT = 10L;
 protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.MINUTES;
 protected final IClusterStateManager clusterStateManager;
 protected final ActiveNotificationHandler handler;

asterixdb git commit: [NO ISSUE] Set JobId in thread name waiting for job completion

2018-08-02 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 39ee11732 -> ad13fc646


[NO ISSUE] Set JobId in thread name waiting for job completion

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- To allow correlation between running tasks and requests waiting
  for job completion, we set the job id in the request thread and
  we also add it to the name of the parallel threads in Super
  Activity Operator Node Pushable.

Change-Id: I003432da0109c253e976f130e91f1f1f5ff628f3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2837
Reviewed-by: Murtadha Hubail 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ad13fc64
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ad13fc64
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ad13fc64

Branch: refs/heads/master
Commit: ad13fc6463b5adae637c80157aa3df1cc26c3a45
Parents: 39ee117
Author: Abdullah Alamoudi 
Authored: Thu Aug 2 11:28:40 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Aug 2 13:52:55 2018 -0700

--
 .../main/java/org/apache/asterix/common/utils/JobUtils.java  | 8 +++-
 .../rewriter/runtime/SuperActivityOperatorNodePushable.java  | 4 
 .../hyracks/control/cc/work/WaitForJobCompletionWork.java| 2 ++
 3 files changed, 13 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ad13fc64/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
index cacbfbc..232c8dd 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
@@ -43,7 +43,13 @@ public class JobUtils {
 spec.setMaxReattempts(0);
 final JobId jobId = hcc.startJob(spec, jobFlags);
 if (waitForCompletion) {
-hcc.waitForCompletion(jobId);
+String nameBefore = Thread.currentThread().getName();
+try {
+Thread.currentThread().setName(nameBefore + " : 
WaitForCompletionForJobId: " + jobId);
+hcc.waitForCompletion(jobId);
+} finally {
+Thread.currentThread().setName(nameBefore);
+}
 }
 return jobId;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ad13fc64/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index d499554..46a1dec 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -205,6 +205,10 @@ public class SuperActivityOperatorNodePushable implements 
IOperatorNodePushable
 tasks.add(ctx.getExecutorService().submit(() -> {
 startSemaphore.release();
 try {
+Thread.currentThread()
+.setName(Thread.currentThread().getName() + 
":" + ctx.getJobletContext().getJobId()
++ ":" + ctx.getTaskAttemptId() + ":"
++ 
SuperActivityOperatorNodePushable.class.getSimpleName());
 action.run(op);
 } catch (Throwable th) { // NOSONAR: Must catch all causes 
of failure
 failures.offer(th);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ad13fc64/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/WaitForJobCompletionWork.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/WaitForJobCompletionWork.java

asterixdb git commit: [NO ISSUE] Improve logging for Http server

2018-08-01 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 93c8b4cdf -> bb8856004


[NO ISSUE] Improve logging for Http server

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Reduce logging level for failures in HttpServerHandler.
- Log as an error failures during active jobs.

Change-Id: I7474a93e869df1cbf5dbb00d16acbe2d4bfaa6e0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2834
Reviewed-by: Murtadha Hubail 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/bb885600
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/bb885600
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/bb885600

Branch: refs/heads/master
Commit: bb885600431bd1c08bf8483ae830356856c62c03
Parents: 93c8b4c
Author: Abdullah Alamoudi 
Authored: Wed Aug 1 18:47:35 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Aug 1 21:12:31 2018 -0700

--
 .../asterix/app/active/ActiveEntityEventsListener.java   | 2 ++
 .../org/apache/hyracks/http/server/HttpServerHandler.java| 8 
 2 files changed, 6 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb885600/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index be43a4e..345080c 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -191,6 +191,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 @SuppressWarnings("unchecked")
 protected void finish(ActiveEvent event) throws HyracksDataException {
 LOGGER.log(level, "the job " + jobId + " finished");
+JobId lastJobId = jobId;
 if (numRegistered != numDeRegistered) {
 LOGGER.log(Level.WARN,
 "the job {} finished with reported runtime registrations = 
{} and deregistrations = {}", jobId,
@@ -204,6 +205,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 if (!jobSuccessfullyTerminated(jobStatus)) {
 jobFailure = exceptions.isEmpty() ? new 
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
 : exceptions.get(0);
+LOGGER.error("Active Job {} failed", lastJobId, jobFailure);
 setState((state == ActivityState.STOPPING || state == 
ActivityState.CANCELLING) ? ActivityState.STOPPED
 : ActivityState.TEMPORARILY_FAILED);
 if (prevState == ActivityState.RUNNING) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb885600/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
index 7b3d18a..baa664a 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
@@ -91,7 +91,7 @@ public class HttpServerHandler extends 
SimpleChannelInboun
 submit(ctx, servlet, request);
 }
 } catch (Exception e) {
-LOGGER.log(Level.ERROR, "Failure Submitting HTTP Request", e);
+LOGGER.log(Level.WARN, "Failure Submitting HTTP Request", e);
 respond(ctx, request.protocolVersion(), new 
HttpResponseStatus(500, e.getMessage()));
 }
 }
@@ -125,15 +125,15 @@ public class HttpServerHandler 
extends SimpleChannelInboun
 }
 
 protected void handleServletNotFound(ChannelHandlerContext ctx, 
FullHttpRequest request) {
-if (LOGGER.isWarnEnabled()) {
-LOGGER.warn("No servlet for " + request.uri());
+if (LOGGER.isDebugEnabled()) {
+LOGGER.debug("No servlet for " + request.uri());
 }
 respond(ctx, request.protocolVersion(), HttpResponseStatus.NOT_FOUND);
 }
 
 @Override
 public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-

asterixdb git commit: [NO ISSUE] Set logging level to WARN in MaterializingPipelinedPartition

2018-08-01 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 1027474db -> 708f3c37e


[NO ISSUE] Set logging level to WARN in MaterializingPipelinedPartition

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
Failures in MaterializingPipelinedPartition can be due to many reasons
such as a job cancellation. WARN log level seems to be adequate for
such failures keeping ERROR for more critical failures.

Change-Id: I45f475afae114dc71d2d89d9deaab7f3b3c5a0b8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2832
Sonar-Qube: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/708f3c37
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/708f3c37
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/708f3c37

Branch: refs/heads/master
Commit: 708f3c37e27bd41226c2e768ff640134b6d4bcac
Parents: 1027474
Author: Abdullah Alamoudi 
Authored: Wed Aug 1 11:35:28 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Aug 1 14:15:35 2018 -0700

--
 .../control/nc/partitions/MaterializingPipelinedPartition.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/708f3c37/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
index 29c2ff2..37f33a9 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
@@ -89,7 +89,7 @@ public class MaterializingPipelinedPartition implements 
IFrameWriter, IPartition
 Thread thread = Thread.currentThread();
 setDataConsumerThread(thread); // Sets the data consumer 
thread to the current thread.
 try {
-
thread.setName(MaterializingPipelinedPartition.class.getName() + pid);
+
thread.setName(MaterializingPipelinedPartition.class.getName() + " " + pid);
 FileReference fRefCopy;
 synchronized (MaterializingPipelinedPartition.this) {
 while (fRef == null && !eos && !failed) {
@@ -164,7 +164,7 @@ public class MaterializingPipelinedPartition implements 
IFrameWriter, IPartition
 }
 }
 } catch (Exception e) {
-LOGGER.log(Level.ERROR, e.getMessage(), e);
+LOGGER.warn("Failure writing to a frame", e);
 } finally {
 setDataConsumerThread(null); // Sets back the data 
consumer thread to null.
 }

asterixdb git commit: [NO ISSUE][TEST] Store results even if there is no expected results

2018-07-30 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a91abc2e5 -> 317628afe


[NO ISSUE][TEST] Store results even if there is no expected results

Change-Id: I659a089ce640b93141828155d144bb1170b072ff
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2814
Reviewed-by: Ian Maxon 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/317628af
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/317628af
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/317628af

Branch: refs/heads/master
Commit: 317628afed8d92bd328fbe486198ea91188c6ceb
Parents: a91abc2
Author: Abdullah Alamoudi 
Authored: Mon Jul 30 14:40:01 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 30 16:33:34 2018 -0700

--
 .../src/test/java/org/apache/asterix/test/common/TestExecutor.java  | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/317628af/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 73b3e70..afc3246 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -1182,6 +1182,7 @@ public class TestExecutor {
 if (testFile.getName().startsWith(DIAGNOSE)) {
 LOGGER.info("Diagnostic output: {}", 
IOUtils.toString(resultStream, StandardCharsets.UTF_8));
 } else {
+LOGGER.info("Unexpected output: {}", 
IOUtils.toString(resultStream, StandardCharsets.UTF_8));
 Assert.fail("no result file for " + testFile.toString() + 
"; queryCount: " + queryCount
 + ", filectxs.size: " + numResultFiles);
 }

asterixdb git commit: [NO ISSUE][TEST] Fix Component Lifecycle Test

2018-07-30 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a731afb17 -> a91abc2e5


[NO ISSUE][TEST] Fix Component Lifecycle Test

Change-Id: Idb5156db0fb0e060d998851756ee3925c02b5791
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2827
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a91abc2e
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a91abc2e
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a91abc2e

Branch: refs/heads/master
Commit: a91abc2e54921d4eb8f98ed84a18142df663b6dd
Parents: a731afb
Author: Abdullah Alamoudi 
Authored: Mon Jul 30 08:39:30 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 30 14:37:23 2018 -0700

--
 .../am/lsm/btree/LSMBTreeComponentLifecycleTest.java  | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a91abc2e/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
index 21f3877..251f57a 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
@@ -37,6 +37,7 @@ import 
org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
 import org.apache.hyracks.storage.am.lsm.btree.impl.NoOpTestCallback;
 import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
 import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -47,7 +48,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
 import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -292,7 +292,7 @@ public class LSMBTreeComponentLifecycleTest {
 ILSMIndexOperationContext opCtx = accessor.getOpContext();
 assertCorrectSearchComponents(opCtx, index, 0);
 // Allow one flush at a time and ensure that inserter didn't succeed
-for (int i = 0; i < numFlushes - 1; i++) {
+for (int i = 0; i < numFlushes - 
index.getNumberOfAllMemoryComponents(); i++) {
 flushSemaphore.release();
 firstUser.step();
 flushRequest.await(2 + i);
@@ -300,21 +300,21 @@ public class LSMBTreeComponentLifecycleTest {
 // also ensure that you get the correct components when searching
 assertCorrectSearchComponents(opCtx, index, i + 1);
 }
-// Allow last flush to proceed
 flushSemaphore.release();
+firstUser.step();
 // wait for the insert to complete
 insertRequest.await();
+// Allow last flush to proceed
+flushSemaphore.release();
 firstUser.step();
 firstUser.step();
 flushRequest.await();
 firstUser.stop();
 secondUser.stop();
-
 int expectedMemoryComponent = numFlushes % numMemoryComponents;
 
Assert.assertEquals(getExpectedMemoryComponentIndex(expectedMemoryComponent),
 index.getCurrentMemoryComponentIndex());
 Assert.assertEquals(0, index.getDiskComponents().size());
-
 EncapsulatingIoCallback encapsulating = (EncapsulatingIoCallback) 
index.getIOOperationCallback();
 CountingIoOperationCallback ioCallback = (CountingIoOperationCallback) 
encapsulating.getEncapsulated();
 // assert equal before,

asterixdb git commit: [ASTERIXDB-2414][STO] Fix name of merge files

2018-07-30 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master ad4761481 -> a731afb17


[ASTERIXDB-2414][STO] Fix name of merge files

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- A bug is found where merge file names are created incorrectly
  where start and end components are reversed.
- The bug is fixed and an explicit check for the invariance was
  added.

Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2820
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Luo Chen 
Reviewed-by: Wail Alkowaileet 
Reviewed-by: Ian Maxon 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a731afb1
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a731afb1
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a731afb1

Branch: refs/heads/master
Commit: a731afb1724644b3f553662ac4cd97004adbd5eb
Parents: ad47614
Author: Abdullah Alamoudi 
Authored: Thu Jul 26 22:14:08 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 30 14:36:51 2018 -0700

--
 .../storage/am/lsm/btree/impls/ExternalBTree.java|  4 ++--
 .../am/lsm/btree/impls/ExternalBTreeWithBuddy.java   |  4 ++--
 .../am/lsm/common/impls/AbstractLSMIndex.java|  4 ++--
 .../common/impls/AbstractLSMIndexFileManager.java| 15 +--
 .../lsm/common/impls/LSMComponentFileReferences.java |  6 ++
 .../storage/am/lsm/rtree/impls/ExternalRTree.java|  4 ++--
 6 files changed, 27 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 8be75be..2be5125 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -188,8 +188,8 @@ public class ExternalBTree extends LSMBTree implements 
ITwoPCIndex {
 }
 }
 LSMBTreeRangeSearchCursor cursor = new 
LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
-BTree firstBTree = ((LSMBTreeDiskComponent) 
mergingComponents.get(0)).getIndex();
-BTree lastBTree = ((LSMBTreeDiskComponent) 
mergingComponents.get(mergingComponents.size() - 1)).getIndex();
+BTree lastBTree = ((LSMBTreeDiskComponent) 
mergingComponents.get(0)).getIndex();
+BTree firstBTree = ((LSMBTreeDiskComponent) 
mergingComponents.get(mergingComponents.size() - 1)).getIndex();
 FileReference firstFile = firstBTree.getFileReference();
 FileReference lastFile = lastBTree.getFileReference();
 LSMComponentFileReferences relMergeFileRefs =

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 5bcf30d..be91244 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -256,8 +256,8 @@ public class ExternalBTreeWithBuddy extends 
AbstractLSMIndex implements ITreeInd
 
 protected LSMComponentFileReferences 
getMergeTargetFileName(List mergingDiskComponents)
 throws HyracksDataException {
-BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) 
mergingDiskComponents.get(0)).getIndex();
-BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) 
mergingDiskComponents.get(mergingDiskComponents.size() - 1))
+BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) 
mergingDiskComponents.get(0)).getIndex();
+BTree firstTree =

asterixdb git commit: [ASTERIXDB-2414][STO] Remove deleted component files from buffer cache

2018-07-29 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master afadf2cad -> 825bf10fb


[ASTERIXDB-2414][STO] Remove deleted component files from buffer cache

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When activating an LSM index, we remove files of components
  that were merged into a bigger component but not cleaned up yet.
- However, we sometimes leave a file reference mapped in the buffer
  cache even when the file is removed from disk.
- This change ensures that all files are removed from the buffer
  cache as well.

Change-Id: If0f11bc222662e4b50c1b47b1dfa6b30d1463b2e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2822
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/825bf10f
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/825bf10f
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/825bf10f

Branch: refs/heads/master
Commit: 825bf10fbbfb7471208f0ba4bb809c771ecb5d07
Parents: afadf2c
Author: Abdullah Alamoudi 
Authored: Sun Jul 29 10:50:00 2018 -0700
Committer: abdullah alamoudi 
Committed: Sun Jul 29 14:15:06 2018 -0700

--
 .../am/lsm/btree/impls/LSMBTreeFileManager.java | 10 
 .../impls/LSMBTreeWithBuddyFileManager.java | 13 +--
 .../impls/AbstractLSMIndexFileManager.java  | 24 
 .../impls/LSMInvertedIndexFileManager.java  | 19 
 .../am/lsm/rtree/impls/LSMRTreeFileManager.java | 17 +++---
 .../TestStorageManagerComponentHolder.java  | 20 
 .../component/TestLsmIndexFileManager.java  | 10 
 .../common/test/LSMIndexFileManagerTest.java| 14 ++--
 8 files changed, 77 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/825bf10f/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 54bc1fe..7fbde73 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -19,7 +19,6 @@
 
 package org.apache.hyracks.storage.am.lsm.btree.impls;
 
-import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -81,7 +80,8 @@ public class LSMBTreeFileManager extends 
AbstractLSMIndexFileManager {
 // create transaction filter 
 FilenameFilter transactionFilter = getTransactionFileFilter(false);
 // List of valid BTree files.
-cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, 
btreeFilter), btreeFactory, allBTreeFiles);
+cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, 
btreeFilter), btreeFactory, allBTreeFiles,
+btreeFactory.getBufferCache());
 HashSet btreeFilesSet = new HashSet<>();
 for (ComparableFileName cmpFileName : allBTreeFiles) {
 int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
@@ -90,7 +90,7 @@ public class LSMBTreeFileManager extends 
AbstractLSMIndexFileManager {
 
 if (hasBloomFilter) {
 validateFiles(btreeFilesSet, allBloomFilterFiles, 
getCompoundFilter(transactionFilter, bloomFilterFilter),
-null);
+null, btreeFactory.getBufferCache());
 // Sanity check.
 if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
 throw 
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
@@ -148,9 +148,9 @@ public class LSMBTreeFileManager extends 
AbstractLSMIndexFileManager {
 && (!hasBloomFilter || 
(currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
 && 
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) {
 // Invalid files are completely contained in last interval.
-IoUtil.delete(new File(currentBTree.fullPath));
+delete(btreeFactory.getBufferCache(), currentBTree.fullPath);
 if (hasBloomFilter) {
-

asterixdb git commit: [NO ISSUE] Allow MetadataProvider config to store non String values

2018-07-26 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master d1867c053 -> eac3de5a0


[NO ISSUE] Allow MetadataProvider config to store non String values

- user model changes: no
- storage format changes: no
- interface changes: yes

Details:
- In many cases, we would like to associate a value with a key
  in MetadataProvider to be accessed during the compilation
  of jobs. However, currently, there is no place to store
  such values, so we ended up storing them in the config
  map.
- The config map is a  map and so, we would
  write our values as a string and then parse them when needed.
- To avoid this, and to avoid introducing a new map, we simply
  change the config stored in MetadataProvider from 
  to .

Change-Id: I55b392ad199d74b0f3cffdc38b54593b12ec1a06
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2795
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 
Integration-Tests: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/eac3de5a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/eac3de5a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/eac3de5a

Branch: refs/heads/master
Commit: eac3de5a01d07da1af192ceab0443fec0891227b
Parents: d1867c0
Author: Abdullah Alamoudi 
Authored: Wed Jul 25 12:27:42 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Jul 26 16:37:09 2018 -0700

--
 .../asterix/optimizer/base/FuzzyUtils.java  |  6 +--
 .../optimizer/rules/am/AccessMethodUtils.java   | 45 +---
 .../apache/asterix/api/common/APIFramework.java | 20 +
 .../asterix/app/active/FeedEventsListener.java  |  3 +-
 .../asterix/app/function/FeedRewriter.java  |  4 +-
 .../asterix/app/translator/QueryTranslator.java |  5 ++-
 .../app/bootstrap/TestNodeController.java   |  9 +++-
 .../asterix/test/sqlpp/ParserTestExecutor.java  |  2 +-
 .../NoOpFrameOperationCallbackFactory.java  |  5 +++
 .../asterix/common/exceptions/ErrorCode.java|  1 +
 .../main/resources/asx_errormsg/en.properties   |  1 +
 .../asterix/lang/common/util/FunctionUtil.java  |  2 +-
 .../lang/sqlpp/rewrites/SqlppQueryRewriter.java |  2 +-
 .../metadata/declared/MetadataProvider.java | 20 ++---
 .../asterix/metadata/entities/Dataset.java  |  2 +-
 .../asterix/metadata/utils/DatasetUtil.java |  2 +-
 .../LSMPrimaryUpsertOperatorNodePushable.java   | 14 +-
 .../management/service/logging/LogManager.java  |  4 +-
 .../algebra/metadata/IMetadataProvider.java |  3 +-
 .../rules/EnforceStructuralPropertiesRule.java  |  2 +-
 .../apache/hyracks/control/nc/CcConnection.java |  2 +-
 .../hyracks/ipc/impl/IPCConnectionManager.java  |  4 +-
 .../lsm/common/api/IFrameOperationCallback.java |  7 +++
 .../java/org/apache/hyracks/util/ExitUtil.java  |  3 ++
 24 files changed, 104 insertions(+), 64 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/eac3de5a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
index 503a631..d7e05b3 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
@@ -60,7 +60,7 @@ public class FuzzyUtils {
 }
 
 public static IAObject getSimThreshold(MetadataProvider metadata, String 
simFuncName) {
-String simThresholValue = 
metadata.getPropertyValue(SIM_THRESHOLD_PROP_NAME);
+String simThresholValue = 
metadata.getProperty(SIM_THRESHOLD_PROP_NAME);
 IAObject ret = null;
 if (simFuncName.equals(JACCARD_FUNCTION_NAME)) {
 if (simThresholValue != null) {
@@ -103,7 +103,7 @@ public class FuzzyUtils {
 
 public static float getSimThreshold(MetadataProvider metadata) {
 float simThreshold = JACCARD_DEFAULT_SIM_THRESHOLD;
-String simThresholValue = 
metadata.getPropertyValue(SIM_THRESHOLD_PROP_NAME);
+String simThresholValue = 
metadata.getProperty(SIM_THRESHOLD_PROP_NAME);
 if (simThresholValue != null) {
 simThreshold = Float.parseFloat(simThresholValue);
 }
@@ -112,7 +112,7 @@ public class FuzzyUtils {
 
 // TODO: The default function depend on the input types.
 public static String getSimFunction(MetadataProvider metadata) {
-String simFunction = metadata.getPropertyValue(SIM_FUNCTION_PROP_NAME);
+String simFunction = metadata.getProperty(SIM_FUNCTION_PROP_NAME);
 if (simFunction == null) {

asterixdb git commit: [NO ISSUE][STO] Delete Resource File Before Checkpoints

2018-07-25 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master cedd961ae -> 3c8e98259


[NO ISSUE][STO] Delete Resource File Before Checkpoints

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- To avoid a case where the index checkpoints can be deleted but
  a failure happens before the resource file is deleted, we delete
  the resource file first.

Change-Id: Ia3351e9375007c0dbc415f5418458314233268d4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2812
Reviewed-by: abdullah alamoudi 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3c8e9825
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3c8e9825
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3c8e9825

Branch: refs/heads/master
Commit: 3c8e982591762fe19ee870ff3c36218996dae8b0
Parents: cedd961
Author: Murtadha Hubail 
Authored: Wed Jul 25 14:50:23 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Jul 25 18:57:58 2018 -0700

--
 .../management/resource/PersistentLocalResourceRepository.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3c8e9825/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
--
diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 93d9414..f9718c4 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -237,13 +237,13 @@ public class PersistentLocalResourceRepository implements 
ILocalResourceReposito
 if (isReplicationEnabled) {
 createReplicationJob(ReplicationOperation.DELETE, 
resourceFile);
 }
-// delete all checkpoints
 final LocalResource localResource = 
readLocalResource(resourceFile.getFile());
-
indexCheckpointManagerProvider.get(DatasetResourceReference.of(localResource)).delete();
 // Invalidate before deleting the file just in case file deletion 
throws some exception.
 // Since it's just a cache invalidation, it should not affect 
correctness.
 resourceCache.invalidate(relativePath);
 IoUtil.delete(resourceFile);
+// delete all checkpoints
+
indexCheckpointManagerProvider.get(DatasetResourceReference.of(localResource)).delete();
 } else {
 throw 
HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.RESOURCE_DOES_NOT_EXIST,
 relativePath);

asterixdb git commit: [NO ISSUE][RT] Abort tasks on local network failures

2018-07-24 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master b2eb44177 -> a7d87c698


[NO ISSUE][RT] Abort tasks on local network failures

- user model changes: no
- storage format changes: no
- interface changes: yes
  Add error code to IInputChannelMonitor.notifyFailure

Details:
- Previously, there was an assumption that all failures
  reported to an IInputChannelMonitor come from a remote
  task.
- This assumption is not always true and could lead
  to jobs hanging.
- To fix this, we report an error code indicating whether
  the failure is local or remote and if the failure is local
  then we fail the local task and report the failure to cc.

Change-Id: I7ea5b9008383faaac7c563671242b03919090b35
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2806
Sonar-Qube: Jenkins 
Reviewed-by: Murtadha Hubail 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a7d87c69
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a7d87c69
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a7d87c69

Branch: refs/heads/master
Commit: a7d87c698736d875e8949fa8bf3ad8ad5541492e
Parents: b2eb441
Author: Abdullah Alamoudi 
Authored: Tue Jul 24 16:28:13 2018 -0700
Committer: abdullah alamoudi 
Committed: Tue Jul 24 18:15:13 2018 -0700

--
 .../api/channels/IInputChannelMonitor.java  |  6 ++---
 .../hyracks/api/exceptions/ErrorCode.java   |  1 +
 .../src/main/resources/errormsg/en.properties   |  1 +
 .../client/dataset/HyracksDatasetReader.java|  2 +-
 .../channels/DatasetNetworkInputChannel.java|  2 +-
 .../comm/channels/NetworkInputChannel.java  |  2 +-
 .../comm/channels/NetworkOutputChannel.java |  4 +--
 .../org/apache/hyracks/control/nc/Task.java |  3 ++-
 .../ReceiveSideMaterializingCollector.java  |  2 +-
 .../hyracks/hyracks-dataflow-std/pom.xml|  5 
 .../std/collectors/InputChannelFrameReader.java | 27 ++--
 .../NonDeterministicChannelReader.java  | 13 +-
 .../muxdemux/AbstractChannelReadInterface.java  |  4 ---
 .../muxdemux/AbstractChannelWriteInterface.java |  6 +++--
 .../net/protocols/muxdemux/ChannelSet.java  |  2 +-
 15 files changed, 54 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7d87c69/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/channels/IInputChannelMonitor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/channels/IInputChannelMonitor.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/channels/IInputChannelMonitor.java
index 52509d3..559f49a 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/channels/IInputChannelMonitor.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/channels/IInputChannelMonitor.java
@@ -19,9 +19,9 @@
 package org.apache.hyracks.api.channels;
 
 public interface IInputChannelMonitor {
-public void notifyFailure(IInputChannel channel);
+void notifyFailure(IInputChannel channel, int errorCode);
 
-public void notifyDataAvailability(IInputChannel channel, int nFrames);
+void notifyDataAvailability(IInputChannel channel, int nFrames);
 
-public void notifyEndOfStream(IInputChannel channel);
+void notifyEndOfStream(IInputChannel channel);
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7d87c69/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index b6d7cc7..09193d9 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -148,6 +148,7 @@ public class ErrorCode {
 public static final int CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT 
= 112;
 public static final int UNDEFINED_INVERTED_LIST_MERGE_TYPE = 113;
 public static final int NODE_IS_NOT_ACTIVE = 114;
+public static final int LOCAL_NETWORK_ERROR = 115;
 
 // Compilation error codes.
 public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 1;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7d87c69/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties

[1/2] asterixdb git commit: [NO ISSUE] Report all BufferCache write failures.

2018-07-24 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 47ab031e8 -> 7c72a503d


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7c72a503/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
index dbead1e..589d697 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
@@ -24,15 +24,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class AsyncFIFOPageQueueManager implements Runnable {
-private final static boolean DEBUG = false;
+private static final boolean DEBUG = false;
+private static final Logger LOGGER = LogManager.getLogger();
 
-protected LinkedBlockingQueue queue = new 
LinkedBlockingQueue();
+protected LinkedBlockingQueue queue = new 
LinkedBlockingQueue<>();
 volatile Thread writerThread;
 protected AtomicBoolean poisoned = new AtomicBoolean(false);
 protected BufferCache bufferCache;
-volatile protected PageQueue pageQueue;
+protected volatile PageQueue pageQueue;
 
 public AsyncFIFOPageQueueManager(BufferCache bufferCache) {
 this.bufferCache = bufferCache;
@@ -57,17 +61,27 @@ public class AsyncFIFOPageQueueManager implements Runnable {
 return writer;
 }
 
+@SuppressWarnings("squid:S2142")
 @Override
-public void put(ICachedPage page) throws HyracksDataException {
+public void put(ICachedPage page, IPageWriteFailureCallback callback) 
throws HyracksDataException {
+failIfPreviousPageFailed(callback);
+page.setFailureCallback(callback);
 try {
 if (!poisoned.get()) {
 queue.put(page);
 } else {
-throw new HyracksDataException("Queue is closing");
+LOGGER.error("An attempt to write a page found buffer 
cache closed");
+ExitUtil.halt(ExitUtil.EC_ABNORMAL_TERMINATION);
 }
 } catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-throw HyracksDataException.create(e);
+LOGGER.error("IO Operation interrupted", e);
+ExitUtil.halt(ExitUtil.EC_ABNORMAL_TERMINATION);
+}
+}
+
+private void failIfPreviousPageFailed(IPageWriteFailureCallback 
callback) throws HyracksDataException {
+if (callback.hasFailed()) {
+throw HyracksDataException.create(callback.getFailure());
 }
 }
 }
@@ -136,18 +150,21 @@ public class AsyncFIFOPageQueueManager implements 
Runnable {
 }
 }
 
+@SuppressWarnings("squid:S2142")
 @Override
 public void run() {
-if (DEBUG)
-System.out.println("[FIFO] Writer started");
+if (DEBUG) {
+LOGGER.info("[FIFO] Writer started");
+}
 boolean die = false;
 while (!die) {
 ICachedPage entry;
 try {
 entry = queue.take();
 } catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-break;
+LOGGER.error("BufferCache Write Queue was interrupted", e);
+ExitUtil.halt(ExitUtil.EC_ABNORMAL_TERMINATION);
+return; // Keep compiler happy
 }
 if (entry.getQueueInfo() != null && 
entry.getQueueInfo().hasWaiters()) {
 synchronized (entry) {
@@ -158,17 +175,11 @@ public class AsyncFIFOPageQueueManager implements 
Runnable {
 continue;
 }
 }
-
-if (DEBUG)
-System.out.println("[FIFO] Write " + 
BufferedFileHandle.getFileId(((CachedPage) entry).dpid) + ","
+if (DEBUG) {
+LOGGER.info("[FIFO] Write " + 
BufferedFileHandle.getFileId(((CachedPage) entry).dpid) + ","
 + BufferedFileHandle.getPageId(((CachedPage) 
entry).dpid));
-
-try {
-pageQueue.getWriter().write(entry, bufferCache);
-

[2/2] asterixdb git commit: [NO ISSUE] Report all BufferCache write failures.

2018-07-24 Thread amoudi

[NO ISSUE] Report all BufferCache write failures.

- user model changes: no
- storage format changes: no
- interface changes: yes
  + IPageWriteFailureCallback: used to notify async
IO caller when something goes wrong.

Details:
- Before this change, it is possible for failures to
  be lost and for bulkload operations to not be
  aware of failure to write some pages. This can be
  dangerous.
- To avoid this, when sending a page to be written
  a PageWriteFailureCallback is associated with the
  page to notify the caller that a failure took place.

Change-Id: I97fd3dccff85dab84d644359be6f66b15ee708ef
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2787
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 
Reviewed-by: Luo Chen 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7c72a503
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7c72a503
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7c72a503

Branch: refs/heads/master
Commit: 7c72a503d18cd870dd18fe61e27fb04c7a18f442
Parents: 47ab031
Author: Abdullah Alamoudi 
Authored: Mon Jul 23 14:31:23 2018 -0700
Committer: abdullah alamoudi 
Committed: Tue Jul 24 11:13:31 2018 -0700

--
 .../am/bloomfilter/impls/BloomFilter.java   | 32 +---
 .../hyracks/storage/am/btree/impls/BTree.java   | 12 ++---
 .../storage/am/common/api/IPageManager.java | 11 -
 .../AppendOnlyLinkedMetadataPageManager.java|  7 ++-
 .../freepage/LinkedMetaDataPageManager.java |  8 ++-
 .../am/common/impls/AbstractTreeIndex.java  | 38 +++
 .../am/lsm/btree/impls/ExternalBTree.java   | 25 --
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 25 --
 ...AbstractLSMWithBloomFilterDiskComponent.java |  5 +-
 .../api/AbstractLSMWithBuddyDiskComponent.java  |  7 +--
 .../am/lsm/common/api/ILSMDiskComponent.java|  5 +-
 .../storage/am/lsm/common/api/ILSMHarness.java  |  6 +--
 .../am/lsm/common/api/ILSMIOOperation.java  |  4 +-
 .../common/freepage/VirtualFreePageManager.java |  3 +-
 .../lsm/common/impls/AbstractIoOperation.java   | 13 -
 .../common/impls/AbstractLSMDiskComponent.java  |  5 +-
 .../lsm/common/impls/BloomFilterBulkLoader.java | 16 ++
 .../ChainedLSMDiskComponentBulkLoader.java  | 43 ++---
 .../am/lsm/common/impls/EmptyComponent.java |  3 +-
 .../lsm/common/impls/ExternalIndexHarness.java  | 13 -
 .../am/lsm/common/impls/FilterBulkLoader.java   | 16 ++
 .../impls/IChainedComponentBulkLoader.java  |  3 +-
 .../common/impls/IndexWithBuddyBulkLoader.java  | 21 
 .../storage/am/lsm/common/impls/LSMHarness.java | 15 --
 .../am/lsm/common/impls/LSMIndexBulkLoader.java | 16 ++
 .../impls/LSMIndexDiskComponentBulkLoader.java  | 18 ++-
 .../am/lsm/common/impls/NoOpIoOperation.java| 11 +
 .../am/lsm/common/impls/TracedIOOperation.java  | 11 +
 .../am/lsm/common/util/ComponentUtils.java  | 11 +++--
 .../impls/LSMInvertedIndexDiskComponent.java| 15 --
 .../ondisk/OnDiskInvertedIndex.java | 11 +++--
 .../am/lsm/rtree/impls/ExternalRTree.java   | 25 --
 .../hyracks/storage/am/rtree/impls/RTree.java   | 10 ++--
 .../hyracks/hyracks-storage-common/pom.xml  |  5 ++
 .../storage/common/IIndexBulkLoader.java|  4 +-
 .../buffercache/AsyncFIFOPageQueueManager.java  | 51 
 .../storage/common/buffercache/CachedPage.java  | 29 +--
 .../common/buffercache/FIFOLocalWriter.java | 28 ++-
 .../buffercache/HaltOnFailureCallback.java  | 44 +
 .../storage/common/buffercache/ICachedPage.java |  4 ++
 .../common/buffercache/ICachedPageInternal.java | 11 +++--
 .../common/buffercache/IFIFOPageQueue.java  | 17 ++-
 .../common/buffercache/IFIFOPageWriter.java |  7 +--
 .../buffercache/IPageWriteFailureCallback.java  | 43 +
 .../buffercache/PageWriteFailureCallback.java   | 42 
 .../storage/common/buffercache/VirtualPage.java | 10 
 .../am/btree/DiskBTreeSearchCursorTest.java |  3 ++
 .../hyracks/storage/common/BufferCacheTest.java |  4 +-
 48 files changed, 610 insertions(+), 156 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7c72a503/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java

asterixdb git commit: [NO ISSUE][RT] Improve logging in task related works

2018-07-23 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master ca5dc823b -> 072da02e4


[NO ISSUE][RT] Improve logging in task related works

Change-Id: Ia4e24a95aaac37b1d3d0d9a35266109ae0315293
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2803
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/072da02e
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/072da02e
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/072da02e

Branch: refs/heads/master
Commit: 072da02e4c6e7845a28ada591fc441f3a78fd20d
Parents: ca5dc82
Author: Abdullah Alamoudi 
Authored: Mon Jul 23 14:22:14 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 23 17:54:13 2018 -0700

--
 .../apache/hyracks/control/cc/job/JobManager.java   | 16 
 .../java/org/apache/hyracks/control/nc/Task.java|  8 +++-
 .../control/nc/work/NotifyTaskCompleteWork.java |  3 ++-
 .../control/nc/work/NotifyTaskFailureWork.java  |  5 +
 .../std/collectors/InputChannelFrameReader.java |  4 
 .../collectors/NonDeterministicChannelReader.java   |  7 +++
 6 files changed, 29 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/072da02e/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index b6b3e40..7dc636c 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -221,17 +221,17 @@ public class JobManager implements IJobManager {
 JobId jobId = run.getJobId();
 Throwable caughtException = null;
 CCServiceContext serviceCtx = ccs.getContext();
-if (serviceCtx != null) {
-try {
-serviceCtx.notifyJobFinish(jobId, run.getPendingStatus(), 
run.getPendingExceptions());
-} catch (Exception e) {
-LOGGER.error("Exception notifying job finish {}", jobId, e);
-caughtException = e;
-}
+try {
+serviceCtx.notifyJobFinish(jobId, run.getPendingStatus(), 
run.getPendingExceptions());
+} catch (Exception e) {
+LOGGER.error("Exception notifying job finish {}", jobId, e);
+caughtException = e;
 }
 run.setStatus(run.getPendingStatus(), run.getPendingExceptions());
 run.setEndTime(System.currentTimeMillis());
-activeRunMap.remove(jobId);
+if (activeRunMap.remove(jobId) == null) {
+LOGGER.warn("Job {} was not found running but is getting archived 
and capacity released", jobId);
+}
 runMapArchive.put(jobId, run);
 runMapHistory.put(jobId, run.getExceptions());
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/072da02e/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
index d7b930c..23a5abb 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
@@ -290,7 +290,7 @@ public class Task implements IHyracksTaskContext, 
ICounterContext, Runnable {
 .schedule(new NotifyTaskFailureWork(ncs, this, exceptions, 
joblet.getJobId(), taskAttemptId));
 return;
 }
-ct.setName(displayName + ":" + taskAttemptId + ":" + 0);
+ct.setName(displayName + ":" + joblet.getJobId() + ":" + taskAttemptId 
+ ":" + 0);
 try {
 Throwable operatorException = null;
 try {
@@ -469,4 +469,10 @@ public class Task implements IHyracksTaskContext, 
ICounterContext, Runnable {
 public boolean isCompleted() {
 return completed;
 }
+
+@Override
+public String

asterixdb git commit: [NO ISSUE][OBS] Log content of index dir with missing checkpoints

2018-07-23 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 19ca14e87 -> ca5dc823b


[NO ISSUE][OBS] Log content of index dir with missing checkpoints

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When an index is missing checkpoint file, it should contain
  the mask and gets cleaned up on bootstrap. when that doesn't
  happen, we should log the content of the dir to help investigating.

Change-Id: I1903a40e129f648d4a0f8be6e820c6d45c722bec
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2801
Sonar-Qube: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ca5dc823
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ca5dc823
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ca5dc823

Branch: refs/heads/master
Commit: ca5dc823b63d723ac3bdfcd4a7bff729ecf6023f
Parents: 19ca14e
Author: Abdullah Alamoudi 
Authored: Mon Jul 23 13:54:46 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 23 16:10:14 2018 -0700

--
 .../java/org/apache/asterix/app/nc/IndexCheckpointManager.java| 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ca5dc823/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
index c1a81a3..64d8e93 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
@@ -27,6 +27,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
@@ -141,6 +142,8 @@ public class IndexCheckpointManager implements 
IIndexCheckpointManager {
 throw HyracksDataException.create(e);
 }
 if (checkpoints.isEmpty()) {
+LOGGER.warn("Couldn't find any checkpoint file for index {}. 
Content of dir are {}.", indexPath,
+Arrays.toString(indexPath.toFile().listFiles()));
 throw new IllegalStateException("Couldn't find any checkpoints for 
resource: " + indexPath);
 }
 
checkpoints.sort(Comparator.comparingLong(IndexCheckpoint::getId).reversed());

asterixdb git commit: [NO ISSUE][STO] Cleanup corrupted resources on failed creation

2018-07-23 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 1708bb553 -> fe759264b


[NO ISSUE][STO] Cleanup corrupted resources on failed creation

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- In some cases, when a rebalance is cancelled by the user,
  we abort the process of creating local resources on
  nodes. When that happens, it can leave corrupted
  resources that causes subsequent drop and/or create
  to fail until the node is restarted.
- To avoid this, we make sure that the operation that
  creates the resource is atomic.

Change-Id: I095a8eb0f1be2a9aa0fc269770978691746c3cec
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2800
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/fe759264
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/fe759264
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/fe759264

Branch: refs/heads/master
Commit: fe759264bbfd3ae683642ec1c8d4aa4eb75e37a7
Parents: 1708bb5
Author: Abdullah Alamoudi 
Authored: Sun Jul 22 09:53:47 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 23 11:08:39 2018 -0700

--
 .../PersistentLocalResourceRepository.java  | 27 +---
 .../java/org/apache/hyracks/util/ExitUtil.java  |  2 ++
 2 files changed, 25 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/fe759264/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
--
diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 7cd31bb..93d9414 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -73,6 +73,7 @@ import 
org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import 
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
 import org.apache.hyracks.storage.common.ILocalResourceRepository;
 import org.apache.hyracks.storage.common.LocalResource;
+import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -182,6 +183,7 @@ public class PersistentLocalResourceRepository implements 
ILocalResourceReposito
 return resource;
 }
 
+@SuppressWarnings("squid:S1181")
 @Override
 public synchronized void insert(LocalResource resource) throws 
HyracksDataException {
 String relativePath = getFileName(resource.getPath());
@@ -194,16 +196,21 @@ public class PersistentLocalResourceRepository implements 
ILocalResourceReposito
 if (!parent.exists() && !parent.mkdirs()) {
 throw HyracksDataException.create(CANNOT_CREATE_FILE, 
parent.getAbsolutePath());
 }
-createResourceFileMask(resourceFile);
+// The next block should be all or nothing
 try {
+createResourceFileMask(resourceFile);
 byte[] bytes = 
OBJECT_MAPPER.writeValueAsBytes(resource.toJson(persistedResourceRegistry));
 final Path path = Paths.get(resourceFile.getAbsolutePath());
 Files.write(path, bytes);
-} catch (IOException e) {
+
indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(null,
 0);
+deleteResourceFileMask(resourceFile);
+} catch (Exception e) {
+cleanup(resourceFile);
 throw HyracksDataException.create(e);
+} catch (Throwable th) {
+LOGGER.error("Error creating resource {}", resourceFile, th);
+ExitUtil.halt(ExitUtil.EC_ERROR_CREATING_RESOURCES);
 }
-
indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(null,
 0);
-deleteResourceFileMask(resourceFile);
 resourceCache.put(resource.getPath(), resource);
 //if replication enabled, send resource metadata info to remote nodes
 if (isReplicationEnabled) {
@@ -211,6 +218,18 @@ public class PersistentLocalResourceRepository implements 
ILocalResourceReposito
 }
 }
 
+@SuppressWarnings("squid:S1181")
+private void cleanup(FileReference resourceFile) {

asterixdb git commit: [NO ISSUE][MISC] Introduce IClientRequest

2018-07-22 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a7aaf7110 -> 7df888fff


[NO ISSUE][MISC] Introduce IClientRequest

- user model changes: no
- storage format changes: no
- interface changes: yes
  + IClientRequest: used to represent a client request
that can be cancelled.

Details:
- Introduce IClientRequest to allow for multiple types of requests
  to be cancellable.

Change-Id: I8f65da1744ea7ecf26ea3f8a576ebaf4472ccd62
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2774
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7df888ff
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7df888ff
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7df888ff

Branch: refs/heads/master
Commit: 7df888fffcb946f639b2c58ef4463d660244a567
Parents: a7aaf71
Author: Abdullah Alamoudi 
Authored: Wed Jul 18 08:30:00 2018 -0700
Committer: abdullah alamoudi 
Committed: Sun Jul 22 14:02:28 2018 -0700

--
 .../algebra/extension/ExtensionStatement.java   |  6 +-
 .../asterix/translator/BaseClientRequest.java   | 54 +
 .../asterix/translator/ClientJobRequest.java| 44 +++
 .../translator/IStatementExecutorContext.java   | 19 +++--
 .../NoOpStatementExecutorContext.java   |  8 +-
 .../api/http/ctx/StatementExecutorContext.java  | 13 ++--
 .../http/server/CcQueryCancellationServlet.java | 76 +++
 .../http/server/NCQueryCancellationServlet.java |  3 +-
 .../http/server/QueryCancellationServlet.java   | 79 
 .../api/http/server/QueryServiceServlet.java|  2 +-
 .../asterix/app/message/CancelQueryRequest.java | 12 ++-
 .../asterix/app/translator/QueryTranslator.java | 11 ++-
 .../hyracks/bootstrap/CCApplication.java|  4 +-
 .../servlet/QueryCancellationServletTest.java   | 15 ++--
 .../asterix/common/api/IClientRequest.java  | 38 ++
 .../metadata/declared/MetadataProvider.java |  1 +
 16 files changed, 262 insertions(+), 123 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7df888ff/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java
index 15267aa..a37f802 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java
@@ -22,6 +22,7 @@ import org.apache.asterix.lang.common.base.AbstractStatement;
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.translator.IRequestParameters;
 import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorContext;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -45,10 +46,11 @@ public abstract class ExtensionStatement extends 
AbstractStatement {
  * @param requestParameters
  * @param metadataProvider
  * @param resultSetId
+ * @param executorCtx
  * @throws HyracksDataException
  * @throws AlgebricksException
  */
 public abstract void handle(IHyracksClientConnection hcc, 
IStatementExecutor statementExecutor,
-IRequestParameters requestParameters, MetadataProvider 
metadataProvider, int resultSetId)
-throws HyracksDataException, AlgebricksException;
+IRequestParameters requestParameters, MetadataProvider 
metadataProvider, int resultSetId,
+IStatementExecutorContext executorCtx) throws 
HyracksDataException, AlgebricksException;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7df888ff/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/BaseClientRequest.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/BaseClientRequest.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/BaseClientRequest.java
new file mode 100644
index 000..a9bd856
--- /dev/null
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/BaseClientRequest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF)

asterixdb git commit: [NO ISSUE][STO] Log more information on File is already mapped

2018-07-19 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 91384f6ca -> 669c1d9c8


[NO ISSUE][STO] Log more information on File is already mapped

Change-Id: Ifcc1d56a29c67e0cfc999defc00894f456c92ca9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2791
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/669c1d9c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/669c1d9c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/669c1d9c

Branch: refs/heads/master
Commit: 669c1d9c829995e4f1fdca883fde31feaf72bdb5
Parents: 91384f6
Author: Abdullah Alamoudi 
Authored: Thu Jul 19 17:16:35 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Jul 19 18:43:47 2018 -0700

--
 .../org/apache/asterix/app/nc/HaltCallback.java |  2 +-
 .../apache/hyracks/api/io/FileReference.java| 21 
 .../src/main/resources/errormsg/en.properties   |  2 +-
 .../storage/common/file/FileMapManager.java | 10 --
 4 files changed, 31 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/669c1d9c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
index db34131..17a4f46 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
@@ -41,7 +41,7 @@ public class HaltCallback implements 
IIoOperationFailedCallback {
 
 @Override
 public void operationFailed(ILSMIOOperation operation, Throwable t) {
-LOGGER.error("Operation {} has failed", t);
+LOGGER.error("Operation {} has failed", operation, t);
 if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
 ExitUtil.halt(ExitUtil.EC_FLUSH_FAILED);
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/669c1d9c/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 0375e9e..4ded855 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -20,6 +20,7 @@ package org.apache.hyracks.api.io;
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.Date;
 
 /**
  * A device handle and a relative path.
@@ -31,6 +32,7 @@ public final class FileReference implements Serializable {
 private final File file;
 private final IODeviceHandle dev;
 private final String path;
+private long registrationTime = 0L;
 
 public FileReference(IODeviceHandle dev, String path) {
 file = new File(dev.getMount(), path);
@@ -90,4 +92,23 @@ public final class FileReference implements Serializable {
 public FileReference getChild(String name) {
 return new FileReference(dev, path + File.separator + name);
 }
+
+public void register() {
+if (registrationTime != 0) {
+throw new IllegalStateException(
+"File " + toString() + " was already registered at " + new 
Date(registrationTime));
+}
+registrationTime = System.currentTimeMillis();
+}
+
+public long registrationTime() {
+return registrationTime;
+}
+
+public void unregister() {
+if (registrationTime == 0) {
+throw new IllegalStateException("File " + toString() + " wasn't 
registered before");
+}
+registrationTime = 0;
+}
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/669c1d9c/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index ef07038..bf73b91 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -97,7 +97,7 @@
 78 = Failed to purge the bloom filter since it is active
 79 = Cannot

asterixdb git commit: [NO ISSUE][MISC] Add ping function

2018-07-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 488162719 -> 36124bdea


[NO ISSUE][MISC] Add ping function

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Add a ping function that takes 0 parameters
  and simply returns a single record from each node
  which contains the node id.

Change-Id: Ic077a11b61dec782756e2dede33ca8220893a2de
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2764
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/36124bde
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/36124bde
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/36124bde

Branch: refs/heads/master
Commit: 36124bdeaa74fdc1bbb4931247dd969d78b4917d
Parents: 4881627
Author: Abdullah Alamoudi 
Authored: Mon Jul 16 00:04:57 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 16 16:08:16 2018 -0700

--
 .../asterix/app/function/PingDatasource.java| 44 +
 .../asterix/app/function/PingFunction.java  | 41 
 .../apache/asterix/app/function/PingReader.java | 50 
 .../asterix/app/function/PingRewriter.java  | 43 +
 .../asterix/util/MetadataBuiltinFunctions.java  |  7 ++-
 .../queries_sqlpp/misc/ping/ping.1.query.sqlpp  | 21 
 .../runtimets/results/misc/ping/ping.1.adm  |  2 +
 .../resources/runtimets/testsuite_sqlpp.xml |  5 ++
 8 files changed, 212 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/36124bde/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
new file mode 100644
index 000..480f3c4
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.function;
+
+import org.apache.asterix.metadata.api.IDatasourceFunction;
+import org.apache.asterix.metadata.declared.DataSourceId;
+import org.apache.asterix.metadata.declared.FunctionDataSource;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import 
org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
+
+public class PingDatasource extends FunctionDataSource {
+
+private static final DataSourceId PING_DATASOURCE_ID =
+new DataSourceId(PingRewriter.PING.getNamespace(), 
PingRewriter.PING.getName());
+
+public PingDatasource(INodeDomain domain) throws AlgebricksException {
+super(PING_DATASOURCE_ID, domain);
+}
+
+@Override
+protected IDatasourceFunction createFunction(MetadataProvider 
metadataProvider,
+AlgebricksAbsolutePartitionConstraint locations) {
+return new PingFunction(locations);
+}
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/36124bde/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingFunction.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingFunction.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingFunction.java
new file mode 100644
index 000..c37ed7d
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingFunction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license

asterixdb git commit: [NO ISSUE][ING] Add timeout to stop active message

2018-07-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 571236390 -> 488162719


[NO ISSUE][ING] Add timeout to stop active message

Change-Id: Ie0416d76670e945cb958f5a1c235201a3e016009
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2762
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/48816271
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/48816271
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/48816271

Branch: refs/heads/master
Commit: 488162719bfc2748bb400960664eafce54dbd191
Parents: 5712363
Author: Abdullah Alamoudi 
Authored: Sun Jul 15 23:08:47 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jul 16 09:27:51 2018 -0700

--
 .../apache/asterix/active/ActiveManager.java| 24 ++
 .../ActiveSourceOperatorNodePushable.java   | 11 +++--
 .../apache/asterix/active/IActiveRuntime.java   |  9 +++-
 .../active/message/StopRuntimeParameters.java   | 50 
 .../app/active/ActiveEntityEventsListener.java  | 21 
 .../test/active/DummyFeedEventsListener.java|  3 +-
 .../asterix/test/active/TestEventsListener.java |  5 +-
 .../FeedIntakeOperatorNodePushable.java |  8 ++--
 .../apache/hyracks/api/util/CleanupUtils.java   |  3 +-
 .../java/org/apache/hyracks/util/ExitUtil.java  |  1 +
 10 files changed, 106 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/48816271/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index 2aebc59..6373d6c 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -33,6 +33,7 @@ import java.util.concurrent.TimeoutException;
 import org.apache.asterix.active.message.ActiveManagerMessage;
 import org.apache.asterix.active.message.ActiveStatsRequestMessage;
 import org.apache.asterix.active.message.ActiveStatsResponse;
+import org.apache.asterix.active.message.StopRuntimeParameters;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.common.memory.ConcurrentFramePool;
@@ -40,6 +41,7 @@ import org.apache.hyracks.api.application.INCServiceContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.util.JavaSerializationUtils;
 import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -139,7 +141,7 @@ public class ActiveManager {
 shutdown = true;
 runtimes.forEach((runtimeId, runtime) -> stopFutures.put(runtimeId, 
executor.submit(() -> {
 // we may already have been stopped- only stop once
-stopIfRunning(runtimeId, runtime);
+stopIfRunning(runtime, SHUTDOWN_TIMEOUT_SECS, TimeUnit.SECONDS);
 return null;
 })));
 stopFutures.entrySet().parallelStream().forEach(entry -> {
@@ -157,8 +159,10 @@ public class ActiveManager {
 LOGGER.warn("Shutdown ActiveManager on node " + nodeId + " complete");
 }
 
+@SuppressWarnings("squid:S1181") // Catch Error
 private void stopRuntime(ActiveManagerMessage message) {
-ActiveRuntimeId runtimeId = (ActiveRuntimeId) message.getPayload();
+StopRuntimeParameters content = (StopRuntimeParameters) 
message.getPayload();
+ActiveRuntimeId runtimeId = content.getRuntimeId();
 IActiveRuntime runtime = runtimes.get(runtimeId);
 if (runtime == null) {
 LOGGER.warn("Request to stop runtime: " + runtimeId
@@ -167,21 +171,23 @@ public class ActiveManager {
 } else {
 executor.execute(() -> {
 try {
-stopIfRunning(runtimeId, runtime);
+stopIfRunning(runtime, content.getTimeout(), 
content.getUnit());
 } catch (Exception e) {
-// TODO(till) Figure out a better way to handle failure to 
stop a runtime
-LOGGER.log(Level.WARN, "Failed to stop runtime: " + 
runtimeId, e);
+LOGGER.warn("Failed to stop runtime: {}", runtimeId, e);
+} catch (Throwable th) {
+

asterixdb git commit: [NO ISSUE][TEST] Improve extensibility of test framework

2018-07-13 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 36d98e9b2 -> 26457e366


[NO ISSUE][TEST] Improve extensibility of test framework

Change-Id: Ibaa9722cbc46a2eced6295fa6b896f38fa3c9e50
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2753
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/26457e36
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/26457e36
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/26457e36

Branch: refs/heads/master
Commit: 26457e366f9c7e3c0fc1931e2e6c33d7f13d0187
Parents: 36d98e9
Author: Abdullah Alamoudi 
Authored: Wed Jul 11 14:40:04 2018 -0700
Committer: abdullah alamoudi 
Committed: Fri Jul 13 12:53:29 2018 -0700

--
 .../org/apache/asterix/test/common/TestExecutor.java |  9 +
 .../testframework/context/TestCaseContext.java   | 15 +++
 .../testframework/context/TestFileContext.java   |  7 +++
 3 files changed, 27 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/26457e36/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 3c51775..5aff17f 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -30,7 +30,6 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.math.BigDecimal;
 import java.net.Inet4Address;
 import java.net.InetSocketAddress;
 import java.net.Socket;
@@ -97,8 +96,6 @@ import org.apache.hyracks.util.StorageUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-
-import com.fasterxml.jackson.databind.util.RawValue;
 import org.junit.Assert;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -107,6 +104,7 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.util.RawValue;
 
 public class TestExecutor {
 
@@ -1909,8 +1907,9 @@ public class TestExecutor {
 }
 }
 
-abstract static class TestLoop extends Exception {
+public abstract static class TestLoop extends Exception {
 
+private static final long serialVersionUID = 1L;
 private final String target;
 
 TestLoop(String target) {
@@ -1920,6 +1919,7 @@ public class TestExecutor {
 static TestLoop createLoop(String target, final int count) {
 LOGGER.info("Starting loop '" + count + " times back to '" + 
target + "'...");
 return new TestLoop(target) {
+private static final long serialVersionUID = 1L;
 int remainingLoops = count;
 
 @Override
@@ -1935,6 +1935,7 @@ public class TestExecutor {
 static TestLoop createLoop(String target, long duration, TimeUnit 
unit) {
 LOGGER.info("Starting loop for " + unit.toSeconds(duration) + "s 
back to '" + target + "'...");
 return new TestLoop(target) {
+private static final long serialVersionUID = 1L;
 long endTime = unit.toMillis(duration) + 
System.currentTimeMillis();
 
 @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/26457e36/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
--
diff --git 
a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
 
b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
index 4735c67..4357f14 100644
--- 
a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
+++ 
b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
@@ -22,7 +22,9 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 
 import

[2/2] asterixdb git commit: [NO ISSUE][FUN] Implement array functions p1

2018-07-13 Thread amoudi

[NO ISSUE][FUN] Implement array functions p1

- user model changes: no
- storage format changes: no
- interface changes: no

details:
This is part of implementing array functions. This patch
includes array_insert(), array_append(), array_prepend()
array_put() and array_remove(). It includes re-organization
of those functions.

Change-Id: I7d9cb80325138daf99fb039793446d109481c94b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2751
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/0d7974f2
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/0d7974f2
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/0d7974f2

Branch: refs/heads/master
Commit: 0d7974f27d6d922e597a3cfce52dd6f856ea2ac6
Parents: 2fff15a
Author: Ali Alsuliman 
Authored: Thu Jul 12 10:42:48 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Jul 12 23:42:11 2018 -0700

--
 .../org/apache/asterix/runtime/ExceptionIT.java |   9 +-
 .../apache/asterix/runtime/NullMissingTest.java |   8 +-
 .../array_append/array_append.3.query.sqlpp |  13 +-
 .../array_insert/array_insert.1.ddl.sqlpp   |  46 
 .../array_insert/array_insert.2.update.sqlpp|  22 ++
 .../array_insert/array_insert.3.query.sqlpp |  43 
 .../array_insert/array_insert.4.ddl.sqlpp   |  20 ++
 .../array_prepend/array_prepend.1.ddl.sqlpp |  56 +
 .../array_prepend/array_prepend.2.update.sqlpp  |  27 +++
 .../array_prepend/array_prepend.3.query.sqlpp   |  32 +++
 .../array_prepend/array_prepend.4.ddl.sqlpp |  20 ++
 .../array_fun/array_put/array_put.1.ddl.sqlpp   |  56 +
 .../array_put/array_put.2.update.sqlpp  |  27 +++
 .../array_fun/array_put/array_put.3.query.sqlpp |  33 +++
 .../array_fun/array_put/array_put.4.query.sqlpp |  22 ++
 .../array_fun/array_put/array_put.5.query.sqlpp |  22 ++
 .../array_fun/array_put/array_put.6.ddl.sqlpp   |  20 ++
 .../array_remove/array_remove.1.ddl.sqlpp   |  56 +
 .../array_remove/array_remove.2.update.sqlpp|  27 +++
 .../array_remove/array_remove.3.query.sqlpp |  32 +++
 .../array_remove/array_remove.4.query.sqlpp |  22 ++
 .../array_remove/array_remove.5.query.sqlpp |  22 ++
 .../array_remove/array_remove.6.ddl.sqlpp   |  20 ++
 .../array_fun/array_append/array_append.3.adm   |   2 +-
 .../array_fun/array_insert/array_insert.3.adm   |   1 +
 .../array_fun/array_prepend/array_prepend.3.adm |   1 +
 .../results/array_fun/array_put/array_put.3.adm |   1 +
 .../array_fun/array_remove/array_remove.3.adm   |   1 +
 .../resources/runtimets/testsuite_sqlpp.xml |  24 +++
 .../asterix/om/functions/BuiltinFunctions.java  |  18 +-
 .../impl/AListFirstTypeComputer.java|  50 +
 .../om/typecomputer/impl/AListTypeComputer.java |  59 +-
 .../impl/ArrayAppendTypeComputer.java   |  54 -
 .../asterix/om/typecomputer/ExceptionTest.java  |  11 +-
 .../om/typecomputer/TypeComputerTest.java   |  26 ++-
 .../functions/AbstractArrayAddRemoveEval.java   | 209 +++
 .../functions/ArrayAppendDescriptor.java| 105 ++
 .../functions/ArrayInsertDescriptor.java| 129 
 .../functions/ArrayPrependDescriptor.java   |  90 
 .../functions/ArrayPutDescriptor.java   | 136 
 .../functions/ArrayRemoveDescriptor.java| 123 +++
 .../evaluators/functions/CastTypeEvaluator.java |  15 +-
 .../evaluators/functions/PointableHelper.java   |   6 +-
 .../runtime/functions/FunctionCollection.java   |  10 +-
 44 files changed, 1529 insertions(+), 197 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0d7974f2/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
index e764c32..2e7445f 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
@@ -50,9 +50,8 @@ public class ExceptionIT {
 String className = func.getClass().getName();
 // We test all generated functions except
 // record and cast functions, which requires type settings.
-String[] splits = className.split("\\.");
-if (className.contains("Gen") && !className.contains("record") && 
!className.contains("Cast")
-&& !splits[splits.length - 1].startsWith("Array")) {
+if

asterixdb git commit: [NO ISSUE][ING] Set state to stop on failure of recovery

2018-07-12 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 30ab3cac8 -> 2fff15a89


[NO ISSUE][ING] Set state to stop on failure of recovery

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Before this change, if a rebalance comes at the time when
  an active recovery retry policy decides to give up, then
  the recovery task would finish while the entity is in the
  suspended state. When the rebalance completes, the resume
  will set the entity back to the temp failure state but
  recovery has given up already.
- To fix this, recovery task doesn't return in case of suspend and
  wait for the resume call to set the state to stopped.

Change-Id: I5a60a4f547a5b7f2c4c5199d48bcc83f5a2e9ccd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2757
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2fff15a8
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2fff15a8
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2fff15a8

Branch: refs/heads/master
Commit: 2fff15a89807ef66db01f310e950412786e5162b
Parents: 30ab3ca
Author: Abdullah Alamoudi 
Authored: Thu Jul 12 15:12:46 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Jul 12 21:19:36 2018 -0700

--
 .../main/java/org/apache/asterix/app/active/RecoveryTask.java | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2fff15a8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
index 9531b63..5d722e7 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
@@ -154,7 +154,12 @@ public class RecoveryTask {
 // Recovery task is essntially over now either through failure or 
through cancellation(stop)
 synchronized (listener) {
 listener.notifyAll();
-if (listener.getState() != ActivityState.TEMPORARILY_FAILED) {
+if (listener.getState() != ActivityState.TEMPORARILY_FAILED
+// Suspend can happen at the same time, the recovery 
policy decides to stop... in that case, we
+// must still do two things:
+// 1. set the state to permanent failure.
+// 2. set the entity to not running to avoid auto recovery 
attempt
+&& listener.getState() != ActivityState.SUSPENDED) {
 return null;
 }
 }

asterixdb git commit: [NO ISSUE][ING] Cleanup active listener

2018-07-12 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master bdbb5a36b -> 9a1b8b753


[NO ISSUE][ING] Cleanup active listener

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Merge STOPPED and PERMANENT_FAILURE into a single
  state since they are equivalent.
- Allow timeout of start and stop of active entities.
- Add test cases for timeout scenarios.
- Refactor common code into the abstract listener class.

Change-Id: I7f3b14aec46728fbe8b256b915d0e30992b2fe47
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2618
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/9a1b8b75
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/9a1b8b75
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/9a1b8b75

Branch: refs/heads/master
Commit: 9a1b8b753d6e35c8906cada71e22d6893a52af5c
Parents: bdbb5a3
Author: Abdullah Alamoudi 
Authored: Tue Jul 10 10:35:04 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Jul 12 14:14:12 2018 -0700

--
 .../apache/asterix/active/ActivityState.java|   9 +-
 .../active/IActiveEntityEventSubscriber.java|   4 +-
 .../active/IActiveEntityEventsListener.java |   4 +-
 .../app/active/ActiveEntityEventsListener.java  | 196 ++-
 .../asterix/app/active/FeedEventsListener.java  |  68 +++
 .../apache/asterix/app/active/RecoveryTask.java |   4 +-
 .../apache/asterix/utils/FeedOperations.java|  19 --
 .../test/active/ActiveEventsListenerTest.java   | 101 +++---
 .../test/active/DummyFeedEventsListener.java|   7 +-
 .../asterix/test/active/TestEventsListener.java | 113 ++-
 .../external/feed/watch/AbstractSubscriber.java |  13 ++
 .../external/feed/watch/NoOpSubscriber.java |   3 +-
 .../external/feed/watch/StatsSubscriber.java|  16 +-
 .../feed/watch/WaitForStateSubscriber.java  |  21 +-
 .../java/org/apache/hyracks/util/ExitUtil.java  |   1 +
 15 files changed, 346 insertions(+), 233 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/9a1b8b75/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivityState.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivityState.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivityState.java
index eb43d10..1f3daac 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivityState.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivityState.java
@@ -24,11 +24,6 @@ public enum ActivityState {
  */
 STOPPED,
 /**
- * Failure to recover from a temporary faliure caused the activity to fail 
permanantly.
- * No further recovery attempts will be made.
- */
-PERMANENTLY_FAILED,
-/**
  * An unexpected failure caused the activity to fail but recovery attempts 
will start taking place
  */
 TEMPORARILY_FAILED,
@@ -41,6 +36,10 @@ public enum ActivityState {
  */
 STARTING,
 /**
+ * The activity was started but is being cancelled. Waiting for the job 
cancellation to complete
+ */
+CANCELLING,
+/**
  * The activity has been started successfully and is running
  */
 RUNNING,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/9a1b8b75/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveEntityEventSubscriber.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveEntityEventSubscriber.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveEntityEventSubscriber.java
index 142e84b..e01d0a7 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveEntityEventSubscriber.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveEntityEventSubscriber.java
@@ -31,7 +31,7 @@ public interface IActiveEntityEventSubscriber {
  * @param event
  * @throws HyracksDataException
  */
-void notify(ActiveEvent event) throws HyracksDataException;
+void notify(ActiveEvent event);
 
 /**
  * Checkcs whether the subscriber is done receiving events
@@ -53,5 +53,5 @@ public interface IActiveEntityEventSubscriber {
  * @param eventsListener
  * @throws HyracksDataException
  */
-void subscribed(IActiveEntityEventsListener eventsListener) throws 
HyracksDataException;
+void subscribed(IActiveEntityEventsListener eventsListener);
 }

asterixdb-bad git commit: Remove unneeded throws declaration

2018-07-12 Thread amoudi

Repository: asterixdb-bad
Updated Branches:
  refs/heads/master 7da30159b -> 5364a835c


Remove unneeded throws declaration

Change-Id: I20a6effa1f01658ec973f85cb7be1e4ccbd7fc1b


Project: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/commit/5364a835
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/tree/5364a835
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb-bad/diff/5364a835

Branch: refs/heads/master
Commit: 5364a835c798148c96b289b78dada87669054c7f
Parents: 7da3015
Author: Abdullah Alamoudi 
Authored: Tue Jul 10 10:37:29 2018 -0700
Committer: Abdullah Alamoudi 
Committed: Tue Jul 10 10:37:29 2018 -0700

--
 .../apache/asterix/bad/metadata/DeployedJobSpecEventListener.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb-bad/blob/5364a835/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/DeployedJobSpecEventListener.java
--
diff --git 
a/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/DeployedJobSpecEventListener.java
 
b/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/DeployedJobSpecEventListener.java
index 4598d71..3ec2c43 100644
--- 
a/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/DeployedJobSpecEventListener.java
+++ 
b/asterix-bad/src/main/java/org/apache/asterix/bad/metadata/DeployedJobSpecEventListener.java
@@ -110,7 +110,6 @@ public class DeployedJobSpecEventListener implements 
IActiveEntityEventsListener
 this.executorService = ses;
 }
 
-
 public ScheduledExecutorService getExecutorService() {
 return executorService;
 }
@@ -192,7 +191,7 @@ public class DeployedJobSpecEventListener implements 
IActiveEntityEventsListener
 }
 
 @Override
-public synchronized void subscribe(IActiveEntityEventSubscriber 
subscriber) throws HyracksDataException {
+public synchronized void subscribe(IActiveEntityEventSubscriber 
subscriber) {
 // no op
 }

[1/2] asterixdb git commit: [NO ISSUE]Introduce convenience methods for pointables

2018-07-03 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master ac3a16a5e -> 5ff3230e7


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff3230e/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index 8500842..5492919 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -35,6 +35,9 @@ public class ExitUtil {
 public static final int EC_FAILED_TO_RECOVER = 3;
 public static final int EC_NC_FAILED_TO_ABORT_ALL_PREVIOUS_TASKS = 4;
 public static final int EC_FAILED_TO_PROCESS_UN_INTERRUPTIBLE_REQUEST = 5;
+public static final int EC_FAILED_TO_COMMIT_METADATA_TXN = 6;
+public static final int EC_FAILED_TO_ABORT_METADATA_TXN = 7;
+public static final int EC_INCONSISTENT_METADATA = 8;
 public static final int EC_UNHANDLED_EXCEPTION = 11;
 public static final int EC_IMMEDIATE_HALT = 33;
 public static final int EC_HALT_ABNORMAL_RESERVED_44 = 44;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff3230e/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
index 1863b28..e6e21fa 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
@@ -88,7 +88,7 @@ public class NetworkUtil {
 }
 
 public static InetSocketAddress toInetSocketAddress(String 
maybeLiteralHost, int port) {
-return 
InetSocketAddress.createUnresolved(decodeIPv6LiteralHost(maybeLiteralHost), 
port);
+return new InetSocketAddress(decodeIPv6LiteralHost(maybeLiteralHost), 
port);
 }
 
 public static List parseInetSocketAddresses(String... 
hostPortStrings) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff3230e/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index cdb82bb..5597058 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -294,6 +294,11 @@ public class UTF8StringUtil {
 return hash(bytes, start, length, true, false, 31, Integer.MAX_VALUE);
 }
 
+public static String toString(byte[] bytes, int start) {
+StringBuilder builder = new StringBuilder();
+return toString(builder, bytes, start).toString();
+}
+
 public static StringBuilder toString(StringBuilder builder, byte[] bytes, 
int start) {
 int utfLen = getUTFLength(bytes, start);
 int offset = getNumBytesToStoreLength(utfLen);
@@ -650,10 +655,10 @@ public class UTF8StringUtil {
 if (writer == null) {
 tempBytes = new byte[utflen + 5];
 } else {
-if (writer.tempBytes == null || writer.tempBytes.length < utflen + 
5) {
-writer.tempBytes = new byte[utflen + 5];
+if (writer.getTempBytes() == null || writer.getTempBytes().length 
< utflen + 5) {
+writer.setTempBytes(new byte[utflen + 5]);
 }
-tempBytes = writer.tempBytes;
+tempBytes = writer.getTempBytes();
 }
 return tempBytes;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff3230e/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
index 485bd7d..3d9a7c5 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++

[2/2] asterixdb git commit: [NO ISSUE]Introduce convenience methods for pointables

2018-07-03 Thread amoudi

port org.apache.asterix.common.transactions.ITransactionContext;
 import 
org.apache.asterix.common.transactions.ITransactionManager.AtomicityLevel;
 import org.apache.asterix.common.transactions.ITransactionSubsystem;
 import org.apache.asterix.common.transactions.ITxnIdFactory;
-import org.apache.asterix.common.transactions.ImmutableDatasetId;
 import org.apache.asterix.common.transactions.TransactionOptions;
 import org.apache.asterix.common.transactions.TxnId;
 import org.apache.asterix.common.utils.StoragePathUtil;
@@ -124,10 +121,13 @@ import org.apache.hyracks.storage.common.IIndexAccessor;
 import org.apache.hyracks.storage.common.IIndexCursor;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class MetadataNode implements IMetadataNode {
 private static final long serialVersionUID = 1L;
-
+private static final Logger LOGGER = LogManager.getLogger();
 // shared between core and extension
 private IDatasetLifecycleManager datasetLifecycleManager;
 private ITransactionSubsystem transactionSubsystem;
@@ -172,14 +172,28 @@ public class MetadataNode implements IMetadataNode {
 
transactionSubsystem.getTransactionManager().beginTransaction(transactionId, 
options);
 }
 
+@SuppressWarnings("squid:S1181")
 @Override
 public void commitTransaction(TxnId txnId) throws RemoteException, 
ACIDException {
-transactionSubsystem.getTransactionManager().commitTransaction(txnId);
+try {
+
transactionSubsystem.getTransactionManager().commitTransaction(txnId);
+} catch (Throwable th) {
+// Metadata node should abort all Metadata transactions on re-start
+LOGGER.fatal("Failure committing a metadata transaction", th);
+ExitUtil.halt(ExitUtil.EC_FAILED_TO_COMMIT_METADATA_TXN);
+}
 }
 
+@SuppressWarnings("squid:S1181")
 @Override
 public void abortTransaction(TxnId txnId) throws RemoteException, 
ACIDException {
-transactionSubsystem.getTransactionManager().abortTransaction(txnId);
+try {
+
transactionSubsystem.getTransactionManager().abortTransaction(txnId);
+} catch (Throwable th) {
+// Metadata node should abort all uncommitted transactions on 
re-start
+LOGGER.fatal("Failure committing a metadata transaction", th);
+ExitUtil.halt(ExitUtil.EC_FAILED_TO_ABORT_METADATA_TXN);
+}
 }
 
 // TODO(amoudi): make all metadata operations go through the generic 
methods

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff3230e/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
--
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index bcd05ee..d0a22b7 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -127,9 +127,6 @@ public class Dataset implements IMetadataEntity, 
IDataset {
  */
 private static final long serialVersionUID = 1L;
 private static final Logger LOGGER = LogManager.getLogger();
-//TODO: Remove Singletons
-private static final BTreeResourceFactoryProvider 
bTreeResourceFactoryProvider =
-BTreeResourceFactoryProvider.INSTANCE;
 private static final RTreeResourceFactoryProvider 
rTreeResourceFactoryProvider =
 RTreeResourceFactoryProvider.INSTANCE;
 private static final InvertedIndexResourceFactoryProvider 
invertedIndexResourceFactoryProvider =
@@ -153,10 +150,6 @@ public class Dataset implements IMetadataEntity, 
IDataset {
 private final long rebalanceCount;
 private int pendingOp;
 
-/*
- * Transient (For caching)
- */
-
 public Dataset(String dataverseName, String datasetName, String 
recordTypeDataverseName, String recordTypeName,
 String nodeGroupName, String compactionPolicy, Map 
compactionPolicyProperties,
 IDatasetDetails datasetDetails, Map hints, 
DatasetType datasetType, int datasetId,
@@ -473,8 +466,9 @@ public class Dataset implements IMetadataEntity, 
IDataset {
 IResourceFactory resourceFactory;
 switch (index.getIndexType()) {
 case BTREE:
-resourceFactory = 
bTreeResourceFactoryProvider.getResourceFactory(mdProvider, this, index, 
recordType,
-metaType, mergePolicyFactory, mergePolicyProperties, 
filterTypeTraits, filterCmpFactories);
+resourceFacto

asterixdb git commit: [NO ISSUE] Improve Span

2018-06-11 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master d8ed19620 -> 99e4d2644


[NO ISSUE] Improve Span

Change-Id: I4378332af6b18720f5abc60259bb9af81f22219b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2694
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/99e4d264
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/99e4d264
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/99e4d264

Branch: refs/heads/master
Commit: 99e4d26441ad512a087194fd723edddf646dfee0
Parents: d8ed196
Author: Abdullah Alamoudi 
Authored: Sun Jun 10 04:39:17 2018 -0700
Committer: abdullah alamoudi 
Committed: Mon Jun 11 22:32:38 2018 -0700

--
 asterixdb/asterix-active/pom.xml|  4 ++
 .../apache/asterix/active/CountRetryPolicy.java | 39 ---
 .../asterix/active/CountRetryPolicyFactory.java |  3 ++
 .../org/apache/asterix/active/IRetryPolicy.java | 29 ---
 .../asterix/active/IRetryPolicyFactory.java |  2 +
 .../asterix/active/InfiniteRetryPolicy.java |  2 +
 .../active/InfiniteRetryPolicyFactory.java  |  2 +
 .../asterix/active/NoRetryPolicyFactory.java|  6 ++-
 .../apache/asterix/app/active/RecoveryTask.java |  2 +-
 .../org/apache/hyracks/api/util/InvokeUtil.java | 32 +++-
 .../apache/hyracks/util/ComputingAction.java| 24 +
 .../apache/hyracks/util/CountRetryPolicy.java   | 39 +++
 .../java/org/apache/hyracks/util/IDelay.java| 26 ++
 .../org/apache/hyracks/util/IRetryPolicy.java   | 29 +++
 .../org/apache/hyracks/util/NoRetryPolicy.java  | 32 
 .../main/java/org/apache/hyracks/util/Span.java | 51 +++-
 16 files changed, 248 insertions(+), 74 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/99e4d264/asterixdb/asterix-active/pom.xml
--
diff --git a/asterixdb/asterix-active/pom.xml b/asterixdb/asterix-active/pom.xml
index a6f6a38..922a0e8 100644
--- a/asterixdb/asterix-active/pom.xml
+++ b/asterixdb/asterix-active/pom.xml
@@ -35,6 +35,10 @@
 
 
   org.apache.hyracks
+  hyracks-util
+
+
+  org.apache.hyracks
   hyracks-api
 
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/99e4d264/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
deleted file mode 100644
index b964430..000
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.active;
-
-public class CountRetryPolicy implements IRetryPolicy {
-
-private final int count;
-private int attempted = 0;
-
-public CountRetryPolicy(int count) {
-this.count = count;
-}
-
-@Override
-public boolean retry(Throwable failure) {
-if (attempted < count) {
-attempted++;
-return true;
-}
-return false;
-}
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/99e4d264/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicyFactory.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicyFactory.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicyFactory.java
index 5e26ae4..477bb5a 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicyFactory.java
+++

asterixdb git commit: [NO ISSUE][ING] Improvements to active retry policy

2018-06-06 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 5c28118b5 -> 7c7ce10f5


[NO ISSUE][ING] Improvements to active retry policy

- user model changes: no
- storage format changes: no
- interface changes: yes
  - IRetryPolicy.retry now takes a Throwable parameter

Details:
- This change improves the retry policy for active
  entities by providing the failure causing the last
  failure.
- The change also removes the lock over the active
  notification handler on the recover call.

Change-Id: I4246e2a276e1f80569a07630e182aab8f49dd115
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2665
Reviewed-by: Michael Blow 
Contrib: Michael Blow 
Integration-Tests: Michael Blow 
Tested-by: Michael Blow 
Contrib: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7c7ce10f
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7c7ce10f
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7c7ce10f

Branch: refs/heads/master
Commit: 7c7ce10f563dd23af8079d836dda17b77a9a2729
Parents: 5c28118
Author: Abdullah Alamoudi 
Authored: Wed Jun 6 21:09:52 2018 +0300
Committer: abdullah alamoudi 
Committed: Wed Jun 6 14:19:42 2018 -0700

--
 .../apache/asterix/active/CountRetryPolicy.java |  2 +-
 .../org/apache/asterix/active/IRetryPolicy.java |  4 +-
 .../asterix/active/InfiniteRetryPolicy.java |  2 +-
 .../asterix/active/NoRetryPolicyFactory.java|  2 +-
 .../app/active/ActiveNotificationHandler.java   |  4 +-
 .../apache/asterix/app/active/RecoveryTask.java |  5 +-
 .../lang/common/util/LangRecordParseUtil.java   |  2 +
 .../hyracks/api/client/HyracksConnection.java   | 80 +---
 .../nc/work/EnsureAllCcTasksCompleted.java  |  4 +-
 .../java/org/apache/hyracks/util/ExitUtil.java  |  6 +-
 10 files changed, 70 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7c7ce10f/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
index 6633810..b964430 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/CountRetryPolicy.java
@@ -28,7 +28,7 @@ public class CountRetryPolicy implements IRetryPolicy {
 }
 
 @Override
-public boolean retry() {
+public boolean retry(Throwable failure) {
 if (attempted < count) {
 attempted++;
 return true;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7c7ce10f/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IRetryPolicy.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IRetryPolicy.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IRetryPolicy.java
index a010984..1daf07e 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IRetryPolicy.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IRetryPolicy.java
@@ -21,7 +21,9 @@ package org.apache.asterix.active;
 @FunctionalInterface
 public interface IRetryPolicy {
 /**
+ * @param failure
+ *the cause of the active entity failure
  * @return true if one more attempt should be done
  */
-boolean retry();
+boolean retry(Throwable failure);
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7c7ce10f/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java
index 074f8f4..fde67e6 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java
@@ -27,7 +27,7 @@ public class InfiniteRetryPolicy implements IRetryPolicy {
 }
 
 @Override
-public boolean retry() {
+public boolean retry(Throwable failure) {
 synchronized (listener) {
 try {
 listener.wait(5000); //NOSONAR this method is being called in 
a while loop

[1/2] asterixdb git commit: [NO ISSUE][TEST] Use reflection in ADMDataParserTest

2018-06-04 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 5c26b4258 -> b54f9bffa


[NO ISSUE][TEST] Use reflection in ADMDataParserTest

Change-Id: I1f88684693230156f27327882f65f05ea3105a8e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2685
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2a182c49
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2a182c49
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2a182c49

Branch: refs/heads/master
Commit: 2a182c495d311757f1aa9291ca407c8068f00ff2
Parents: e2a536e
Author: Abdullah Alamoudi 
Authored: Sun Jun 3 03:19:00 2018 +0300
Committer: Michael Blow 
Committed: Sun Jun 3 11:23:40 2018 -0700

--
 .../external/parser/test/ADMDataParserTest.java | 45 ++--
 1 file changed, 32 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2a182c49/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ADMDataParserTest.java
--
diff --git 
a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ADMDataParserTest.java
 
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ADMDataParserTest.java
index f06528e..1485f4f 100644
--- 
a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ADMDataParserTest.java
+++ 
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ADMDataParserTest.java
@@ -22,21 +22,22 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.asterix.external.parser.ADMDataParser;
+import org.apache.asterix.external.parser.AbstractDataParser;
 import org.apache.asterix.om.base.AMutableDate;
 import org.apache.asterix.om.base.AMutableDateTime;
 import org.apache.asterix.om.base.AMutableTime;
 import org.junit.Assert;
 import org.junit.Test;
 
-import junit.extensions.PA;
-
 public class ADMDataParserTest {
 
 @Test
-public void test() throws IOException {
+public void test() throws IOException, NoSuchMethodException, 
SecurityException, NoSuchFieldException {
 String[] dates = { "-9537-08-04", "9656-06-03", "-9537-04-04", 
"9656-06-04", "-9537-10-04", "9626-09-05" };
 AMutableDate[] parsedDates =
 new AMutableDate[] { new AMutableDate(-4202630), new 
AMutableDate(2807408), new AMutableDate(-4202752),
@@ -55,6 +56,27 @@ public class ADMDataParserTest {
 new AMutableDateTime(-45286270768513L), new 
AMutableDateTime(151729886421653L),
 new AMutableDateTime(5047449515758L), new 
AMutableDateTime(210721439419691L) };
 
+Method parseDateMethod =
+AbstractDataParser.class.getDeclaredMethod("parseDate", 
String.class, DataOutput.class);
+parseDateMethod.setAccessible(true);
+
+Field aDateField = AbstractDataParser.class.getDeclaredField("aDate");
+aDateField.setAccessible(true);
+
+Method parseTimeMethod =
+AbstractDataParser.class.getDeclaredMethod("parseTime", 
String.class, DataOutput.class);
+parseTimeMethod.setAccessible(true);
+
+Field aTimeField = AbstractDataParser.class.getDeclaredField("aTime");
+aTimeField.setAccessible(true);
+
+Method parseDateTimeMethod =
+AbstractDataParser.class.getDeclaredMethod("parseDateTime", 
String.class, DataOutput.class);
+parseDateTimeMethod.setAccessible(true);
+
+Field aDateTimeField = 
AbstractDataParser.class.getDeclaredField("aDateTime");
+aDateTimeField.setAccessible(true);
+
 Thread[] threads = new Thread[16];
 AtomicInteger errorCount = new AtomicInteger(0);
 for (int i = 0; i < threads.length; ++i) {
@@ -70,25 +92,22 @@ public class ADMDataParserTest {
 while (round++ < 1) {
 // Test parseDate.
 for (int index = 0; index < dates.length; ++index) 
{
-PA.invokeMethod(parser, 
"parseDate(java.lang.String, java.io.DataOutput)", dates[index],
-dos);
-AMutableDate aDate = (AMutableDate) 
PA.getValue(parser, "aDate");
+parseDateMethod.invoke(parser, dates[index], 
dos);
+AMutableDate aDate = (AMutableDate)

asterixdb git commit: [NO ISSUE][ING] Allow Readers to add their own stats

2018-06-01 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master bc7e16e05 -> dbae78f5a


[NO ISSUE][ING] Allow Readers to add their own stats

- user model changes: no
- storage format changes: no
- interface changes: yes
  - Add IRecordReader.getStats()

Details:
- Allow record reader to contribute their own stats.

Change-Id: Ib064bd97b33f2d9da330bac3a49bd4c6831faaca
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2642
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/dbae78f5
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/dbae78f5
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/dbae78f5

Branch: refs/heads/master
Commit: dbae78f5a0a412bb95158627b2fe7e17488b233c
Parents: bc7e16e
Author: Abdullah Alamoudi 
Authored: Thu May 31 23:47:00 2018 +0300
Committer: abdullah alamoudi 
Committed: Fri Jun 1 01:23:34 2018 -0700

--
 .../active/ActiveSourceOperatorNodePushable.java   |  4 +++-
 .../org/apache/asterix/external/api/IRecordReader.java |  7 +++
 .../dataflow/FeedRecordDataFlowController.java | 13 +++--
 3 files changed, 21 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
index 1df9020..2f30df4 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
@@ -63,8 +63,10 @@ public abstract class ActiveSourceOperatorNodePushable 
extends AbstractUnaryOutp
 
 @Override
 public final void stop() throws HyracksDataException, InterruptedException 
{
-abort();
 synchronized (this) {
+if (!done) {
+abort();
+}
 while (!done) {
 wait();
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index 08ffe18..9033814 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -77,4 +77,11 @@ public interface IRecordReader extends Closeable {
 public default IFeedMarker getProgressReporter() {
 return null;
 }
+
+/**
+ * @return JSON String containing ingestion stats
+ */
+default String getStats() {
+return null;
+}
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dbae78f5/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 164ff68..f392139 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -40,6 +40,7 @@ import org.apache.logging.log4j.Logger;
 public class FeedRecordDataFlowController extends 
AbstractFeedDataFlowController {
 public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = 
"incoming-records-count";
 public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = 
"failed-at-parser-records-count";
+public static final String READER_STATS_FIELD_NAME = "reader-stats";
 
 public enum State {
 CREATED,
@@ -257,7 +258,15 @@ public class FeedRecordDataFlowController extends 
AbstractFeedDataFlowControl
 
 @Override
 public String getStats() {
-return "{\"" + INCOMING_RECORDS_COUNT_FIELD_NAME + "\": " +

asterixdb git commit: [NO ISSUE][RT] Make start and cancel job uninterruptible

2018-05-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 8eb7413b7 -> 1102ed587


[NO ISSUE][RT] Make start and cancel job uninterruptible

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Previously, a Hyracks data connection that starts
  a job might be interrupted before it receives the
  job id. This would leak resources since that job
  will run even though, no one is going to read its
  result.
- Similarly, job cancellation can be interrupted and
  so the job which was meant to be cancelled will
  continue running.
- To avoid this, a new thread is added to Hyracks
  Client Connection which takes care of starting
  and cancelling of jobs. The thread submitting these
  requests will be un-interruptible until those calls
  return.

Change-Id: I27b2aaae902b19829bd2df2ae04c5e704f5ca8e8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2639
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/1102ed58
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/1102ed58
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/1102ed58

Branch: refs/heads/master
Commit: 1102ed587d81b88bd2c6c8e012ccda5be47c2350
Parents: 8eb7413
Author: Abdullah Alamoudi 
Authored: Fri May 25 00:21:43 2018 +0300
Committer: abdullah alamoudi 
Committed: Sun May 27 18:51:45 2018 -0700

--
 .../external/indexing/IndexingScheduler.java|   9 +-
 .../apache/asterix/external/util/HDFSUtils.java |   7 +-
 .../hyracks/api/client/HyracksConnection.java   | 196 ++-
 .../java/org/apache/hyracks/util/ExitUtil.java  |   1 +
 4 files changed, 197 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1102ed58/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingScheduler.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingScheduler.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingScheduler.java
index edac0fa..f22693a 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingScheduler.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingScheduler.java
@@ -33,8 +33,6 @@ import java.util.Random;
 
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hyracks.api.client.HyracksConnection;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.HyracksException;
@@ -53,19 +51,14 @@ public class IndexingScheduler {
 /** a map from the NC name to the index */
 private Map ncNameToIndex = new HashMap();
 
-/** a map from NC name to the NodeControllerInfo */
-private Map ncNameToNcInfos;
-
 /**
  * The constructor of the scheduler.
  *
  * @param ncNameToNcInfos
  * @throws HyracksException
  */
-public IndexingScheduler(String ipAddress, int port) throws 
HyracksException {
+public IndexingScheduler(Map ncNameToNcInfos) 
throws HyracksException {
 try {
-IHyracksClientConnection hcc = new HyracksConnection(ipAddress, 
port);
-this.ncNameToNcInfos = hcc.getNodeControllerInfos();
 loadIPAddressToNCMap(ncNameToNcInfos);
 } catch (Exception e) {
 throw HyracksException.create(e);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1102ed58/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
index bd50352..1b13ec5 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;

asterixdb git commit: [NO ISSUE][STO] Improve logging when failing to schedule a flush

2018-05-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master af17baa7d -> 8eb7413b7


[NO ISSUE][STO] Improve logging when failing to schedule a flush

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Log partition info when failing to schedule a flush operation.
- Fix isCurrentMutableComponentEmpty() to consider the
  UNREADABLE_UNWRITABLE state.

Change-Id: I9bbc0be36633b896becc16d3eeddeef980db5802
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2662
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8eb7413b
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8eb7413b
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8eb7413b

Branch: refs/heads/master
Commit: 8eb7413b7b65f78e2687d3f07bc3d0fac7db4863
Parents: af17baa
Author: Abdullah Alamoudi 
Authored: Mon May 28 01:35:39 2018 +0300
Committer: abdullah alamoudi 
Committed: Sun May 27 18:50:37 2018 -0700

--
 .../context/PrimaryIndexOperationTracker.java   | 21 +---
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  2 +-
 2 files changed, 19 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8eb7413b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 52c8962..0aa0d2b 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -51,8 +51,11 @@ import 
org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class PrimaryIndexOperationTracker extends BaseOperationTracker 
implements IoOperationCompleteListener {
+private static final Logger LOGGER = LogManager.getLogger();
 private final int partition;
 // Number of active operations on an ILSMIndex instance.
 private final AtomicInteger numActiveOperations;
@@ -113,23 +116,34 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker implement
 }
 }
 
+ILSMIndex primaryLsmIndex = null;
 if (needsFlush || flushOnExit) {
 flushOnExit = false;
 // make the current mutable components READABLE_UNWRITABLE to stop 
coming modify operations from entering
 // them until the current flush is scheduled.
 LSMComponentId primaryId = null;
 //Double check that the primary index has been modified
+
 synchronized (this) {
 if (numActiveOperations.get() > 0) {
 throw new IllegalStateException(
 "Can't request a flush on an index with active 
operations: " + numActiveOperations.get());
 }
 for (ILSMIndex lsmIndex : indexes) {
-if (lsmIndex.isPrimaryIndex() && 
lsmIndex.isCurrentMutableComponentEmpty()) {
-return;
+if (lsmIndex.isPrimaryIndex()) {
+if (lsmIndex.isCurrentMutableComponentEmpty()) {
+LOGGER.info("Primary index on dataset {} and 
partition {} is empty... skipping flush");
+return;
+}
+primaryLsmIndex = lsmIndex;
+break;
 }
 }
 }
+if (primaryLsmIndex == null) {
+throw new IllegalStateException(
+"Primary index not found in dataset " + 
dsInfo.getDatasetID() + " and partition " + partition);
+}
 for (ILSMIndex lsmIndex : indexes) {
 ILSMOperationTracker opTracker = 
lsmIndex.getOperationTracker();
 synchronized (opTracker) {
@@ -148,7

[3/5] asterixdb git commit: [NO ISSUE][STO] Misc Storage Fixes and Improvements

2018-05-20 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/10a3f21d/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 681669f..52c8962 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -19,7 +19,10 @@
 
 package org.apache.asterix.common.context;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -37,16 +40,19 @@ import 
org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import 
org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.ISearchOperationCallback;
 
-public class PrimaryIndexOperationTracker extends BaseOperationTracker {
+public class PrimaryIndexOperationTracker extends BaseOperationTracker 
implements IoOperationCompleteListener {
 private final int partition;
 // Number of active operations on an ILSMIndex instance.
 private final AtomicInteger numActiveOperations;
@@ -54,6 +60,7 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 private final ILSMComponentIdGenerator idGenerator;
 private boolean flushOnExit = false;
 private boolean flushLogCreated = false;
+private final Map scheduledFlushes = new 
HashMap<>();
 
 public PrimaryIndexOperationTracker(int datasetID, int partition, 
ILogManager logManager, DatasetInfo dsInfo,
 ILSMComponentIdGenerator idGenerator) {
@@ -107,6 +114,7 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 }
 
 if (needsFlush || flushOnExit) {
+flushOnExit = false;
 // make the current mutable components READABLE_UNWRITABLE to stop 
coming modify operations from entering
 // them until the current flush is scheduled.
 LSMComponentId primaryId = null;
@@ -143,7 +151,6 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 throw new IllegalStateException("Primary index not found in 
dataset " + dsInfo.getDatasetID());
 }
 LogRecord logRecord = new LogRecord();
-flushOnExit = false;
 if (dsInfo.isDurable()) {
 /*
  * Generate a FLUSH log.
@@ -182,16 +189,36 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 Map flushMap = new HashMap<>();
 flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
 flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, 
nextComponentId);
-for (ILSMIndex lsmIndex : 
dsInfo.getDatasetPartitionOpenIndexes(partition)) {
-ILSMIndexAccessor accessor = 
lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE);
-accessor.getOpContext().setParameters(flushMap);
-accessor.scheduleFlush();
+synchronized (scheduledFlushes) {
+for (ILSMIndex lsmIndex : 
dsInfo.getDatasetPartitionOpenIndexes(partition)) {
+ILSMIndexAccessor accessor = 
lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+accessor.getOpContext().setParameters(flushMap);
+ILSMIOOperation flush = accessor.scheduleFlush();
+scheduledFlushes.put(flush.getTarget().getRelativePath(), 
(FlushOperation) flush);
+flush.addCompleteListener(this);
+}

[4/5] asterixdb git commit: [NO ISSUE][STO] Misc Storage Fixes and Improvements

2018-05-20 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/10a3f21d/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
new file mode 100644
index 000..a0ed26e
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
@@ -0,0 +1,577 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.asterix.app.bootstrap.TestNodeController;
+import org.apache.asterix.app.bootstrap.TestNodeController.PrimaryIndexInfo;
+import org.apache.asterix.app.bootstrap.TestNodeController.SecondaryIndexInfo;
+import org.apache.asterix.app.data.gen.RecordTupleGenerator;
+import org.apache.asterix.app.data.gen.RecordTupleGenerator.GenerationFunction;
+import org.apache.asterix.app.nc.NCAppRuntimeContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.storage.IIndexCheckpointManager;
+import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
+import org.apache.asterix.common.storage.IndexCheckpoint;
+import org.apache.asterix.common.storage.ResourceReference;
+import org.apache.asterix.common.transactions.ITransactionContext;
+import org.apache.asterix.common.transactions.ITransactionManager;
+import org.apache.asterix.common.transactions.TransactionOptions;
+import org.apache.asterix.external.util.DataflowUtils;
+import org.apache.asterix.file.StorageComponentProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import 
org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import 
org.apache.asterix.runtime.operators.LSMIndexBulkLoadOperatorNodePushable;
+import org.apache.asterix.test.common.TestHelper;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.util.SingleThreadEventProcessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import 
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.datagen.TupleGenerator;
+import org.apache.hyracks.storage.am.common.datagen.IFieldValueGenerator;
+import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId;
+import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import

[1/5] asterixdb git commit: [NO ISSUE][STO] Misc Storage Fixes and Improvements

2018-05-20 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 1a6de0b91 -> 10a3f21de


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/10a3f21d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index f3162c9..f902153 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -30,7 +31,6 @@ import 
org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
 import 
org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
 import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
@@ -48,8 +48,10 @@ import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
+import 
org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader;
 import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
 import 
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.LoadOperation;
 import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
 import org.apache.hyracks.storage.common.IIndexAccessParameters;
 import org.apache.hyracks.storage.common.IIndexBulkLoader;
@@ -433,131 +435,99 @@ public class ExternalRTree extends LSMRTree implements 
ITwoPCIndex {
 
 // For initial load
 @Override
-public IIndexBulkLoader createBulkLoader(float fillLevel, boolean 
verifyInput, long numElementsHint)
-throws HyracksDataException {
-return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, false);
+public IIndexBulkLoader createBulkLoader(float fillLevel, boolean 
verifyInput, long numElementsHint,
+Map parameters) throws HyracksDataException {
+return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, false, 
parameters);
 }
 
 // For transaction bulk load <- could consolidate with the above method ->
 @Override
-public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, 
boolean verifyInput, long numElementsHint)
-throws HyracksDataException {
-return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, true);
+public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, 
boolean verifyInput, long numElementsHint,
+Map parameters) throws HyracksDataException {
+return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, true, parameters);
 }
 
 // The bulk loader used for both initial loading and transaction
 // modifications
 public class LSMTwoPCRTreeBulkLoader implements IIndexBulkLoader, 
ITwoPCIndexBulkLoader {
 private final ILSMDiskComponent component;
-private final IIndexBulkLoader rtreeBulkLoader;
-private final BTreeBulkLoader btreeBulkLoader;
-private final IIndexBulkLoader builder;
-private boolean cleanedUpArtifacts = false;
-private boolean isEmptyComponent = true;
-private boolean endedBloomFilterLoad = false;
 private final boolean isTransaction;
+private final LoadOperation loadOp;
+private final ChainedLSMDiskComponentBulkLoader componentBulkLoader;
 
 public LSMTwoPCRTreeBulkLoader(float fillFactor, boolean verifyInput, 
long numElementsHint,
-boolean isTransaction) throws HyracksDataException {
+boolean isTransaction, Map parameters) throws

[2/5] asterixdb git commit: [NO ISSUE][STO] Misc Storage Fixes and Improvements

2018-05-20 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/10a3f21d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index cfffbe1..a504f7e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -395,16 +396,16 @@ public class ExternalBTree extends LSMBTree implements 
ITwoPCIndex {
 
 // For initial load
 @Override
-public IIndexBulkLoader createBulkLoader(float fillLevel, boolean 
verifyInput, long numElementsHint)
-throws HyracksDataException {
-return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, false);
+public IIndexBulkLoader createBulkLoader(float fillLevel, boolean 
verifyInput, long numElementsHint,
+Map parameters) throws HyracksDataException {
+return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, false, parameters);
 }
 
 // For transaction bulk load <- could consolidate with the above method ->
 @Override
-public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, 
boolean verifyInput, long numElementsHint)
-throws HyracksDataException {
-return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, true);
+public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, 
boolean verifyInput, long numElementsHint,
+Map parameters) throws HyracksDataException {
+return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, 
numElementsHint, true, parameters);
 }
 
 // The bulk loader used for both initial loading and transaction
@@ -412,19 +413,35 @@ public class ExternalBTree extends LSMBTree implements 
ITwoPCIndex {
 public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader, 
ITwoPCIndexBulkLoader {
 private final ILSMDiskComponent component;
 private final ILSMDiskComponentBulkLoader componentBulkLoader;
+private final LoadOperation loadOp;
 
 private final boolean isTransaction;
 
 public LSMTwoPCBTreeBulkLoader(float fillFactor, boolean verifyInput, 
long numElementsHint,
-boolean isTransaction) throws HyracksDataException {
+boolean isTransaction, Map parameters) throws 
HyracksDataException {
 this.isTransaction = isTransaction;
 // Create the appropriate target
+LSMComponentFileReferences componentFileRefs;
 if (isTransaction) {
-component = createTransactionTarget();
+try {
+componentFileRefs = 
fileManager.getNewTransactionFileReference();
+} catch (IOException e) {
+throw HyracksDataException.create(e);
+}
+component = createDiskComponent(transactionComponentFactory,
+componentFileRefs.getInsertIndexFileReference(), null,
+componentFileRefs.getBloomFilterFileReference(), true);
 } else {
-component = createBulkLoadTarget();
+componentFileRefs = fileManager.getRelFlushFileReference();
+component =
+createDiskComponent(bulkLoadComponentFactory, 
componentFileRefs.getInsertIndexFileReference(),
+
componentFileRefs.getDeleteIndexFileReference(),
+
componentFileRefs.getBloomFilterFileReference(), true);
 }
-LoadOperation loadOp = new LoadOperation(ioOpCallback, 
getIndexIdentifier());
+loadOp = new LoadOperation(componentFileRefs, ioOpCallback, 
getIndexIdentifier(), parameters);
+loadOp.setNewComponent(component);
+ioOpCallback.scheduled(loadOp);
+ioOpCallback.beforeOperation(loadOp);
 componentBulkLoader =
 component.createBulkLoader(loadOp, fillFactor, 
verifyInput, numElementsHint, false, true, true);
 }
@@ -438,16 +455,23 @@ public class

asterixdb git commit: [NO ISSUE][TEST] Fix Sporadic failures in CancellationTests

2018-04-18 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 07a0acc54 -> 4d6bce750


[NO ISSUE][TEST] Fix Sporadic failures in CancellationTests

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Multiple queries in a single request can cause an abrupt drop
  of connection from the server if the first query was processed
  successfully and then an error occurred.

Change-Id: I4df583e2afb0213361f420dc2490e1b5f3d8d835
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2606
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/4d6bce75
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/4d6bce75
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/4d6bce75

Branch: refs/heads/master
Commit: 4d6bce750b6621a72ba42499a34ee695efccb3a9
Parents: 07a0acc
Author: Abdullah Alamoudi 
Authored: Wed Apr 18 20:36:56 2018 +0300
Committer: abdullah alamoudi 
Committed: Wed Apr 18 15:29:40 2018 -0700

--
 .../push-socket-with-auuid.10.ddl.sqlpp | 29 +++
 .../push-socket-with-auuid.6.query.sqlpp|  4 +--
 .../push-socket-with-auuid.7.query.sqlpp| 30 
 .../push-socket-with-auuid.7.server.sqlpp   | 28 --
 .../push-socket-with-auuid.8.ddl.sqlpp  | 29 ---
 .../push-socket-with-auuid.8.query.sqlpp| 30 
 .../push-socket-with-auuid.9.server.sqlpp   | 28 ++
 .../udf25/udf25.3.query.sqlpp   |  4 ---
 .../udf25/udf25.4.query.sqlpp   | 20 +
 .../push-socket-with-auuid.1.adm|  2 --
 .../push-socket-with-auuid.2.adm|  1 +
 .../push-socket-with-auuid.3.adm|  1 +
 .../user-defined-functions/udf25/udf25.1.adm|  1 -
 .../user-defined-functions/udf25/udf25.2.adm|  1 +
 14 files changed, 141 insertions(+), 67 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4d6bce75/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.10.ddl.sqlpp
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.10.ddl.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.10.ddl.sqlpp
new file mode 100644
index 000..1ec8753
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.10.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a socket feed with datatype without auuid and connect
+ * it to three different datasets with different datatype: one with auuid, one
+ * open datatype with auuid and one without auuid using one of the attribute
+ * as primary key.
+ * Expected Res : Success
+ * Date : 20th Mar 2017
+ */
+
+use experiments;
+drop dataverse experiments;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4d6bce75/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.6.query.sqlpp
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.6.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/push-socket-with-auuid/push-socket-with-auuid.6.query.sqlpp
index

asterixdb git commit: [NO ISSUE][RT] Fix getting file descriptor counters

2018-04-18 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 34b3577c5 -> 07a0acc54


[NO ISSUE][RT] Fix getting file descriptor counters

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- The OS Bean methods return Longs. Prior to this change, there
  was a cast to Integer which always throws a ClassCastException.

Change-Id: Ia6c7cdd26475075975e5850d9fe19b3063d5d9f6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2607
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/07a0acc5
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/07a0acc5
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/07a0acc5

Branch: refs/heads/master
Commit: 07a0acc5423018dd97d2ea6bb2e3fa28fc939a2d
Parents: 34b3577
Author: Abdullah Alamoudi 
Authored: Wed Apr 18 22:24:47 2018 +0300
Committer: abdullah alamoudi 
Committed: Wed Apr 18 15:29:01 2018 -0700

--
 .../main/java/org/apache/hyracks/util/MXHelper.java | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07a0acc5/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
index 2b65106..2545c25 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
@@ -64,27 +64,27 @@ public class MXHelper {
 return getOpenFileDescriptorCount != null;
 }
 
-public static Integer getCurrentOpenFileCount() {
+public static Long getCurrentOpenFileCount() {
 if (getOpenFileDescriptorCount == null) {
-return -1;
+return -1L;
 }
 try {
-return (Integer) getOpenFileDescriptorCount.invoke(osMXBean);
+return (Long) getOpenFileDescriptorCount.invoke(osMXBean);
 } catch (Throwable e) { // NOSONAR
 LOGGER.log(Level.WARN, "Failure invoking 
getOpenFileDescriptorCount", e);
-return -1;
+return -1L;
 }
 }
 
-public static Integer getMaxOpenFileCount() {
+public static Long getMaxOpenFileCount() {
 if (getMaxFileDescriptorCount == null) {
-return -1;
+return -1L;
 }
 try {
-return (Integer) getMaxFileDescriptorCount.invoke(osMXBean);
+return (Long) getMaxFileDescriptorCount.invoke(osMXBean);
 } catch (Throwable e) { // NOSONAR
 LOGGER.log(Level.WARN, "Failure invoking 
getMaxFileDescriptorCount", e);
-return -1;
+return -1L;
 }
 }

asterixdb git commit: [NO ISSUE][TEST] Enable setting HTTP parameters in test queries

2018-04-14 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 33a656d1d -> 95c55f750


[NO ISSUE][TEST] Enable setting HTTP parameters in test queries

- user model changes: no
- storage format changes: no
- interface changes: no

Change-Id: I82721bd7b2702f0e3d294996764b03ff4738a457
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2593
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/95c55f75
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/95c55f75
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/95c55f75

Branch: refs/heads/master
Commit: 95c55f750f1881f86ea3a5a4db25b62dff8da139
Parents: 33a656d
Author: Abdullah Alamoudi 
Authored: Fri Apr 13 19:00:48 2018 -0700
Committer: abdullah alamoudi 
Committed: Sat Apr 14 21:29:37 2018 -0700

--
 .../asterix/api/http/server/NCQueryCancellationServlet.java  | 3 ++-
 .../apache/asterix/api/http/server/QueryCancellationServlet.java | 4 ++--
 .../test/java/org/apache/asterix/test/common/TestExecutor.java   | 4 
 3 files changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/95c55f75/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
index da621d2..5ad451f 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryCancellationServlet.java
@@ -23,6 +23,7 @@ import static 
org.apache.asterix.app.message.ExecuteStatementRequestMessage.DEFA
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.asterix.api.http.server.QueryServiceServlet.Parameter;
 import org.apache.asterix.app.message.CancelQueryRequest;
 import org.apache.asterix.app.message.CancelQueryResponse;
 import org.apache.asterix.common.messaging.api.INCMessageBroker;
@@ -54,7 +55,7 @@ public class NCQueryCancellationServlet extends 
QueryCancellationServlet {
 @Override
 protected void delete(IServletRequest request, IServletResponse response) {
 // gets the parameter client_context_id from the request.
-String clientContextId = request.getParameter(CLIENT_CONTEXT_ID);
+String clientContextId = 
request.getParameter(Parameter.CLIENT_ID.str());
 if (clientContextId == null) {
 response.setStatus(HttpResponseStatus.BAD_REQUEST);
 return;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/95c55f75/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryCancellationServlet.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryCancellationServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryCancellationServlet.java
index f8655ad..a8b3aef 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryCancellationServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryCancellationServlet.java
@@ -21,6 +21,7 @@ package org.apache.asterix.api.http.server;
 import java.io.IOException;
 import java.util.concurrent.ConcurrentMap;
 
+import org.apache.asterix.api.http.server.QueryServiceServlet.Parameter;
 import org.apache.asterix.translator.IStatementExecutorContext;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.job.JobId;
@@ -38,7 +39,6 @@ import io.netty.handler.codec.http.HttpResponseStatus;
  */
 public class QueryCancellationServlet extends AbstractServlet {
 private static final Logger LOGGER = LogManager.getLogger();
-protected static final String CLIENT_CONTEXT_ID = "client_context_id";
 
 public QueryCancellationServlet(ConcurrentMap ctx, 
String... paths) {
 super(ctx, paths);
@@ -46,7 +46,7 @@ public class QueryCancellationServlet extends AbstractServlet 
{
 
 @Override
 protected void delete(IServletRequest request, IServletResponse response) 
throws IOException {
-String

asterixdb git commit: [NOISSUE][ING] Leave permanently failed actives during recovery

2018-04-05 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 78dc83b12 -> b4e07eab4


[NOISSUE][ING] Leave permanently failed actives during recovery

Change-Id: I4cbeae666b1e5d5a9b29a33bac141d60244283ab
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2570
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b4e07eab
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b4e07eab
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b4e07eab

Branch: refs/heads/master
Commit: b4e07eab41be86b3414261461607ee63a627eff0
Parents: 78dc83b
Author: Abdullah Alamoudi 
Authored: Thu Apr 5 11:19:59 2018 -0700
Committer: abdullah alamoudi 
Committed: Thu Apr 5 15:36:16 2018 -0700

--
 .../asterix/app/active/ActiveNotificationHandler.java| 11 +--
 1 file changed, 1 insertion(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4e07eab/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
index e8bcb10..053e6cd 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
@@ -25,7 +25,6 @@ import java.util.Map;
 
 import org.apache.asterix.active.ActiveEvent;
 import org.apache.asterix.active.ActiveEvent.Kind;
-import org.apache.asterix.active.ActivityState;
 import org.apache.asterix.active.EntityId;
 import org.apache.asterix.active.IActiveEntityEventsListener;
 import org.apache.asterix.active.IActiveNotificationHandler;
@@ -33,7 +32,6 @@ import 
org.apache.asterix.active.message.ActivePartitionMessage;
 import org.apache.asterix.common.api.IMetadataLockManager;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.metadata.api.IActiveEntityController;
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.utils.DatasetUtil;
@@ -214,14 +212,7 @@ public class ActiveNotificationHandler extends 
SingleThreadEventProcessor

asterixdb git commit: [NO ISSUE][ING] Follow the IFrameWriter contract in Feed Pipeline

2018-03-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 282a79198 -> d89934cc1


[NO ISSUE][ING] Follow the IFrameWriter contract in Feed Pipeline

- user model changes: no
- storage format changes: no
- interface changes: yes
  - it removes unneeded ITupleForwarder interface.

Details:
- Previously, if a failure happens at the writer.open
  call in the FeedIntake operator, the IFrameWriter
  protocol is not followed since we don't fail before
  close is called.
- Previously, fail calls can be done as well inside
  of the feed adapter.
- This change moves failing the pipeline outside the
  adapter and so the adapter's responsibility is
  getting records from outside the system while
  the operator takes care of opening, failing, and
  closing the writer.

Change-Id: Ife679fb9643dc6b39d035e0eecdb915b227503a5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2463
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d89934cc
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d89934cc
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d89934cc

Branch: refs/heads/master
Commit: d89934cc15477352fab319e94e502fae4b72d381
Parents: 282a791
Author: Abdullah Alamoudi 
Authored: Thu Mar 8 22:34:00 2018 -0800
Committer: abdullah alamoudi 
Committed: Fri Mar 16 10:15:40 2018 -0700

--
 .../asterix/external/api/ITupleForwarder.java   |  43 -
 .../dataflow/AbstractDataFlowController.java|   5 +-
 .../AbstractFeedDataFlowController.java |  28 ++--
 .../dataflow/ChangeFeedDataFlowController.java  |   9 +-
 .../ChangeFeedWithMetaDataFlowController.java   |   9 +-
 .../dataflow/CounterTimerTupleForwarder.java| 165 ---
 .../dataflow/FeedRecordDataFlowController.java  |  26 +--
 .../dataflow/FeedStreamDataFlowController.java  |  13 +-
 .../external/dataflow/FeedTupleForwarder.java   | 126 --
 .../FeedWithMetaDataFlowController.java |   7 +-
 .../dataflow/FrameFullTupleForwarder.java   |  67 
 .../dataflow/IndexingDataFlowController.java|   8 +-
 .../dataflow/RateControlledTupleForwarder.java  |  95 ---
 .../dataflow/RecordDataFlowController.java  |  11 +-
 .../dataflow/StreamDataFlowController.java  |  10 +-
 .../external/dataflow/TupleForwarder.java   |  53 ++
 .../external/dataset/adapter/FeedAdapter.java   |  10 +-
 .../FeedIntakeOperatorNodePushable.java |  13 +-
 .../provider/DataflowControllerProvider.java|  31 ++--
 .../asterix/external/util/DataflowUtils.java|  65 
 .../asterix/external/util/FeedLogManager.java   |  22 +--
 .../adapter/TestTypedAdapterFactory.java|  18 +-
 .../api/dataflow/EnforceFrameWriter.java|   4 +-
 23 files changed, 154 insertions(+), 684 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d89934cc/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ITupleForwarder.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ITupleForwarder.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ITupleForwarder.java
deleted file mode 100644
index 22d0d6b..000
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ITupleForwarder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.api;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-

asterixdb git commit: [NO ISSUE][RT] Fix wait for IO operations

2018-03-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 8798c324a -> 282a79198


[NO ISSUE][RT] Fix wait for IO operations

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Some operations such as close dataset, delete components,
  and drop index need to wait for IO operations.
- Before this change, the wait for IO operation would just check
  the count of IO operations on the dataset info. This is not
  enough as a flush might have started by writing the flush log to
  the log tail but only on the flush of that log, we trigger the
  flush operation and the count of IO operation increases.
- To address this problem, we write a wait log before we check the
  IO operation count ensuring that any flush logs in the log tail
  have been flushed and counts incremented.

Change-Id: Ibfa883410cd24e0af54732f7ea6f1b4eb2184e8e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2495
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/282a7919
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/282a7919
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/282a7919

Branch: refs/heads/master
Commit: 282a79198af022563c3d93c51e0d9a052e140ab2
Parents: 8798c32
Author: Abdullah Alamoudi 
Authored: Thu Mar 15 22:52:53 2018 -0700
Committer: abdullah alamoudi 
Committed: Fri Mar 16 10:08:33 2018 -0700

--
 .../asterix/common/context/DatasetInfo.java | 43 
 .../common/context/DatasetLifecycleManager.java |  2 +-
 .../CorrelatedPrefixMergePolicyTest.java|  2 +-
 .../management/service/logging/LogManager.java  |  3 +-
 4 files changed, 31 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/282a7919/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 44baf77..f4d764a 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -23,6 +23,9 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.asterix.common.transactions.ILogManager;
+import org.apache.asterix.common.transactions.LogRecord;
+import org.apache.asterix.common.transactions.LogType;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.logging.log4j.LogManager;
@@ -35,6 +38,8 @@ public class DatasetInfo extends Info implements 
Comparable {
 // resourceID -> index
 private final Map indexes;
 private final int datasetID;
+private final ILogManager logManager;
+private final LogRecord waitLog = new LogRecord();
 private int numActiveIOOps;
 private long lastAccess;
 private boolean isExternal;
@@ -42,13 +47,16 @@ public class DatasetInfo extends Info implements 
Comparable {
 private boolean memoryAllocated;
 private boolean durable;
 
-public DatasetInfo(int datasetID) {
+public DatasetInfo(int datasetID, ILogManager logManager) {
 this.partitionIndexes = new HashMap<>();
 this.indexes = new HashMap<>();
 this.setLastAccess(-1);
 this.datasetID = datasetID;
 this.setRegistered(false);
 this.setMemoryAllocated(false);
+this.logManager = logManager;
+waitLog.setLogType(LogType.WAIT);
+waitLog.computeAndSetLogSize();
 }
 
 @Override
@@ -199,23 +207,26 @@ public class DatasetInfo extends Info implements 
Comparable {
 this.lastAccess = lastAccess;
 }
 
-public synchronized void waitForIO() throws HyracksDataException {
-while (numActiveIOOps > 0) {
-try {
-/**
- * Will be Notified by {@link 
DatasetInfo#undeclareActiveIOOperation()}
- */
-wait();
-} catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-throw HyracksDataException.create(e);
+public void waitForIO() throws HyracksDataException {
+logManager.log(waitLog);
+

asterixdb git commit: [NO ISSUE][OTH] Cancel queries on abrupt close of user connection

2018-03-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master e45528725 -> 8798c324a


[NO ISSUE][OTH] Cancel queries on abrupt close of user connection

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Since there is no point of processing user queries when the user
  disconnects before getting the response of their requests, the
  QueryServiceServlet now cancels the user request when that happens.

Change-Id: I902b52322153d114f7ca5f0f06e0af9c5ead8f2a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2490
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8798c324
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8798c324
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8798c324

Branch: refs/heads/master
Commit: 8798c324aa7e99a92356df2a8567be35fcbd9efe
Parents: e455287
Author: Abdullah Alamoudi 
Authored: Thu Mar 15 23:06:22 2018 -0700
Committer: abdullah alamoudi 
Committed: Fri Mar 16 09:38:25 2018 -0700

--
 .../api/http/server/NCQueryServiceServlet.java  |  8 +
 .../http/server/InterruptOnCloseHandler.java| 37 
 .../hyracks/http/test/HttpServerTest.java   |  3 +-
 3 files changed, 47 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8798c324/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 5cbac64..3564736 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -47,7 +47,10 @@ import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hyracks.api.application.INCServiceContext;
 import org.apache.hyracks.api.dataset.ResultSetId;
 import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.http.api.IChannelClosedHandler;
 import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.server.HttpServer;
+import org.apache.hyracks.http.server.InterruptOnCloseHandler;
 import org.apache.hyracks.ipc.exceptions.IPCException;
 import org.apache.logging.log4j.Level;
 
@@ -160,4 +163,9 @@ public class NCQueryServiceServlet extends 
QueryServiceServlet {
 super.handleExecuteStatementException(t, execution);
 }
 }
+
+@Override
+public IChannelClosedHandler getChannelClosedHandler(HttpServer server) {
+return InterruptOnCloseHandler.INSTANCE;
+}
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8798c324/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/InterruptOnCloseHandler.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/InterruptOnCloseHandler.java
 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/InterruptOnCloseHandler.java
new file mode 100644
index 000..84b802f
--- /dev/null
+++ 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/InterruptOnCloseHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.http.server;
+
+import java.util.concurrent.Future;
+
+import org.apache.hyracks.http.api.IChannelClosedHandler;
+import

asterixdb git commit: [NO ISSUE] Avoid sleep, shutdown log manager on halt

2018-03-15 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master c3ee83138 -> 7ffdaba94


[NO ISSUE] Avoid sleep, shutdown log manager on halt

Change-Id: I64749ffd98563099bd68926cd7ca0e697f066baf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2485
Sonar-Qube: Jenkins 
Reviewed-by: abdullah alamoudi 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7ffdaba9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7ffdaba9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7ffdaba9

Branch: refs/heads/master
Commit: 7ffdaba94ad10e959f64720846a00e17c0979846
Parents: c3ee831
Author: Michael Blow 
Authored: Wed Mar 14 18:47:10 2018 -0700
Committer: Michael Blow 
Committed: Wed Mar 14 23:13:29 2018 -0700

--
 .../src/main/java/org/apache/hyracks/util/ExitUtil.java | 9 ++---
 1 file changed, 2 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7ffdaba9/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index ec6587e..db5cd13 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -74,15 +74,10 @@ public class ExitUtil {
 exit(status);
 }
 
-@SuppressWarnings("squid:S2142") // catch interrupted
 public static void halt(int status) {
 LOGGER.fatal("JVM halting with status " + status + "; bye!", new 
Throwable("halt stacktrace"));
-try {
-// try to give time for the log to be emitted...
-Thread.sleep(2000);
-} catch (InterruptedException e) {
-// ignore
-}
+// try to give time for the log to be emitted...
+LogManager.shutdown();
 Runtime.getRuntime().halt(status);
 }

asterixdb git commit: [NO ISSUE][OTH] Log open file count on unexpected HttpServer close

2018-03-14 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master d6d42c861 -> 57df8db1b


[NO ISSUE][OTH] Log open file count on unexpected HttpServer close

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Recently, there has been some cases where the Http server
  repeatedly drops its listener. One of the possible causes
  for this is a leak in open file descriptor. To determine if
  this is the case, we log the open files count when that happens.

Change-Id: Ie43f3392268b3994839d1f98f4de9fe669c7be62
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2483
Sonar-Qube: Jenkins 
Reviewed-by: Michael Blow 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/57df8db1
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/57df8db1
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/57df8db1

Branch: refs/heads/master
Commit: 57df8db1b4188940d2f3c8632dcdf1a4c10d0229
Parents: d6d42c8
Author: Abdullah Alamoudi 
Authored: Wed Mar 14 15:25:29 2018 -0700
Committer: abdullah alamoudi 
Committed: Wed Mar 14 17:26:57 2018 -0700

--
 .../control/common/heartbeat/HeartbeatData.java |   2 +-
 .../hyracks/control/common/utils/MXHelper.java  |  48 -
 .../control/nc/NodeControllerService.java   |   8 +-
 .../nc/heartbeat/HeartbeatComputeTask.java  |   8 +-
 .../apache/hyracks/http/server/HttpServer.java  |   2 +
 .../java/org/apache/hyracks/util/MXHelper.java  | 103 +++
 6 files changed, 114 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
index d09d0b4..76b68bb 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -18,7 +18,7 @@
  */
 package org.apache.hyracks.control.common.heartbeat;
 
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
 
 import java.io.DataInput;
 import java.io.DataOutput;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
deleted file mode 100644
index cfe8c21..000
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.common.utils;
-
-import java.lang.management.GarbageCollectorMXBean;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import

asterixdb git commit: [NO ISSUE][STO] Improve the LSMIOOperationCallback interface.

2018-03-06 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 17ad87595 -> 517e1d27b


[NO ISSUE][STO] Improve the LSMIOOperationCallback interface.

- user model changes: no
- storage format changes: no
- interface changes: yes
  + ILSMIndexOperationContext.getIoOperationType()
  + ILSMIndexOperationContext.getNewComponent()
  * before, after, and finalize
calls of ILSMIOOperationCallback now take
ILSMIndexOperationContext as a parameter

Details:
- Before, some calls to ILSMIOOperationCallback
  take just an enum LSMIOOperationType, some of them
  take an enum and a component object. These sometimes don't
  provide enough information to different implementations of
  the callback that might be interested in more than that.
- Having the operation context object passed allow for
  better exchange of information between different callers
  and callees throughout the IO operation.

Change-Id: Ib7120c40a1a2256ed528dfd2e5853db9dba247c6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2455
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/517e1d27
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/517e1d27
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/517e1d27

Branch: refs/heads/master
Commit: 517e1d27b6baaddec4ad6b01d10ffc8d9ac370b0
Parents: 17ad875
Author: Abdullah Alamoudi 
Authored: Mon Mar 5 20:34:55 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Mar 5 23:58:10 2018 -0800

--
 .../TestLsmBtreeIoOpCallbackFactory.java|  29 ++-
 .../AbstractLSMIOOperationCallback.java |  53 +++---
 .../AbstractLSMIOOperationCallbackTest.java | 112 +++-
 .../TestLSMIndexOperationContext.java   | 177 +++
 .../am/lsm/btree/impls/ExternalBTree.java   |   6 +-
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |   5 +-
 .../am/lsm/common/api/ILSMIOOperation.java  |   3 +-
 .../lsm/common/api/ILSMIOOperationCallback.java |  15 +-
 .../common/api/ILSMIndexOperationContext.java   |  25 +++
 .../am/lsm/common/impls/AbstractLSMIndex.java   |   7 +-
 .../impls/AbstractLSMIndexOperationContext.java |  23 +++
 .../BlockingIOOperationCallbackWrapper.java |  20 +--
 .../impls/ComponentReplacementContext.java  |  21 +++
 .../lsm/common/impls/ExternalIndexHarness.java  |  18 +-
 .../storage/am/lsm/common/impls/LSMHarness.java |  29 +--
 .../impls/LSMIndexDiskComponentBulkLoader.java  |  28 +--
 .../impls/NoOpIOOperationCallbackFactory.java   |  14 +-
 .../common/impls/StubIOOperationCallback.java   |  23 +--
 .../am/lsm/rtree/impls/ExternalRTree.java   |   8 +-
 19 files changed, 442 insertions(+), 174 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/517e1d27/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
index c69ffe5..5852ad9 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
@@ -18,20 +18,17 @@
  */
 package org.apache.asterix.test.dataflow;
 
-import java.util.List;
-
 import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
 import 
org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
 import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGeneratorFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
 import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
 import

asterixdb git commit: [NO ISSUE][TEST] Introduce dynamic expected result poll test type

2018-03-01 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 71e4b4c00 -> 90883159b


[NO ISSUE][TEST] Introduce dynamic expected result poll test type

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- This change allows for a new type of tests. The new test can
  poll for both expected results and a query and compare the two
  values.
- A test case is added.

Change-Id: Ifc132b2d2286eea1d1e119984c33ca5eef9be92a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2438
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/90883159
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/90883159
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/90883159

Branch: refs/heads/master
Commit: 90883159b99f12d98b648dff112605863822a0ac
Parents: 71e4b4c
Author: Abdullah Alamoudi 
Authored: Thu Mar 1 15:56:00 2018 -0800
Committer: abdullah alamoudi 
Committed: Thu Mar 1 21:35:24 2018 -0800

--
 .../test/common/IExpectedResultPoller.java  |  27 +
 .../apache/asterix/test/common/IPollTask.java   |  51 
 .../asterix/test/common/TestExecutor.java   | 116 ---
 .../poll-dynamic.1.polldynamic.sqlpp|  22 
 .../resources/runtimets/testsuite_sqlpp.xml |   5 +
 5 files changed, 203 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90883159/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IExpectedResultPoller.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IExpectedResultPoller.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IExpectedResultPoller.java
new file mode 100644
index 000..ce2760d
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IExpectedResultPoller.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.common;
+
+@FunctionalInterface
+public interface IExpectedResultPoller {
+/**
+ * @return the expected result as a string
+ */
+String poll() throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90883159/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
new file mode 100644
index 000..6d32518
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.common;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import

asterixdb git commit: [NO ISSUE][ING] Close and return on interrupt of ingestion

2018-02-28 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 38ca962f4 -> 0ef0e7648


[NO ISSUE][ING] Close and return on interrupt of ingestion

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Previously, we would still attempt to write through the network
  in case an ingestion task is interrupted.
- The goal was to try and get as much data in as possible but in
  the case where the cluster was in a bad state, this could
  lead to hanging threads.
- After this change, each record reader must implement the stop
  method correctly to allow for graceful stop while interrupts
  will always mean abort the task and return as soon as possible.

Change-Id: I6119617d133fb161a48b39f9812ec79e0189975b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2428
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/0ef0e764
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/0ef0e764
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/0ef0e764

Branch: refs/heads/master
Commit: 0ef0e76482fd1485d633394c5a02f39acc0614fc
Parents: 38ca962
Author: Abdullah Alamoudi 
Authored: Mon Feb 26 20:23:21 2018 -0800
Committer: abdullah alamoudi 
Committed: Wed Feb 28 10:44:35 2018 -0800

--
 .../asterix/common/exceptions/ErrorCode.java  |  1 -
 .../src/main/resources/asx_errormsg/en.properties |  1 -
 .../dataflow/FeedRecordDataFlowController.java| 18 --
 .../reader/twitter/TwitterPullRecordReader.java   |  7 +--
 .../operators/FeedIntakeOperatorNodePushable.java |  2 +-
 5 files changed, 10 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0ef0e764/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 1a83603..04054f1 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -250,7 +250,6 @@ public class ErrorCode {
 public static final int ACTIVE_RUNTIME_IS_ALREADY_REGISTERED = 3105;
 public static final int ACTIVE_RUNTIME_IS_NOT_REGISTERED = 3106;
 public static final int ACTIVE_EVENT_HANDLER_ALREADY_SUSPENDED = 3107;
-public static final int FEED_STOPPED_WHILE_WAITING_FOR_A_NEW_RECORD = 3108;
 public static final int METADATA_DROP_FUCTION_IN_USE = 3109;
 public static final int FEED_FAILED_WHILE_GETTING_A_NEW_RECORD = 3110;
 public static final int FEED_START_FEED_WITHOUT_CONNECTION = 3111;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0ef0e764/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
--
diff --git 
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties 
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 995b541..1fb8480 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -239,7 +239,6 @@
 3105 = %1$s is already registered
 3106 = %1$s is not registered
 3107 = Active Notification Handler is already suspended
-3108 = Feed stopped while waiting for a new record
 3109 = Function %1$s is being used. It cannot be dropped
 3110 = Feed failed while reading a new record
 3111 = Feed %1$s is not connected to any dataset

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0ef0e764/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 306a2a5..9826be7 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -95,11 +95,7 @@ public

asterixdb git commit: [NO ISSUE][TEST] Test flushing of empty component

2018-02-24 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 3fe67422d -> 3278316ab


[NO ISSUE][TEST] Test flushing of empty component

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
Some use cases require flushing of component that have no data
but some Metadata. This change adds a test to ensure that always
works.

Change-Id: If921323dfe03cbd70edc3a8ea8e01226d7527bb3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2421
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3278316a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3278316a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3278316a

Branch: refs/heads/master
Commit: 3278316abc26ad8e9fd04e5ebec540006bada072
Parents: 3fe6742
Author: Abdullah Alamoudi 
Authored: Fri Feb 23 22:32:16 2018 -0800
Committer: abdullah alamoudi 
Committed: Sat Feb 24 09:40:23 2018 -0800

--
 .../test/dataflow/FlushMetadataOnlyTest.java| 120 +++
 .../context/PrimaryIndexOperationTracker.java   |   3 +-
 .../apache/hyracks/data/std/util/DataUtils.java |  41 +++
 3 files changed, 162 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3278316a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java
new file mode 100644
index 000..f9421a1
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.nio.file.Paths;
+
+import org.apache.asterix.app.bootstrap.TestNodeController;
+import org.apache.asterix.app.bootstrap.TestNodeController.PrimaryIndexInfo;
+import org.apache.asterix.app.nc.NCAppRuntimeContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.test.base.TestMethodTracer;
+import org.apache.asterix.test.common.TestHelper;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.DataUtils;
+import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import 
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
+import 
org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+public class FlushMetadataOnlyTest {
+private static TestNodeController nc;
+private static TestLsmBtree lsmBtree;
+private static NCAppRuntimeContext ncAppCtx;
+private static IDatasetLifecycleManager dsLifecycleMgr;
+private static IHyracksTaskContext ctx;
+private static IIndexDataflowHelper indexDataflowHelper;
+private static final int PARTITION = 0;
+
+@Rule
+public TestRule watcher = new TestMethodTracer();
+
+

asterixdb git commit: [NO ISSUE][TXN] Prevent deadlock in Metadata transactions

2018-02-21 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master fdf862eac -> 05365fd06


[NO ISSUE][TXN] Prevent deadlock in Metadata transactions

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Flushes in metadata datasets are triggerred by entity update
  logs, unlike regular transactions where flushes are triggerred
  by entity commit logs.
- Because entity update logs can be writting to disk before the
  operation completes, there is a chance that an operation that
  caused the component to be full exits after the log is flushed
  and so, a flush operation is not scheduled.
- This change proposes a simple fix. The fix is that metadata
  operation will also check if a flush is needed and will schedule
  one if needed.

Change-Id: I07a18840dc54fe052b7bd294595f816f6d8a4d2f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2413
Sonar-Qube: Jenkins 
Reviewed-by: Murtadha Hubail 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/05365fd0
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/05365fd0
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/05365fd0

Branch: refs/heads/master
Commit: 05365fd0639c14eed80bfbe67ae3f24fef72228a
Parents: fdf862e
Author: Abdullah Alamoudi 
Authored: Wed Feb 21 08:20:39 2018 -0800
Committer: abdullah alamoudi 
Committed: Wed Feb 21 09:53:09 2018 -0800

--
 .../common/context/PrimaryIndexOperationTracker.java | 15 ++-
 .../org/apache/asterix/metadata/MetadataNode.java|  4 +++-
 2 files changed, 13 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/05365fd0/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
--
diff --git 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 1a76b66..f7f2806 100644
--- 
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ 
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -86,17 +86,19 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 throws HyracksDataException {
 if (opType == LSMOperationType.MODIFICATION || opType == 
LSMOperationType.FORCE_MODIFICATION) {
 decrementNumActiveOperations(modificationCallback);
-if (numActiveOperations.get() == 0) {
-flushIfRequested();
-} else if (numActiveOperations.get() < 0) {
-throw new HyracksDataException("The number of active 
operations cannot be negative!");
-}
+flushIfNeeded();
 } else if (opType == LSMOperationType.FLUSH || opType == 
LSMOperationType.MERGE
 || opType == LSMOperationType.REPLICATE) {
 dsInfo.undeclareActiveIOOperation();
 }
 }
 
+public synchronized void flushIfNeeded() throws HyracksDataException {
+if (numActiveOperations.get() == 0) {
+flushIfRequested();
+}
+}
+
 public void flushIfRequested() throws HyracksDataException {
 // If we need a flush, and this is the last completing operation, then 
schedule the flush,
 // or if there is a flush scheduled by the checkpoint (flushOnExit), 
then schedule it
@@ -177,6 +179,9 @@ public class PrimaryIndexOperationTracker extends 
BaseOperationTracker {
 //modificationCallback can be NoOpOperationCallback when redo/undo 
operations are executed.
 if (modificationCallback != NoOpOperationCallback.INSTANCE) {
 numActiveOperations.decrementAndGet();
+if (numActiveOperations.get() < 0) {
+throw new IllegalStateException("The number of active 
operations cannot be negative!");
+}
 ((AbstractOperationCallback) 
modificationCallback).afterOperation();
 }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/05365fd0/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
--
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java

asterixdb git commit: [NO ISSUE][ING] Prevent duplicate active runtimes in NCs

2018-02-19 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 02825e092 -> b227a0d37


[NO ISSUE][ING] Prevent duplicate active runtimes in NCs

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, when an active runtime is registered, we ignore
  if the runtime id is already registered.
- After this change, such operation will throw an exception.

Change-Id: Id7065625537f930394fa7387af2fd1e851582c86
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2392
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b227a0d3
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b227a0d3
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b227a0d3

Branch: refs/heads/master
Commit: b227a0d37893831f94a971cc116600860d8d23d7
Parents: 02825e0
Author: Abdullah Alamoudi 
Authored: Mon Feb 19 15:55:56 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Feb 19 22:06:21 2018 -0800

--
 .../main/java/org/apache/asterix/active/ActiveManager.java   | 4 +++-
 .../asterix/app/active/ActiveEntityEventsListener.java   | 8 
 2 files changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b227a0d3/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index f9aef4c..bb76204 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -73,7 +73,9 @@ public class ActiveManager {
 if (shutdown) {
 throw new RuntimeDataException(ErrorCode.ACTIVE_MANAGER_SHUTDOWN);
 }
-runtimes.putIfAbsent(runtime.getRuntimeId(), runtime);
+if (runtimes.putIfAbsent(runtime.getRuntimeId(), runtime) != null) {
+throw new IllegalStateException("Active Runtime " + 
runtime.getRuntimeId() + " is already registered");
+}
 }
 
 public void deregisterRuntime(ActiveRuntimeId id) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b227a0d3/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 1758daa..2a214e3 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -182,8 +182,9 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 protected void finish(ActiveEvent event) throws HyracksDataException {
 LOGGER.log(level, "the job " + jobId + " finished");
 if (numRegistered != numDeRegistered) {
-LOGGER.log(Level.WARN, "the job " + jobId + " finished with 
reported runtime registrations = "
-+ numRegistered + " and deregistrations = " + 
numDeRegistered + " on node controllers");
+LOGGER.log(Level.WARN,
+"the job {} finished with reported runtime registrations = 
{} and deregistrations = {}", jobId,
+numRegistered, numDeRegistered);
 }
 jobId = null;
 Pair status = (Pair) event.getEventObject();
@@ -194,8 +195,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 jobFailure = exceptions.isEmpty() ? new 
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
 : exceptions.get(0);
 setState((state == ActivityState.STOPPING) ? ActivityState.STOPPED 
: ActivityState.TEMPORARILY_FAILED);
-if (prevState != ActivityState.SUSPENDING && prevState != 
ActivityState.RECOVERING
-&& prevState != ActivityState.RESUMING && prevState != 
ActivityState.STOPPING) {
+if (prevState == ActivityState.RUNNING) {
 recover();
 }

asterixdb git commit: [NO ISSUE][OTH] Enable adding request channel close listener

2018-02-19 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 15589b42e -> 02825e092


[NO ISSUE][OTH] Enable adding request channel close listener

- user model changes: no
- storage format changes: no
- interface changes: yes
  - Introduce IChannelCloseHandler.handle that gets called when
the request channel is closed.
  - Add HttpServer.getChannelCloseHandler
  - Add IServlet.getChannelCloseHandler

details:
- Previously, we didn't know that an Http client closed the
  connection until we try to write and find that the channel has
  been closed.
- After this change, the moment the channel is closed, the http
  channel close handler is called.
- A test is added with a handler that interrupts the execution.

Change-Id: I42f1857c0158af6f447282cab8fbd600767b08d5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1972
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/02825e09
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/02825e09
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/02825e09

Branch: refs/heads/master
Commit: 02825e092e1dcd9f6c4b12a15087426374e6d336
Parents: 15589b4
Author: Abdullah Alamoudi 
Authored: Mon Feb 19 17:09:02 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Feb 19 22:05:38 2018 -0800

--
 .../hyracks/http/api/IChannelClosedHandler.java |  39 ++
 .../org/apache/hyracks/http/api/IServlet.java   |  13 ++
 .../apache/hyracks/http/server/HttpServer.java  |  19 ++-
 .../hyracks/http/server/HttpServerHandler.java  |  12 +-
 .../hyracks/http/servlet/SleepyServlet.java |  17 ++-
 .../hyracks/http/test/HttpRequestTask.java  | 109 +
 .../hyracks/http/test/HttpServerTest.java   | 119 +++
 7 files changed, 249 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/02825e09/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IChannelClosedHandler.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IChannelClosedHandler.java
 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IChannelClosedHandler.java
new file mode 100644
index 000..4e433ad
--- /dev/null
+++ 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IChannelClosedHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.http.api;
+
+import java.util.concurrent.Future;
+
+import org.apache.hyracks.http.server.HttpServer;
+
+@FunctionalInterface
+public interface IChannelClosedHandler {
+
+/**
+ * Handle a request channel closed event
+ *
+ * @param server
+ *the server handling the request
+ * @param servlet
+ *the servlet handling the request
+ * @param task
+ *the task handling the request
+ */
+void channelClosed(HttpServer server, IServlet servlet, Future task);
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/02825e09/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServlet.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServlet.java
 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServlet.java
index 157eef5..186fb0e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServlet.java
+++

asterixdb git commit: [NO ISSUE][OTH] Remove logs for legal destroy calls

2018-02-19 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 18ef4b99a -> 15589b42e


[NO ISSUE][OTH] Remove logs for legal destroy calls

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, we logged when a cursor was destroyed multiple times.
- Since this is a legal call, we don't log this anymore.

Change-Id: I8fc5c4ad165e44d5259446aad8e64826a672fd77
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2403
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Murtadha Hubail 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/15589b42
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/15589b42
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/15589b42

Branch: refs/heads/master
Commit: 15589b42e590da67b9b5392541e58827ac1bb804
Parents: 18ef4b9
Author: Abdullah Alamoudi 
Authored: Mon Feb 19 11:01:24 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Feb 19 14:44:22 2018 -0800

--
 .../org/apache/hyracks/storage/common/EnforcedIndexCursor.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15589b42/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/EnforcedIndexCursor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/EnforcedIndexCursor.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/EnforcedIndexCursor.java
index 929faff..b7caf7f 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/EnforcedIndexCursor.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/EnforcedIndexCursor.java
@@ -85,8 +85,6 @@ public abstract class EnforcedIndexCursor implements 
IIndexCursor {
 public final void destroy() throws HyracksDataException {
 if (ENFORCE_OPEN_CLOSE_DESTROY) {
 if (state == State.DESTROYED) {
-LOGGER.log(Level.WARN,
-"multiple cursor.destroy() call in " + 
Arrays.toString(new Throwable().getStackTrace()));
 return;
 } else if (state != State.CLOSED) {
 if (STORE_TRACES && openCallStack != null) {

asterixdb git commit: [NO ISSUE][RT] follow IFrameWriter protocol in SplitOperatorDescriptor

2018-02-18 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 9b1cd7968 -> 914be5ade


[NO ISSUE][RT] follow IFrameWriter protocol in SplitOperatorDescriptor

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, the SplitOperatorDescriptor didn't follow the
  IFrameWriter protocol in case of failure which lead to having
  some open resources after the job.
- This caused so many failures in Cancellation tests.
- This change also increases the rate of cancellation during the
  cancellation tests to ensure that similar problems are found.

Change-Id: I3166895589e1ab7355d689397f676f7da5c9809f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2399
Reviewed-by: Taewoo Kim 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/914be5ad
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/914be5ad
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/914be5ad

Branch: refs/heads/master
Commit: 914be5adee7be015fa7804bcbe8a94052cb66985
Parents: 9b1cd79
Author: Abdullah Alamoudi 
Authored: Sun Feb 18 17:27:04 2018 -0800
Committer: abdullah alamoudi 
Committed: Sun Feb 18 19:07:57 2018 -0800

--
 .../test/common/CancellationTestExecutor.java   |  15 ++-
 .../indexing/ExternalFileIndexAccessor.java |   4 +-
 .../operators/std/SplitOperatorDescriptor.java  |  43 +---
 .../apache/hyracks/api/util/CleanupUtils.java   | 102 +++
 .../apache/hyracks/api/util/DestroyUtils.java   |  53 --
 .../storage/am/btree/impls/BTreeOpContext.java  |   4 +-
 .../IndexSearchOperatorNodePushable.java|   6 +-
 ...eIndexDiskOrderScanOperatorNodePushable.java |   4 +-
 .../am/common/util/ResourceReleaseUtils.java|  27 -
 .../impls/LSMBTreeDiskComponentScanCursor.java  |   4 +-
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |   8 +-
 .../btree/impls/LSMBTreeRangeSearchCursor.java  |   6 +-
 .../impls/LSMInvertedIndexOpContext.java|   6 +-
 .../am/lsm/rtree/impls/LSMRTreeOpContext.java   |  10 +-
 14 files changed, 169 insertions(+), 123 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/914be5ad/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index 41aa23f..a1e70dc 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -63,11 +63,16 @@ public class CancellationTestExecutor extends TestExecutor {
 }
 };
 Future future = executor.submit(query);
-if (cancellable) {
-Thread.sleep(20);
-// Cancels the query request while the query is executing.
-int rc = cancelQuery(getEndpoint(Servlets.RUNNING_REQUESTS), 
newParams);
-Assert.assertTrue(rc == 200 || rc == 404);
+while (!future.isDone()) {
+if (cancellable) {
+Thread.sleep(10);
+// Cancels the query request while the query is executing.
+int rc = cancelQuery(getEndpoint(Servlets.RUNNING_REQUESTS), 
newParams);
+Assert.assertTrue(rc == 200 || rc == 404);
+if (rc == 200) {
+break;
+}
+}
 }
 InputStream inputStream = future.get();
 // Since the current cancellation (i.e., abort) implementation is 
based on thread.interrupt and we did not

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/914be5ad/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
--
diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
index 5a9852e..58a2413 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
@@

asterixdb git commit: [NO ISSUE][STO] Perform IO reads in uninterruptible threads

2018-02-16 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master c3c235743 -> 98574960c


[NO ISSUE][STO] Perform IO reads in uninterruptible threads

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, IO reads are performed on the task threads.
- Task threads can be interrupted which can cause deadlocks
  due to a JDK synchronization bug.
- After this change, 2 IO read threads are there per IO device.
- Threads will receive IO read requests and process them.
- Such threads are never interrupted and are killed through
  the use of a poison pill.

Change-Id: Id28d57a222f42962284b24296cb9b91658e5dc77
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2387
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/98574960
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/98574960
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/98574960

Branch: refs/heads/master
Commit: 98574960cf9163ca3b15faaf8dcea1581cd22e43
Parents: c3c2357
Author: Abdullah Alamoudi 
Authored: Wed Feb 14 10:27:40 2018 -0800
Committer: abdullah alamoudi 
Committed: Fri Feb 16 14:34:16 2018 -0800

--
 .../hyracks/hyracks-storage-common/pom.xml  |  14 +-
 .../hyracks/storage/common/IIndexCursor.java|  21 ++-
 .../storage/common/buffercache/BufferCache.java | 157 +++
 .../storage/common/buffercache/CachedPage.java  |  42 -
 4 files changed, 188 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/98574960/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
--
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml 
b/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
index 9936e1a..40422f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/pom.xml
@@ -16,18 +16,16 @@
  ! specific language governing permissions and limitations
  ! under the License.
  !-->
-
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd;>
   4.0.0
   hyracks-storage-common
   hyracks-storage-common
-
   
 org.apache.hyracks
 hyracks
 0.3.4-SNAPSHOT
   
-
   
 
   Apache License, Version 2.0
@@ -36,7 +34,6 @@
   A business-friendly OSS license
 
   
-
   
 ${basedir}/../..
   
@@ -52,8 +49,13 @@
   ${project.version}
 
 
+  org.apache.hyracks
+  hyracks-util
+  ${project.version}
+
+
   org.apache.logging.log4j
   log4j-api
 
   
-
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/98574960/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexCursor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexCursor.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexCursor.java
index f704921..00c5dce 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexCursor.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexCursor.java
@@ -26,14 +26,20 @@ import 
org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 /**
  * Represents an index cursor. The expected use
  * cursor = new cursor();
- * while (more predicates){
- * -cursor.open(predicate);
- * -while (cursor.hasNext()){
- * --cursor.next()
+ * try{
+ * -while (more predicates){
+ * --cursor.open(predicate);
+ * --try{
+ * ---while (cursor.hasNext()){
+ * cursor.next()
+ * ---}
+ * --} finally{
+ * ---cursor.close();
+ * --}
  * -}
- * -cursor.close();
+ * } finally{
+ * -cursor.destroy();
  * }
- * cursor.destroy();
  * Each created cursor must have destroy called
  * Each successfully opened cursor must have close called
  *
@@ -47,7 +53,8 @@ import 
org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
  * When a cursor object is created, it is in the

asterixdb git commit: [NO ISSUE][ING] Ensure Active recovery is triggerred

2018-02-14 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 254eeaa59 -> 0f294d0d1


[NO ISSUE][ING] Ensure Active recovery is triggerred

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, when the cluster recovers and tries to notify the
  active events handler to resume recovery tasks, the notification
  is performed on the listener but the task waits on itself.
- After this change, the task waits on the listener and so
  the notification wakes up the recovery task correctly.

Change-Id: I5fe3a2013a4cb92313648fd04ff2ef3c417270fb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2385
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/0f294d0d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/0f294d0d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/0f294d0d

Branch: refs/heads/master
Commit: 0f294d0d1e2d5718a02e163dcd34c9bea1956963
Parents: 254eeaa
Author: Abdullah Alamoudi 
Authored: Sun Feb 11 13:41:48 2018 -0800
Committer: abdullah alamoudi 
Committed: Wed Feb 14 10:29:32 2018 -0800

--
 .../org/apache/asterix/active/IActiveNotificationHandler.java   | 5 +
 .../apache/asterix/app/active/ActiveEntityEventsListener.java   | 2 +-
 .../apache/asterix/app/active/ActiveNotificationHandler.java| 2 +-
 .../main/java/org/apache/asterix/app/active/RecoveryTask.java   | 2 +-
 .../apache/asterix/metadata/api/IActiveEntityController.java| 3 +--
 5 files changed, 5 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f294d0d/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveNotificationHandler.java
--
diff --git 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveNotificationHandler.java
 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveNotificationHandler.java
index 8b9f232..1d29828 100644
--- 
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveNotificationHandler.java
+++ 
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveNotificationHandler.java
@@ -21,7 +21,6 @@ package org.apache.asterix.active;
 
 import org.apache.asterix.active.message.ActivePartitionMessage;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.IJobLifecycleListener;
 
 /**
  * Represents the notification handler for events of active entity jobs
@@ -30,10 +29,8 @@ public interface IActiveNotificationHandler {
 
 /**
  * Recover all active jobs that failed
- *
- * @throws HyracksDataException
  */
-void recover() throws HyracksDataException;
+void recover();
 
 /**
  * Set whether handler initialization has completed or not

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f294d0d/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 12bcc0d..1758daa 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -353,7 +353,7 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 }
 
 @Override
-public synchronized void recover() throws HyracksDataException {
+public synchronized void recover() {
 LOGGER.log(level, "Recover is called on " + entityId);
 if (retryPolicyFactory == NoRetryPolicyFactory.INSTANCE) {
 LOGGER.log(level, "But it has no recovery policy, so it is set to 
permanent failure");

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f294d0d/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
index b8ed03f..e8bcb10 100644
---

asterixdb git commit: [NO ISSUE] Only close the index in the close of the Upsert Operator

2018-02-12 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 4ff6a36d1 -> e19da1fc3


[NO ISSUE] Only close the index in the close of the Upsert Operator

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, a failure in the opening of the upsert operator will cause
  the index to be closed twice.
- After this change, we only close the index in the close call of the
  operator.

Change-Id: I1d76de2781362ef5c80aa926e1112e7872049c71
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2378
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e19da1fc
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e19da1fc
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e19da1fc

Branch: refs/heads/master
Commit: e19da1fc3ae763ba0adbd5b85fec2163ed23f86b
Parents: 4ff6a36
Author: Abdullah Alamoudi 
Authored: Fri Feb 9 16:11:49 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Feb 12 18:03:08 2018 -0800

--
 .../operators/LSMPrimaryUpsertOperatorNodePushable.java| 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e19da1fc/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
--
diff --git 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 6bd9d99..e33dfb6 100644
--- 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -244,10 +244,8 @@ public class LSMPrimaryUpsertOperatorNodePushable extends 
LSMIndexInsertUpdateDe
 appender.write(writer, true);
 }
 };
-
-} catch (Exception e) {
-indexHelper.close();
-throw new HyracksDataException(e);
+} catch (Throwable e) { // NOSONAR: Re-thrown
+throw HyracksDataException.create(e);
 }
 }

[5/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index b4990d6..4610fd2 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -180,22 +180,24 @@ public class LSMBTree extends AbstractLSMIndex implements 
ITreeIndex {
 predicate.setLowKey(tuple);
 if (needKeyDupCheck) {
 // first check the inmemory component
+boolean found;
 ctx.getCurrentMutableBTreeAccessor().search(memCursor, predicate);
 try {
-if (memCursor.hasNext()) {
+found = memCursor.hasNext();
+if (found) {
 memCursor.next();
 LSMBTreeTupleReference lsmbtreeTuple = 
(LSMBTreeTupleReference) memCursor.getTuple();
 if (!lsmbtreeTuple.isAntimatter()) {
 throw 
HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
-} else {
-memCursor.destroy();
-
ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple,
-AntimatterAwareTupleAcceptor.INSTANCE);
-return true;
 }
 }
 } finally {
-memCursor.destroy();
+memCursor.close();
+}
+if (found) {
+
ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple,
+AntimatterAwareTupleAcceptor.INSTANCE);
+return true;
 }
 
 // TODO: Can we just remove the above code that search the mutable
@@ -213,7 +215,7 @@ public class LSMBTree extends AbstractLSMIndex implements 
ITreeIndex {
 throw HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
 }
 } finally {
-searchCursor.destroy();
+searchCursor.close();
 // Add the current active mutable component back
 ctx.getComponentHolder().add(0, firstComponent);
 }
@@ -241,7 +243,8 @@ public class LSMBTree extends AbstractLSMIndex implements 
ITreeIndex {
 MultiComparator comp = 
MultiComparator.create(getComparatorFactories());
 ISearchPredicate pred = new RangePredicate(null, null, true, true, 
comp, comp);
 ctx.getSearchInitialState().reset(pred, operationalComponents);
-((LSMBTreeSearchCursor) cursor).scan(ctx.getSearchInitialState(), 
pred);
+ctx.getSearchInitialState().setDiskComponentScan(true);
+((LSMBTreeSearchCursor) cursor).open(ctx.getSearchInitialState(), 
pred);
 }
 
 @Override
@@ -249,46 +252,55 @@ public class LSMBTree extends AbstractLSMIndex implements 
ITreeIndex {
 LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
 LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) 
flushOp.getFlushingComponent();
 IIndexAccessor accessor = 
flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
-
-RangePredicate nullPred = new RangePredicate(null, null, true, true, 
null, null);
-long numElements = 0L;
-if (hasBloomFilter) {
-//count elements in btree for creating Bloomfilter
-IIndexCursor countingCursor = ((BTreeAccessor) 
accessor).createCountingSearchCursor();
-accessor.search(countingCursor, nullPred);
+ILSMDiskComponent component;
+ILSMDiskComponentBulkLoader componentBulkLoader;
+try {
+RangePredicate nullPred = new RangePredicate(null, null, true, 
true, null, null);
+long numElements = 0L;
+if (hasBloomFilter) {
+//count elements in btree for creating Bloomfilter
+IIndexCursor countingCursor = ((BTreeAccessor) 
accessor).createCountingSearchCursor();
+accessor.search(countingCursor, nullPred);
+try {
+while (countingCursor.hasNext()) {
+countingCursor.next();
+ITupleReference countTuple = countingCursor.getTuple();
+numElements =
+

[2/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 68a3984..ff28470 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -60,48 +60,34 @@ import org.apache.hyracks.storage.common.IIndexCursor;
 import org.apache.hyracks.storage.common.MultiComparator;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 
 public class BTreeSearchCursorTest extends AbstractBTreeTest {
-protected final int fieldCount = 2;
-protected final ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
-protected final BTreeTypeAwareTupleWriterFactory tupleWriterFactory =
-new BTreeTypeAwareTupleWriterFactory(typeTraits, false);
-protected final ITreeIndexMetadataFrameFactory metaFrameFactory = new 
LIFOMetaDataFrameFactory();
-protected final Random rnd = new Random(50);
-
-@Override
-@Before
-public void setUp() throws HyracksDataException {
-super.setUp();
-typeTraits[0] = IntegerPointable.TYPE_TRAITS;
-typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-}
+public static final int FIELD_COUNT = 2;
+public static final ITypeTraits[] TYPE_TRAITS = { 
IntegerPointable.TYPE_TRAITS, IntegerPointable.TYPE_TRAITS };
+public static final BTreeTypeAwareTupleWriterFactory TUPLE_WRITER_FACTORY =
+new BTreeTypeAwareTupleWriterFactory(TYPE_TRAITS, false);
+public static final ITreeIndexMetadataFrameFactory META_FRAME_FACTORY = 
new LIFOMetaDataFrameFactory();
+public static final int KEY_FIELDS_COUNT = 1;
+public static final IBinaryComparatorFactory[] CMP_FACTORIES =
+{ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
+public static final ITreeIndexFrameFactory LEAF_FRAME_FACTORY = new 
BTreeNSMLeafFrameFactory(TUPLE_WRITER_FACTORY);
+public static final ITreeIndexFrameFactory INTERIOR_FRAME_FACTORY =
+new BTreeNSMInteriorFrameFactory(TUPLE_WRITER_FACTORY);
+public static final Random RANDOM = new Random(50);
 
 @Test
 public void uniqueIndexTest() throws Exception {
 if (LOGGER.isInfoEnabled()) {
 LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
 }
-
 IBufferCache bufferCache = harness.getBufferCache();
-
 // declare keys
-int keyFieldCount = 1;
-IBinaryComparatorFactory[] cmpFactories = new 
IBinaryComparatorFactory[keyFieldCount];
-cmpFactories[0] = 
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
-
-ITreeIndexFrameFactory leafFrameFactory = new 
BTreeNSMLeafFrameFactory(tupleWriterFactory);
-ITreeIndexFrameFactory interiorFrameFactory = new 
BTreeNSMInteriorFrameFactory(tupleWriterFactory);
-
-IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) 
leafFrameFactory.createFrame();
-IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) 
interiorFrameFactory.createFrame();
-
-IMetadataPageManager freePageManager = new 
LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
-
-BTree btree = new BTree(bufferCache, freePageManager, 
interiorFrameFactory, leafFrameFactory, cmpFactories,
-fieldCount, harness.getFileReference());
+IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) 
LEAF_FRAME_FACTORY.createFrame();
+IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) 
INTERIOR_FRAME_FACTORY.createFrame();
+IMetadataPageManager freePageManager = new 
LinkedMetaDataPageManager(bufferCache, META_FRAME_FACTORY);
+BTree btree = new BTree(bufferCache, freePageManager, 
INTERIOR_FRAME_FACTORY, LEAF_FRAME_FACTORY, CMP_FACTORIES,
+FIELD_COUNT, harness.getFileReference());
 btree.create();
 btree.activate();
 
@@ -111,7 +97,7 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest 
{
 TreeSet uniqueKeys = new TreeSet<>();
 ArrayList keys = new ArrayList<>();
 while (uniqueKeys.size() < numKeys) {
-int key = rnd.nextInt() % maxKey;
+int key = RANDOM.nextInt() % maxKey;
 uniqueKeys.add(key);
 }
 for (Integer i :

[7/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

[ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

- user model changes: no
- storage format changes: no
- interface changes: yes
  - IIndexCursor.close() is now idempotent and can be called on
a closed cursor.
  - IIndexCursor.destroy() is now idempotent and can be called
on a destroyed cursor.
  - Add IIndexAccessor.destroy() letting the accessor know it is
safe to destroy its reusable cursors and operation contexts.
  - Add IIndexOperationContext.destroy() letting the context
know that the user is done with it and allow it to release
resources

details:
- Previously, implementations of the IIndexCursor interface
  didn't enforce the interface contract. This change enforces
  the contract for all the implementations.
- With the enforcement of the contract, all the users of the
  cursors are expected to follow and enforce the expected lifecycle.
- Test cases were added.

Change-Id: I98a7a8b931eb24dbe11bf2bdc61b754ca28ebdf9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2324
Reviewed-by: Michael Blow 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/4ff6a36d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/4ff6a36d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/4ff6a36d

Branch: refs/heads/master
Commit: 4ff6a36d108182e5f36ceac40a27dd89cc29cd23
Parents: 2f392e8
Author: Abdullah Alamoudi 
Authored: Mon Feb 12 09:24:15 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Feb 12 15:27:53 2018 -0800

--
 .../apache/asterix/app/nc/RecoveryManager.java  |  46 +--
 .../message/MetadataNodeRequestMessage.java |   7 +-
 .../asterix/messaging/CCMessageBroker.java  |   2 +-
 .../asterix/test/storage/DiskIsFullTest.java|  60 ++--
 .../dataflow/FeedRecordDataFlowController.java  |  35 +-
 .../indexing/ExternalFileIndexAccessor.java |  13 +-
 .../input/stream/SocketServerInputStream.java   |  37 +--
 .../asterix/external/util/DataflowUtils.java|  37 ++-
 .../apache/asterix/metadata/MetadataNode.java   | 192 ++-
 .../LSMPrimaryUpsertOperatorNodePushable.java   |  26 +-
 .../management/service/logging/LogManager.java  |  67 ++--
 .../hyracks/api/dataflow/IDestroyable.java  |  35 ++
 .../api/exceptions/HyracksDataException.java|  32 +-
 .../apache/hyracks/api/util/DestroyUtils.java   |  53 +++
 .../apache/hyracks/api/util/ExceptionUtils.java | 108 +++
 .../apache/hyracks/control/cc/job/JobRun.java   |   2 +-
 .../control/common/utils/ExceptionUtils.java|  86 -
 .../org/apache/hyracks/control/nc/Task.java |   2 +-
 .../hyracks/control/nc/work/StartTasksWork.java |   2 +-
 .../std/collectors/InputChannelFrameReader.java |   1 -
 .../tests/unit/EnforcedIndexCursorTest.java |  49 ---
 .../hyracks/tests/unit/IIndexCursorTest.java| 263 ---
 .../BTreeUpdateSearchOperatorNodePushable.java  |   4 +-
 .../hyracks/storage/am/btree/impls/BTree.java   |  10 +
 .../btree/impls/BTreeCountingSearchCursor.java  |  25 +-
 .../storage/am/btree/impls/BTreeOpContext.java  |  14 +
 .../am/btree/impls/BTreeRangeSearchCursor.java  |  37 ++-
 .../storage/am/btree/impls/DiskBTree.java   |   5 +-
 .../btree/impls/DiskBTreePointSearchCursor.java |   6 +-
 .../btree/impls/DiskBTreeRangeSearchCursor.java |   6 +-
 .../storage/am/btree/test/FramewriterTest.java  |   2 +
 .../hyracks/hyracks-storage-am-common/pom.xml   |  30 +-
 .../am/common/api/IIndexOperationContext.java   |  15 +-
 .../am/common/api/ITreeIndexAccessor.java   |   2 +
 .../dataflow/IndexDropOperatorNodePushable.java |   3 +-
 .../IndexSearchOperatorNodePushable.java|  61 ++--
 ...eIndexDiskOrderScanOperatorNodePushable.java |  79 +++--
 .../impls/TreeIndexDiskOrderScanCursor.java |  34 +-
 .../am/common/util/ResourceReleaseUtils.java| 158 +
 .../common/test/DummyEnforcedIndexCursor.java   |  59 
 .../am/common/test/EnforcedIndexCursorTest.java |  61 
 .../am/common/test/IIndexCursorTest.java| 294 +
 .../hyracks-storage-am-lsm-btree/pom.xml|  10 +-
 .../lsm/btree/impls/ExternalBTreeOpContext.java |   6 +
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |   6 +-
 .../impls/ExternalBTreeWithBuddyOpContext.java  |   6 +
 .../storage/am/lsm/btree/impls/LSMBTree.java| 175 ++
 .../btree/impls/LSMBTreeCursorInitialState.java |  12 +-
 .../impls/LSMBTreeDiskComponentScanCursor.java  |  33 +-
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |  17 +
 .../btree/impls/LSMBTreePointSearchCursor.java  |  45 ++-
 .../btree/impls/LSMBTreeRangeSearchCursor.java  |  21 +-

[6/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/IIndexCursorTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/IIndexCursorTest.java
 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/IIndexCursorTest.java
deleted file mode 100644
index 4d0f287..000
--- 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/IIndexCursorTest.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.tests.unit;
-
-import org.apache.hyracks.storage.common.ICursorInitialState;
-import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * This is a test class that forms the basis for unit tests of different 
implementations of the IIndexCursor interface
- */
-public abstract class IIndexCursorTest {
-@Test
-public void testNormalLifeCycle() throws Exception {
-IIndexCursor cursor = createCursor();
-ICursorInitialState initialState = createCursorInitialState();
-List predicates = createSearchPredicates();
-for (ISearchPredicate predicate : predicates) {
-cursor.open(initialState, predicate);
-while (cursor.hasNext()) {
-cursor.next();
-}
-cursor.close();
-}
-cursor.destroy();
-}
-
-@Test
-public void testCreateDestroySucceed() throws Exception {
-IIndexCursor cursor = createCursor();
-cursor.destroy();
-}
-
-@Test
-public void testDoubleOpenFails() throws Exception {
-IIndexCursor cursor = createCursor();
-ICursorInitialState initialState = createCursorInitialState();
-List predicates = createSearchPredicates();
-cursor.open(initialState, predicates.get(0));
-boolean expectedExceptionThrown = false;
-try {
-cursor.open(initialState, predicates.get(0));
-} catch (Exception e) {
-expectedExceptionThrown = true;
-}
-cursor.close();
-cursor.destroy();
-Assert.assertTrue(expectedExceptionThrown);
-}
-
-@Test
-public void testCloseWithoutOpenFails() throws Exception {
-IIndexCursor cursor = createCursor();
-boolean expectedExceptionThrown = false;
-try {
-cursor.close();
-} catch (Exception e) {
-expectedExceptionThrown = true;
-}
-cursor.destroy();
-Assert.assertTrue(expectedExceptionThrown);
-}
-
-@Test
-public void testDoubleCloseFails() throws Exception {
-IIndexCursor cursor = createCursor();
-ICursorInitialState initialState = createCursorInitialState();
-List predicates = createSearchPredicates();
-cursor.open(initialState, predicates.get(0));
-cursor.close();
-boolean expectedExceptionThrown = false;
-try {
-cursor.close();
-} catch (Exception e) {
-expectedExceptionThrown = true;
-}
-cursor.destroy();
-Assert.assertTrue(expectedExceptionThrown);
-}
-
-@Test
-public void testHasNextBeforeOpenFails() throws Exception {
-IIndexCursor cursor = createCursor();
-boolean expectedExceptionThrown = false;
-try {
-cursor.hasNext();
-} catch (Exception e) {
-expectedExceptionThrown = true;
-}
-cursor.destroy();
-Assert.assertTrue(expectedExceptionThrown);
-}
-
-@Test
-public void testHasNextAfterCloseFails() throws Exception {
-IIndexCursor cursor = createCursor();
-ICursorInitialState initialState = createCursorInitialState();
-List predicates =

[4/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index 7bf5322..b2660a4 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -81,7 +81,11 @@ public class InMemoryInvertedListCursor implements 
IInvertedListCursor {
 
 @Override
 public int compareTo(IInvertedListCursor cursor) {
-return size() - cursor.size();
+try {
+return size() - cursor.size();
+} catch (HyracksDataException hde) {
+throw new IllegalStateException(hde);
+}
 }
 
 public void reset(ITupleReference tuple) throws HyracksDataException {
@@ -113,7 +117,7 @@ public class InMemoryInvertedListCursor implements 
IInvertedListCursor {
 @Override
 public void unpinPages() throws HyracksDataException {
 if (cursorNeedsClose) {
-btreeCursor.destroy();
+btreeCursor.close();
 cursorNeedsClose = false;
 }
 }
@@ -135,29 +139,22 @@ public class InMemoryInvertedListCursor implements 
IInvertedListCursor {
 }
 
 @Override
-public int size() {
+public int size() throws HyracksDataException {
 if (numElements < 0) {
 btreePred.setLowKeyComparator(tokenFieldsCmp);
 btreePred.setHighKeyComparator(tokenFieldsCmp);
 btreePred.setLowKey(tokenTuple, true);
 btreePred.setHighKey(tokenTuple, true);
-
 // Perform the count.
+btreeAccessor.search(countingCursor, btreePred);
 try {
-btreeAccessor.search(countingCursor, btreePred);
 while (countingCursor.hasNext()) {
 countingCursor.next();
 ITupleReference countTuple = countingCursor.getTuple();
 numElements = 
IntegerPointable.getInteger(countTuple.getFieldData(0), 
countTuple.getFieldStart(0));
 }
-} catch (HyracksDataException e) {
-e.printStackTrace();
 } finally {
-try {
-countingCursor.destroy();
-} catch (HyracksDataException e) {
-e.printStackTrace();
-}
+countingCursor.close();
 }
 }
 return numElements;
@@ -197,7 +194,6 @@ public class InMemoryInvertedListCursor implements 
IInvertedListCursor {
 try {
 containsKey = btreeCursor.hasNext();
 } finally {
-btreeCursor.destroy();
 btreeCursor.close();
 btreeSearchTuple.removeLastTuple();
 }
@@ -219,7 +215,6 @@ public class InMemoryInvertedListCursor implements 
IInvertedListCursor {
 strBuilder.append(o.toString() + " ");
 }
 } finally {
-btreeCursor.destroy();
 btreeCursor.close();
 }
 btreeAccessor.search(btreeCursor, btreePred);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index 75bdbc4..eec2993 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -70,10 +70,7 @@ public class FixedSizeElementInvertedListCursor implements 
IInvertedListCursor {
 
 @Override
 public boolean hasNext() {
-

[1/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 2f392e855 -> 4ff6a36d1


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeSearchCursorTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeSearchCursorTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeSearchCursorTest.java
new file mode 100644
index 000..19b3880
--- /dev/null
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeSearchCursorTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.cursor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
+import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.common.test.IIndexCursorTest;
+import org.apache.hyracks.storage.am.lsm.btree.LSMBTreeExamplesTest;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class LSMBTreeSearchCursorTest extends IIndexCursorTest {
+
+private static final LSMBTreeTestHarness harness = new 
LSMBTreeTestHarness();
+private static LSMBTree lsmBtree;
+
+@BeforeClass
+public static void setup() throws HyracksDataException {
+harness.setUp();
+lsmBtree = LSMBTreeExamplesTest.createTreeIndex(harness, 
LSMBTreePointSearchCursorTest.TYPE_TRAITS,
+LSMBTreePointSearchCursorTest.CMP_FACTORIES, 
LSMBTreePointSearchCursorTest.BLOOM_FILTER_KEY_FIELDS,
+null, null, null, null);
+lsmBtree.create();
+lsmBtree.activate();
+LSMBTreePointSearchCursorTest.insertData(lsmBtree);
+}
+
+@AfterClass
+public static void teardown() throws HyracksDataException {
+try {
+lsmBtree.deactivate();
+lsmBtree.destroy();
+} finally {
+harness.tearDown();
+}
+}
+
+@Override
+protected List createSearchPredicates() throws Exception 
{
+// exact and windows of length = 50
+List predicates = new ArrayList<>();
+for (int i = 0; i < 10; i++) {
+// Build low key.
+ArrayTupleBuilder lowKeyTb = new 
ArrayTupleBuilder(LSMBTreePointSearchCursorTest.KEY_FIELD_COUNT);
+ArrayTupleReference lowKey = new ArrayTupleReference();
+TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -100 + (i * 50));
+// Build high key.
+ArrayTupleBuilder highKeyTb = new 
ArrayTupleBuilder(LSMBTreePointSearchCursorTest.KEY_FIELD_COUNT);
+ArrayTupleReference highKey = new ArrayTupleReference();
+TupleUtils.createIntegerTuple(highKeyTb, highKey, -100 + (i * 50) 
+ 50);
+MultiComparator lowKeySearchCmp =
+
BTreeUtils.getSearchMultiComparator(LSMBTreePointSearchCursorTest.CMP_FACTORIES,
 lowKey);
+MultiComparator highKeySearchCmp =
+
BTreeUtils.getSearchMultiComparator(LSMBTreePointSearchCursorTest.CMP_FACTORIES,
 highKey);
+

[3/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations and usages of IIndexCursor

2018-02-12 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
index 6fe6b60..9ef305c 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
@@ -25,6 +25,7 @@ import 
org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
 import org.apache.hyracks.storage.common.ICursorInitialState;
 import org.apache.hyracks.storage.common.ISearchPredicate;
 import org.apache.hyracks.storage.common.MultiComparator;
@@ -32,7 +33,7 @@ import 
org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 
-public class TreeTupleSorter implements ITreeIndexCursor {
+public class TreeTupleSorter extends EnforcedIndexCursor implements 
ITreeIndexCursor {
 private final static int INITIAL_SIZE = 100;
 private int numTuples;
 private int currentTupleIndex;
@@ -62,13 +63,13 @@ public class TreeTupleSorter implements ITreeIndexCursor {
 }
 
 @Override
-public void close() {
+public void doClose() {
 numTuples = 0;
 currentTupleIndex = 0;
 }
 
 @Override
-public boolean hasNext() throws HyracksDataException {
+public boolean doHasNext() throws HyracksDataException {
 if (numTuples <= currentTupleIndex) {
 return false;
 }
@@ -87,12 +88,12 @@ public class TreeTupleSorter implements ITreeIndexCursor {
 }
 
 @Override
-public void next() {
+public void doNext() {
 currentTupleIndex++;
 }
 
 @Override
-public ITupleReference getTuple() {
+public ITupleReference doGetTuple() {
 return frameTuple1;
 }
 
@@ -180,34 +181,32 @@ public class TreeTupleSorter implements ITreeIndexCursor {
 private int compare(int[] tPointers, int tp1, int tp2i, int tp2j) throws 
HyracksDataException {
 int i1 = tPointers[tp1 * 2];
 int j1 = tPointers[tp1 * 2 + 1];
-
 int i2 = tp2i;
 int j2 = tp2j;
-
 ICachedPage node1 = 
bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i1), false);
-leafFrame1.setPage(node1);
-ICachedPage node2 = 
bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i2), false);
-leafFrame2.setPage(node2);
-
 try {
-frameTuple1.resetByTupleOffset(leafFrame1.getBuffer().array(), j1);
-frameTuple2.resetByTupleOffset(leafFrame2.getBuffer().array(), j2);
-
-return cmp.selectiveFieldCompare(frameTuple1, frameTuple2, 
comparatorFields);
-
+leafFrame1.setPage(node1);
+ICachedPage node2 = 
bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i2), false);
+try {
+leafFrame2.setPage(node2);
+frameTuple1.resetByTupleOffset(leafFrame1.getBuffer().array(), 
j1);
+frameTuple2.resetByTupleOffset(leafFrame2.getBuffer().array(), 
j2);
+return cmp.selectiveFieldCompare(frameTuple1, frameTuple2, 
comparatorFields);
+} finally {
+bufferCache.unpin(node2);
+}
 } finally {
 bufferCache.unpin(node1);
-bufferCache.unpin(node2);
 }
 }
 
 @Override
-public void open(ICursorInitialState initialState, ISearchPredicate 
searchPred) throws HyracksDataException {
+public void doOpen(ICursorInitialState initialState, ISearchPredicate 
searchPred) throws HyracksDataException {
 // do nothing
 }
 
 @Override
-public void destroy() throws HyracksDataException {
+public void doDestroy() throws HyracksDataException {
 // do nothing
 }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
--
diff --git

[1/2] asterixdb git commit: [NO ISSUE][STO] Recover from failure in memory allocation callback

2018-01-30 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 248e18aad -> caf43069b


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/caf43069/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index e9f410d..749b3ba 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -438,9 +438,29 @@ public abstract class AbstractLSMIndex implements 
ILSMIndex {
 if (memoryComponentsAllocated || memoryComponents == null) {
 return;
 }
-for (ILSMMemoryComponent c : memoryComponents) {
-c.allocate();
-ioOpCallback.allocated(c);
+int i = 0;
+boolean allocated = false;
+try {
+for (; i < memoryComponents.size(); i++) {
+allocated = false;
+ILSMMemoryComponent c = memoryComponents.get(i);
+c.allocate();
+allocated = true;
+ioOpCallback.allocated(c);
+}
+} finally {
+if (i < memoryComponents.size()) {
+// something went wrong
+if (allocated) {
+ILSMMemoryComponent c = memoryComponents.get(i);
+c.deallocate();
+}
+// deallocate all previous components
+for (int j = i - 1; j >= 0; j--) {
+ILSMMemoryComponent c = memoryComponents.get(j);
+c.deallocate();
+}
+}
 }
 memoryComponentsAllocated = true;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/caf43069/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
index c0bef7d..3fbef18 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
@@ -237,19 +237,40 @@ public abstract class AbstractLSMMemoryComponent extends 
AbstractLSMComponent im
 
 @Override
 public final void allocate() throws HyracksDataException {
+boolean allocated = false;
 ((IVirtualBufferCache) getIndex().getBufferCache()).open();
-doAllocate();
+try {
+doAllocate();
+allocated = true;
+} finally {
+if (!allocated) {
+((IVirtualBufferCache) getIndex().getBufferCache()).close();
+}
+}
 }
 
 protected void doAllocate() throws HyracksDataException {
-getIndex().create();
-getIndex().activate();
+boolean created = false;
+boolean activated = false;
+try {
+getIndex().create();
+created = true;
+getIndex().activate();
+activated = true;
+} finally {
+if (created && !activated) {
+getIndex().destroy();
+}
+}
 }
 
 @Override
 public final void deallocate() throws HyracksDataException {
-doDeallocate();
-getIndex().getBufferCache().close();
+try {
+doDeallocate();
+} finally {
+getIndex().getBufferCache().close();
+}
 }
 
 protected void doDeallocate() throws HyracksDataException {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/caf43069/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java

[2/2] asterixdb git commit: [NO ISSUE][STO] Recover from failure in memory allocation callback

2018-01-30 Thread amoudi

[NO ISSUE][STO] Recover from failure in memory allocation callback

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, if an exception is thrown in the
  ILSMIOOperationCallback.allocated call, then the memory component
  is allocated but the flag memoryComponentsAllocated is false.
- Any subsequent attempt to modify the index will try to allocate
  the component but since it has already been allocated, it will fail
  with the exception: File is already mapped.
- In this change, if an exception is thrown from the callback, then
  the component is de-allocated before throwing the exception.
- Test is case is added.

Change-Id: I80e605461df18c7f6d7785cd7504ca3acb4f45b1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2336
Reviewed-by: Murtadha Hubail 
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/caf43069
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/caf43069
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/caf43069

Branch: refs/heads/master
Commit: caf43069baf53259b10087745d931e341f109be1
Parents: 248e18a
Author: Abdullah Alamoudi 
Authored: Tue Jan 30 16:13:01 2018 -0800
Committer: abdullah alamoudi 
Committed: Tue Jan 30 21:24:08 2018 -0800

--
 .../test/dataflow/ComponentRollbackTest.java| 428 +--
 .../test/dataflow/IoCallbackFailureTest.java| 159 +++
 .../dataflow/MultiPartitionLSMIndexTest.java|  10 +-
 .../SearchCursorComponentSwitchTest.java|  18 +-
 .../asterix/test/dataflow/StorageTestUtils.java | 293 +
 .../am/common/impls/AbstractTreeIndex.java  |  28 +-
 .../am/lsm/common/api/ILSMMemoryComponent.java  |   3 +-
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  26 +-
 .../impls/AbstractLSMMemoryComponent.java   |  31 +-
 .../impls/MultitenantVirtualBufferCache.java|  10 +-
 .../apache/hyracks/storage/common/IIndex.java   |   1 +
 .../am/lsm/btree/impl/ITestOpCallback.java  |   6 +-
 .../storage/am/lsm/btree/impl/TestLsmBtree.java |  22 +-
 13 files changed, 664 insertions(+), 371 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/caf43069/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
index 9828424..a46b029 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
@@ -20,64 +20,38 @@ package org.apache.asterix.test.dataflow;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.Semaphore;
 import java.util.function.Predicate;
 
 import org.apache.asterix.app.bootstrap.TestNodeController;
 import org.apache.asterix.app.bootstrap.TestNodeController.PrimaryIndexInfo;
-import org.apache.asterix.app.data.gen.TestTupleCounterFrameWriter;
 import org.apache.asterix.app.data.gen.TupleGenerator;
-import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction;
 import org.apache.asterix.app.nc.NCAppRuntimeContext;
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.context.PrimaryIndexOperationTracker;
 import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
 import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
 import org.apache.asterix.common.transactions.ITransactionContext;
 import org.apache.asterix.common.transactions.ITransactionManager;
 import org.apache.asterix.common.transactions.TransactionOptions;
 import org.apache.asterix.external.util.DataflowUtils;
-import org.apache.asterix.file.StorageComponentProvider;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import

asterixdb git commit: [NOISSUE][TEST] Make store and validate requests non cancellable

2018-01-30 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a252d78a6 -> 248e18aad


[NOISSUE][TEST] Make store and validate requests non cancellable

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Make store and validate requests non cancellable.

Change-Id: I722681d96f1b0de286dcf614ff15fc7aa625aadd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2341
Reviewed-by: Murtadha Hubail 
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/248e18aa
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/248e18aa
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/248e18aa

Branch: refs/heads/master
Commit: 248e18aad2e2482b97a19b0c62eccfd6d4799d0e
Parents: a252d78
Author: Abdullah Alamoudi 
Authored: Tue Jan 30 13:29:40 2018 -0800
Committer: abdullah alamoudi 
Committed: Tue Jan 30 15:48:48 2018 -0800

--
 .../java/org/apache/asterix/test/common/TestExecutor.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/248e18aa/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 9cce657..eeea05a 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -126,6 +126,8 @@ public class TestExecutor {
 private static final Pattern MAX_RESULT_READS_PATTERN =
 Pattern.compile("maxresultreads=(\\d+)(\\D|$)", Pattern.MULTILINE);
 public static final int TRUNCATE_THRESHOLD = 16384;
+public static final Set NON_CANCELLABLE =
+Collections.unmodifiableSet(new HashSet<>(Arrays.asList("store", 
"validate")));
 
 public static final String DELIVERY_ASYNC = "async";
 public static final String DELIVERY_DEFERRED = "deferred";
@@ -1221,7 +1223,7 @@ public class TestExecutor {
 }
 final URI uri = getEndpoint(Servlets.QUERY_SERVICE);
 if (DELIVERY_IMMEDIATE.equals(delivery)) {
-resultStream = executeQueryService(statement, fmt, uri, 
params, true, null, true);
+resultStream = executeQueryService(statement, fmt, uri, 
params, true, null, isCancellable(reqType));
 resultStream = METRICS_QUERY_TYPE.equals(reqType) ? 
ResultExtractor.extractMetrics(resultStream)
 : ResultExtractor.extract(resultStream);
 } else {
@@ -1832,4 +1834,8 @@ public class TestExecutor {
 return target;
 }
 }
+
+private static boolean isCancellable(String type) {
+return !NON_CANCELLABLE.contains(type);
+}
 }

asterixdb git commit: [NO ISSUE][ING] Only delete records if found

2018-01-26 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 61cb03102 -> de7660714


[NO ISSUE][ING] Only delete records if found

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Previously, change feeds that perform upserts and deletes
  used to blindly upsert a record for upserts and upsert
  an anti-matter for deletes in the absense of secondary
  indexes or filters.
- This lead to size blow out when there are a lot of deletes
  for keys that don't have values in the index.
- In this change, we only apply deletes strictly when a prevous
  value with the same key is found.

Change-Id: Iac32ef87f9da2725bc1b6381a65dba6390b0d3ae
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2327
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/de766071
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/de766071
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/de766071

Branch: refs/heads/master
Commit: de7660714f467395c7733c94d9656fde912ffbbe
Parents: 61cb031
Author: Abdullah Alamoudi 
Authored: Fri Jan 26 13:31:58 2018 -0800
Committer: abdullah alamoudi 
Committed: Fri Jan 26 17:16:42 2018 -0800

--
 .../operators/LSMPrimaryUpsertOperatorNodePushable.java  | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/de766071/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
--
diff --git 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 0d74e30..21259cc 100644
--- 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -136,8 +136,9 @@ public class LSMPrimaryUpsertOperatorNodePushable extends 
LSMIndexInsertUpdateDe
 tb.reset();
 boolean recordWasInserted = false;
 boolean recordWasDeleted = false;
+boolean isDelete = isDeleteOperation(tuple, 
numOfPrimaryKeys);
 resetSearchPredicate(index);
-if (isFiltered || hasSecondaries) {
+if (isFiltered || isDelete || hasSecondaries) {
 lsmAccessor.search(cursor, searchPred);
 if (cursor.hasNext()) {
 cursor.next();
@@ -154,12 +155,13 @@ public class LSMPrimaryUpsertOperatorNodePushable extends 
LSMIndexInsertUpdateDe
 searchCallback.before(key); // lock
 appendPreviousTupleAsMissing();
 }
-if (isDeleteOperation(tuple, numOfPrimaryKeys)) {
+if (isDelete && prevTuple != null) {
 // Only delete if it is a delete and not upsert
+// And previous tuple with the same key was found
 abstractModCallback.setOp(Operation.DELETE);
 lsmAccessor.forceDelete(tuple);
 recordWasDeleted = true;
-} else {
+} else if (!isDelete) {
 abstractModCallback.setOp(Operation.UPSERT);
 lsmAccessor.forceUpsert(tuple);
 recordWasInserted = true;

asterixdb git commit: [NO ISSUE][STO] Introduce ILSMIndexCursor

2018-01-18 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master bfa06f8de -> d8d4eefbe


[NO ISSUE][STO] Introduce ILSMIndexCursor

- user model changes: no
- storage format changes: no
- interface changes: yes
  - introduce ILSMIndexCursor as a sub interface for IIndexCursor
  - move getFilterMinTuple and getFilterMaxTuple to the
ILSMIndexCursor interface.

details:
- IIndexCursor.getFilterMinTuple and IIndexCursor.getFilterMaxTuple
  don't make sense for any non LSM cursor and all the existing
  implementations return null for those calls.
- In this change, we move those methods to a new interface for
  LSM Index cursors.

Change-Id: I2fabe46e4c3a6370c48c2ccd9689fce507b37009
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2296
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d8d4eefb
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d8d4eefb
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d8d4eefb

Branch: refs/heads/master
Commit: d8d4eefbe168ca82a316925300a236af9f0d8385
Parents: bfa06f8
Author: Abdullah Alamoudi 
Authored: Wed Jan 17 17:53:57 2018 -0800
Committer: abdullah alamoudi 
Committed: Thu Jan 18 12:48:44 2018 -0800

--
 .../hyracks/storage/am/btree/impls/BTree.java   |  1 +
 .../btree/impls/BTreeCountingSearchCursor.java  | 10 --
 .../am/btree/impls/BTreeCursorInitialState.java |  3 +-
 .../storage/am/btree/impls/BTreeOpContext.java  |  2 +-
 .../am/btree/impls/BTreeRangeSearchCursor.java  | 10 --
 .../storage/am/common/api/ILSMIndexCursor.java  | 36 
 .../IndexSearchOperatorNodePushable.java|  5 +--
 .../impls/TreeIndexDiskOrderScanCursor.java | 10 --
 .../btree/impls/LSMBTreePointSearchCursor.java  |  3 +-
 .../lsm/btree/impls/LSMBTreeSearchCursor.java   |  5 +--
 .../impls/LSMBTreeWithBuddyAbstractCursor.java  |  3 +-
 .../lsm/common/impls/LSMIndexSearchCursor.java  |  3 +-
 .../impls/LSMInvertedIndexSearchCursor.java |  3 +-
 .../OnDiskInvertedIndexRangeSearchCursor.java   | 10 --
 .../ondisk/OnDiskInvertedIndexSearchCursor.java | 10 --
 .../lsm/rtree/impls/LSMRTreeAbstractCursor.java |  3 +-
 ...LSMRTreeWithAntiMatterTuplesFlushCursor.java |  3 +-
 .../am/lsm/rtree/impls/TreeTupleSorter.java | 10 --
 .../am/rtree/impls/RTreeSearchCursor.java   | 10 --
 .../storage/common/EnforcedIndexCursor.java | 10 --
 .../hyracks/storage/common/IIndexCursor.java| 11 --
 21 files changed, 57 insertions(+), 104 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d8d4eefb/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 26564e0..6e2d694 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -718,6 +718,7 @@ public class BTree extends AbstractTreeIndex {
 }
 if (restartOp) {
 // Wait for the SMO to persistFrontiers before restarting.
+// We didn't release the pin on the page!!
 treeLatch.readLock().lock();
 treeLatch.readLock().unlock();
 ctx.getPageLsns().removeLast();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d8d4eefb/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
index 82df92c..10f79a2 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
+++

asterixdb git commit: [ASTERIXDB-2204][STO] Add an abstract test case for an IIndexCursor

2018-01-17 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master a03ff60a8 -> a49c7cce0


[ASTERIXDB-2204][STO] Add an abstract test case for an IIndexCursor

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Add a base test for all implementations of IIndexCursor
- The test case includes the following scenarios:
--- testNormalLifeCycle
--- testCreateDestroySucceed
--- testDoubleOpenFails
--- testCloseWithoutOpenFails
--- testDoubleCloseFails
--- testHasNextBeforeOpenFails
--- testHasNextAfterCloseFails
--- testNextBeforeOpenFails
--- testNextAfterCloseFails
--- testDestroyWhileOpenFails
--- testOpenAfterDestroyFails
--- testCloseAfterDestroyFails
--- testNextAfterDestroyFails
--- testHasNextAfterDestroyFails
--- testGetTupleReturnsNullAfterDestroy
- Add a base implementation of the interface that conforms
  to the expected lifecycle and can be extended by
  any cursor implementation.
- The test is run on the base implementation.

Change-Id: I7c32dd560367d84403ffa3d9cb69ff80d715fdc5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2291
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a49c7cce
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a49c7cce
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a49c7cce

Branch: refs/heads/master
Commit: a49c7cce09f7dfb5d0424d7eeef1b61efa6a9aef
Parents: a03ff60
Author: Abdullah Alamoudi 
Authored: Tue Jan 16 17:53:23 2018 -0800
Committer: abdullah alamoudi 
Committed: Wed Jan 17 10:47:36 2018 -0800

--
 .../tests/unit/EnforcedIndexCursorTest.java |  49 
 .../hyracks/tests/unit/IIndexCursorTest.java| 263 +++
 .../storage/common/EnforcedIndexCursor.java | 111 
 3 files changed, 423 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a49c7cce/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/EnforcedIndexCursorTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/EnforcedIndexCursorTest.java
 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/EnforcedIndexCursorTest.java
new file mode 100644
index 000..8fe689e
--- /dev/null
+++ 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/unit/EnforcedIndexCursorTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.tests.unit;
+
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class EnforcedIndexCursorTest extends IIndexCursorTest {
+@Override
+protected List createSearchPredicates() {
+List predicates = new ArrayList<>();
+for (int i = 0; i < 10; i++) {
+predicates.add(Mockito.mock(ISearchPredicate.class));
+}
+return predicates;
+}
+
+@Override
+protected ICursorInitialState createCursorInitialState() {
+return Mockito.mock(ICursorInitialState.class);
+}
+
+@Override
+protected IIndexCursor createCursor() {
+return new EnforcedIndexCursor();
+}
+}

asterixdb git commit: [NO ISSUE][RT] Add test cases for runtime failures

2018-01-17 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master e54115d7f -> a03ff60a8


[NO ISSUE][RT] Add test cases for runtime failures

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- Add three test cases:
  - RuntimeException during initialize()
  - RuntimeException during deinitialize()
  - RuntimeException during both

Change-Id: If26ade138b003349cfd9619188bd9129ecd1034a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2290
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a03ff60a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a03ff60a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a03ff60a

Branch: refs/heads/master
Commit: a03ff60a8328867da5c20853c4407d90cb92f60d
Parents: e54115d
Author: Abdullah Alamoudi 
Authored: Tue Jan 16 18:44:59 2018 -0800
Committer: abdullah alamoudi 
Committed: Wed Jan 17 10:46:53 2018 -0800

--
 .../AbstractMultiNCIntegrationTest.java |  6 +-
 .../tests/integration/JobFailureTest.java   | 66 ---
 ...nitializeDeInitializeOperatorDescriptor.java | 84 
 3 files changed, 142 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a03ff60a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
index a455cc9..7100895 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
@@ -143,7 +143,7 @@ public abstract class AbstractMultiNCIntegrationTest {
 hcc.cancelJob(jobId);
 }
 
-protected void runTest(JobSpecification spec, String expectedErrorMessage) 
throws Exception {
+protected JobId runTest(JobSpecification spec, String 
expectedErrorMessage) throws Exception {
 if (LOGGER.isInfoEnabled()) {
 LOGGER.info(spec.toJSON().asText());
 }
@@ -195,6 +195,7 @@ public abstract class AbstractMultiNCIntegrationTest {
 // Waiting a second time should lead to the same behavior
 waitForCompletion(jobId, expectedErrorMessage);
 dumpOutputFiles();
+return jobId;
 }
 
 protected void waitForCompletion(JobId jobId, String expectedErrorMessage) 
throws Exception {
@@ -244,7 +245,8 @@ public abstract class AbstractMultiNCIntegrationTest {
 @Override
 public JobSubmissionStatus allocate(JobSpecification job) 
throws HyracksException {
 return maxRAM > 
job.getRequiredClusterCapacity().getAggregatedMemoryByteSize()
-? JobSubmissionStatus.EXECUTE : 
JobSubmissionStatus.QUEUE;
+? JobSubmissionStatus.EXECUTE
+: JobSubmissionStatus.QUEUE;
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a03ff60a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
index 34b1480..478138e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
@@ -26,6 +26,7 @@ import

asterixdb git commit: [NO ISSUE][STO] Unpin pages when interrupted during reads

2018-01-15 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master d15f1 -> f3e4397b3


[NO ISSUE][STO] Unpin pages when interrupted during reads

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- This change fixes a bug that happens when a thread pinning
  a page that is not already in the buffer cache is interrupted.
- The fix is that if a failure happens during the read call,
  the page is unpinned.
- A test case is added

Change-Id: I8d1c52fcf89ed90e8ef6019cd77842dd7468df49
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2274
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Michael Blow 
Tested-by: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f3e4397b
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f3e4397b
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f3e4397b

Branch: refs/heads/master
Commit: f3e4397b35804394bcf421d216a53832c0855d55
Parents: d15f888
Author: Abdullah Alamoudi 
Authored: Mon Jan 15 10:08:57 2018 -0800
Committer: abdullah alamoudi 
Committed: Mon Jan 15 16:33:28 2018 -0800

--
 .../storage/common/buffercache/BufferCache.java |  13 ++-
 .../hyracks-storage-common-test/pom.xml |   8 ++
 .../hyracks/storage/common/BufferCacheTest.java | 107 +++
 3 files changed, 126 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f3e4397b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 6212896..302c7b2 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -186,8 +186,17 @@ public class BufferCache implements IBufferCacheInternal, 
ILifeCycleComponent {
 // disk.
 synchronized (cPage) {
 if (!cPage.valid) {
-tryRead(cPage);
-cPage.valid = true;
+try {
+tryRead(cPage);
+cPage.valid = true;
+} catch (Exception e) {
+LOGGER.log(Level.WARN, "Failure while trying to read a 
page from disk", e);
+throw e;
+} finally {
+if (!cPage.valid) {
+unpin(cPage);
+}
+}
 }
 }
 } else {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f3e4397b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
index f93df0a..bb31885 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
@@ -68,5 +68,13 @@
   commons-io
   commons-io
 
+
+  org.apache.commons
+  commons-lang3
+
+
+  org.apache.logging.log4j
+  log4j-api
+
   
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f3e4397b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
index 26ad457..f94914c 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
+++

asterixdb git commit: [NO ISSUE][OTR] Remove Unused IndexOp Enum

2018-01-07 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 333e38ef1 -> 178e4f6d1


[NO ISSUE][OTR] Remove Unused IndexOp Enum

- user model changes: no
- storage format changes: no
- interface changes: no

Change-Id: Ic7ccbeff5a2de5acf16394c30ba1cf7617c25ccb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2264
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/178e4f6d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/178e4f6d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/178e4f6d

Branch: refs/heads/master
Commit: 178e4f6d19b0b99614202da1cfb16c6a563b433f
Parents: 333e38e
Author: Murtadha Hubail 
Authored: Sun Jan 7 00:21:57 2018 +0300
Committer: abdullah alamoudi 
Committed: Sun Jan 7 07:45:48 2018 -0800

--
 .../storage/am/common/ophelpers/IndexOp.java| 24 
 1 file changed, 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/178e4f6d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOp.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOp.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOp.java
deleted file mode 100644
index 7a0d4e8..000
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOp.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.ophelpers;
-
-public enum IndexOp {
-INSERT, DELETE, UPDATE, UPSERT, SEARCH, DISKORDERSCAN, PHYSICALDELETE
-}

[2/2] asterixdb git commit: [ASTERIXDB-2204][STO] Fix the IIndexCursor interface

2018-01-04 Thread amoudi

[ASTERIXDB-2204][STO] Fix the IIndexCursor interface

- user model changes: no
- storage format changes: no
- interface changes: yes
  - replace IIndexCursor.reset with close
  - replace IIndexCursor.close with destroy

Details:
- This change is the first step towards fixing the behavior
  of implementors/callers of the IIndexCursor interface
- In this change, we simply rename the reset -> close
  and close -> destroy and we write down the javadocs
  explaining the semantics of the interface.
- LSM Index Cursors don't implements ITreeIndexCursor
  anymore.

Change-Id: I64cf8c0a5473268bdfd71fd560ee6b3bff219ce9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2238
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Contrib: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/08dc8597
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/08dc8597
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/08dc8597

Branch: refs/heads/master
Commit: 08dc8597e2c9bcbf133250487c71b82d53fd1224
Parents: b4899c8
Author: Abdullah Alamoudi 
Authored: Thu Jan 4 17:56:38 2018 +0300
Committer: abdullah alamoudi 
Committed: Thu Jan 4 16:35:06 2018 -0800

--
 .../indexing/ExternalFileIndexAccessor.java |  4 +-
 .../apache/asterix/metadata/MetadataNode.java   | 23 +--
 .../LSMPrimaryUpsertOperatorNodePushable.java   |  7 ++--
 .../btree/impls/BTreeCountingSearchCursor.java  |  6 +--
 .../am/btree/impls/BTreeRangeSearchCursor.java  |  6 +--
 .../storage/am/btree/test/FramewriterTest.java  |  2 +-
 .../IndexSearchOperatorNodePushable.java|  4 +-
 ...eIndexDiskOrderScanOperatorNodePushable.java |  2 +-
 .../impls/TreeIndexDiskOrderScanCursor.java |  4 +-
 .../am/lsm/btree/impls/ExternalBTree.java   |  3 +-
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  7 ++--
 .../storage/am/lsm/btree/impls/LSMBTree.java| 15 ---
 .../impls/LSMBTreeDiskComponentScanCursor.java  |  4 +-
 .../lsm/btree/impls/LSMBTreeMergeOperation.java |  4 +-
 .../btree/impls/LSMBTreePointSearchCursor.java  | 42 ++--
 .../btree/impls/LSMBTreeRangeSearchCursor.java  | 14 +++
 .../lsm/btree/impls/LSMBTreeSearchCursor.java   | 32 ---
 .../impls/LSMBTreeWithBuddyAbstractCursor.java  | 32 ---
 .../impls/LSMBTreeWithBuddyMergeOperation.java  |  4 +-
 .../impls/LSMBTreeWithBuddySearchCursor.java| 18 -
 .../impls/LSMBTreeWithBuddySortedCursor.java| 12 +++---
 .../am/lsm/common/impls/AbstractLSMIndex.java   | 12 +++---
 .../lsm/common/impls/LSMIndexSearchCursor.java  | 29 +++---
 .../lsm/common/impls/LSMTreeIndexAccessor.java  |  5 +--
 .../invertedindex/impls/LSMInvertedIndex.java   | 10 ++---
 .../LSMInvertedIndexRangeSearchCursor.java  |  4 +-
 .../impls/LSMInvertedIndexSearchCursor.java | 16 
 .../inmemory/InMemoryInvertedListCursor.java| 12 +++---
 .../ondisk/OnDiskInvertedIndex.java |  4 +-
 .../OnDiskInvertedIndexRangeSearchCursor.java   |  8 ++--
 .../ondisk/OnDiskInvertedIndexSearchCursor.java |  4 +-
 .../ondisk/PartitionedOnDiskInvertedIndex.java  |  2 +-
 .../am/lsm/rtree/impls/ExternalRTree.java   |  9 ++---
 .../storage/am/lsm/rtree/impls/LSMRTree.java| 14 +++
 .../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 33 +--
 .../lsm/rtree/impls/LSMRTreeMergeOperation.java |  4 +-
 .../lsm/rtree/impls/LSMRTreeSearchCursor.java   | 18 -
 .../lsm/rtree/impls/LSMRTreeSortedCursor.java   | 12 +++---
 .../impls/LSMRTreeWithAntiMatterTuples.java | 12 +++---
 ...LSMRTreeWithAntiMatterTuplesFlushCursor.java | 23 ++-
 ...SMRTreeWithAntiMatterTuplesSearchCursor.java | 25 ++--
 .../am/lsm/rtree/impls/TreeTupleSorter.java |  4 +-
 .../am/rtree/impls/RTreeSearchCursor.java   |  6 +--
 .../hyracks/storage/common/IIndexCursor.java| 37 ++---
 .../AbstractSearchOperationCallbackTest.java|  2 +-
 .../am/btree/OrderedIndexExamplesTest.java  |  9 ++---
 .../storage/am/btree/OrderedIndexTestUtils.java |  9 ++---
 .../am/common/AbstractIndexTestWorker.java  |  2 +-
 .../storage/am/common/TreeIndexTestUtils.java   | 10 +++--
 .../am/rtree/AbstractRTreeExamplesTest.java | 12 +++---
 .../storage/am/rtree/RTreeTestUtils.java|  8 ++--
 .../storage/am/btree/BTreeSearchCursorTest.java |  2 +-
 .../storage/am/btree/BTreeUpdateSearchTest.java |  4 +-
 .../am/btree/multithread/BTreeTestWorker.java   |  8 ++--
 .../LSMBTreeSearchOperationCallbackTest.java|  2 +-
 ...TreeUpdateInPlaceScanDiskComponentsTest.java |  2 +-
 .../btree/multithread/LSMBTreeTestWorker.java   |  4 +-

asterixdb git commit: [NO ISSUE][TEST] Store a query output for subsequent test calls

2018-01-04 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 4756d1ca0 -> 32905418e


[NO ISSUE][TEST] Store a query output for subsequent test calls

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- With this change, the test framework allow for a query to store
  its output for access by subsequent calls. The change also allow
  for a query to validate its output against a previous query.
- This change also fixes a bug in Function datasource field
  access.

Change-Id: Iab329fbdf4def5b46686a61c9399520ae71518c4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2235
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/32905418
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/32905418
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/32905418

Branch: refs/heads/master
Commit: 32905418e6560496aa33df25ad497075605cfdae
Parents: 4756d1c
Author: Abdullah Alamoudi 
Authored: Thu Jan 4 04:30:49 2018 +0300
Committer: abdullah alamoudi 
Committed: Thu Jan 4 08:32:02 2018 -0800

--
 .../optimizer/rules/PushFieldAccessRule.java|  23 ++--
 .../app/function/StorageComponentsFunction.java |   2 +-
 .../app/function/StorageComponentsReader.java   |   6 +-
 .../asterix/test/common/TestExecutor.java   | 112 +--
 .../validate-expected.1.store.sqlpp |  21 
 .../validate-expected.2.validate.sqlpp  |  20 
 .../resources/runtimets/testsuite_sqlpp.xml |   5 +
 7 files changed, 140 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/32905418/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
index e0fb1db..1766010 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
@@ -99,8 +99,8 @@ public class PushFieldAccessRule implements 
IAlgebraicRewriteRule {
 @SuppressWarnings("unchecked")
 private boolean isAccessToIndexedField(AssignOperator assign, 
IOptimizationContext context)
 throws AlgebricksException {
-AbstractFunctionCallExpression accessFun = 
(AbstractFunctionCallExpression) assign.getExpressions().get(0)
-.getValue();
+AbstractFunctionCallExpression accessFun =
+(AbstractFunctionCallExpression) 
assign.getExpressions().get(0).getValue();
 ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
 if (e0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
 return false;
@@ -161,8 +161,7 @@ public class PushFieldAccessRule implements 
IAlgebraicRewriteRule {
 if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) {
 return false;
 }
-ILogicalExpression e1 = (ILogicalExpression) access.getAnnotations()
-.get(OperatorAnnotation.FIELD_ACCESS);
+ILogicalExpression e1 = (ILogicalExpression) 
access.getAnnotations().get(OperatorAnnotation.FIELD_ACCESS);
 if (e1 == null) {
 return false;
 }
@@ -215,8 +214,7 @@ public class PushFieldAccessRule implements 
IAlgebraicRewriteRule {
 pushItDown = true;
 } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
 GroupByOperator g = (GroupByOperator) op2;
-List> varMappings =
-new ArrayList<>();
+List> varMappings = new 
ArrayList<>();
 for (Pair p : 
g.getDecorList()) {
 ILogicalExpression e = p.second.getValue();
 if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
@@ -249,8 +247,8 @@ public class PushFieldAccessRule implements 
IAlgebraicRewriteRule {
 }
 if (pushItDown) {
 if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-Mutable childOfSubplan = 
((NestedTupleSourceOperator) op2).getDataSourceReference()
-

asterixdb git commit: [NO ISSUE][ING] Stop Succeeds When Feed Fails

2018-01-03 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 665e9fe4d -> 36b543d3b


[NO ISSUE][ING] Stop Succeeds When Feed Fails

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Before this change, if a feed fails after stop is issued, it
  goes to a failure state and the stop request will be stuck
  waiting for the feed to go to a stopped state.
- With this change, a failure in the feed will cause it to go
  to a stopped state if a stop request was issued.
- Test case is added

Change-Id: I486e11a339c1dfadc6a2a357c95a83dfc05772a1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2258
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Murtadha Hubail 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/36b543d3
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/36b543d3
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/36b543d3

Branch: refs/heads/master
Commit: 36b543d3b8a1e9c178d2c354aa2d544693be0832
Parents: 665e9fe
Author: Abdullah Alamoudi 
Authored: Wed Jan 3 15:06:30 2018 +0300
Committer: abdullah alamoudi 
Committed: Wed Jan 3 06:18:08 2018 -0800

--
 .../app/active/ActiveEntityEventsListener.java  | 10 --
 .../test/active/ActiveEventsListenerTest.java   | 21 
 .../org/apache/asterix/test/active/Actor.java   |  5 +
 .../asterix/test/active/TestEventsListener.java |  5 +++--
 .../api/util/SingleThreadEventProcessor.java|  2 +-
 5 files changed, 38 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/36b543d3/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 8cbc109..12bcc0d 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -193,9 +193,9 @@ public abstract class ActiveEntityEventsListener implements 
IActiveEntityControl
 if (jobStatus.equals(JobStatus.FAILURE)) {
 jobFailure = exceptions.isEmpty() ? new 
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
 : exceptions.get(0);
-setState(ActivityState.TEMPORARILY_FAILED);
+setState((state == ActivityState.STOPPING) ? ActivityState.STOPPED 
: ActivityState.TEMPORARILY_FAILED);
 if (prevState != ActivityState.SUSPENDING && prevState != 
ActivityState.RECOVERING
-&& prevState != ActivityState.RESUMING) {
+&& prevState != ActivityState.RESUMING && prevState != 
ActivityState.STOPPING) {
 recover();
 }
 } else {
@@ -551,4 +551,10 @@ public abstract class ActiveEntityEventsListener 
implements IActiveEntityControl
 public String getDisplayName() throws HyracksDataException {
 return this.getEntityId().toString();
 }
+
+@Override
+public String toString() {
+return "{\"class\":\"" + getClass().getSimpleName() + "\"" + 
"\"entityId\":\"" + entityId + "\""
++ "\"state\":\"" + state + "\"" + "}";
+}
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/36b543d3/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index 40d4f6a..093d150 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -346,6 +346,27 @@ public class ActiveEventsListenerTest {
 Assert.assertEquals(ActivityState.RUNNING, listener.getState());
 }
 
+@Test
+public void testStopFromRunningAndJobFails() throws Exception {
+testStartWhenStartSucceed();
+Assert.assertEquals(ActivityState.RUNNING, listener.getState());
+listener.onStop(Behavior.STEP_SUCCEED);
+

asterixdb git commit: [ASTERIXDB-920] Fix BTree Composite Key Search

2017-12-21 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master d90723359 -> 791ff5f2d


[ASTERIXDB-920] Fix BTree Composite Key Search

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
When we search a primary BTree with composite keys, we should always add
another select operator to filter out results, unless only the last key
is a range search. The reason is that when we search a BTree, we iterate
from the start index (based on the composite low range) to the end index
(based on the composite high range). However, during the iteration,
there could be a lot of false positives.

This patch fixes this bug by preserving the select operator in case of
composite keys.

Change-Id: I8654e52aec59601c533255cd57c2824612066fc0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2227
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: abdullah alamoudi 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/791ff5f2
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/791ff5f2
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/791ff5f2

Branch: refs/heads/master
Commit: 791ff5f2d58e1f42d1851131496ac14e981d458e
Parents: d907233
Author: luochen01 
Authored: Fri Dec 22 06:30:39 2017 +0800
Committer: abdullah alamoudi 
Committed: Thu Dec 21 22:36:32 2017 -0800

--
 .../optimizer/rules/am/BTreeAccessMethod.java   | 23 +--
 .../primary-composite-key-join_01.plan  | 13 +++---
 .../primary-composite-key-join_02.plan  | 13 +++---
 .../results/btree-index/btree-primary-15.plan   | 11 ++---
 .../results/btree-index/btree-primary-16.plan   | 11 ++---
 .../results/btree-index/btree-primary-19.plan   | 11 ++---
 .../results/btree-index/btree-primary-20.plan   | 11 ++---
 .../primary-composite-key-join_01.plan  | 23 ++-
 .../primary-composite-key-join_02.plan  | 23 ++-
 .../btree-index/btree-primary-15.plan   | 13 --
 .../btree-index/btree-primary-16.plan   | 13 --
 .../btree-index/btree-primary-19.plan   | 13 --
 .../btree-index/btree-primary-20.plan   | 13 --
 .../skip-index/dont-skip-primary-index.plan | 11 ++---
 .../composite-low-high.1.ddl.sqlpp  | 43 
 .../composite-low-high.2.update.sqlpp   | 38 +
 .../composite-low-high.3.query.sqlpp| 27 
 .../query-ASTERIXDB-920.1.ddl.sqlpp | 40 ++
 .../query-ASTERIXDB-920.2.update.sqlpp  | 32 +++
 .../query-ASTERIXDB-920.3.query.sqlpp   | 26 
 .../composite-low-high/composite-low-high.1.adm |  3 ++
 .../query-ASTERIXDB-920.1.adm   |  1 +
 .../resources/runtimets/testsuite_sqlpp.xml | 12 ++
 23 files changed, 339 insertions(+), 85 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/791ff5f2/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index ddc93f8..29d00d0 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -416,22 +416,21 @@ public class BTreeAccessMethod implements IAccessMethod {
 return null;
 }
 
-// If the select condition contains mixed open/closed intervals on 
multiple keys, then we make all intervals
-// closed to obtain a superset of answers and leave the original 
selection in place.
+// if we have composite search keys, we should always need a 
post-processing to ensure the correctness
+// of search results because of the way a BTree is searched, unless 
only the last key is a range search.
+// During a BTree search, we iterate from the start index
+// (based on the low keys) to the end index (based on the high keys). 
During the iteration,
+// we can encounter a lot of false positives
 boolean primaryIndexPostProccessingIsNeeded = false;
-for (int i = 1; i < numSecondaryKeys; ++i) {
-if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
-

asterixdb git commit: [NO ISSUE][TEST] Allow Diagnostic files to be run for failing tests

2017-12-20 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 56295d9bb -> ee30e8bde


[NO ISSUE][TEST] Allow Diagnostic files to be run for failing tests

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- With this change, one can add diagnostic files to test cases
  those files are not to be executed during the tests but can be
  run to collect information in failing test cases.

Change-Id: I798726514cd46bcee04a8f290df54ae5bb4fb0a6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2230
Sonar-Qube: Jenkins 
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ee30e8bd
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ee30e8bd
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ee30e8bd

Branch: refs/heads/master
Commit: ee30e8bde16b86fbe67c579aa1209c90addb9db9
Parents: 56295d9
Author: Abdullah Alamoudi 
Authored: Wed Dec 20 09:00:43 2017 +0300
Committer: abdullah alamoudi 
Committed: Wed Dec 20 10:50:34 2017 -0800

--
 .../asterix/test/common/TestExecutor.java   | 95 ++--
 1 file changed, 68 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ee30e8bd/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index a86dbf9..d66c2e9 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -127,8 +127,10 @@ public class TestExecutor {
 public static final String DELIVERY_ASYNC = "async";
 public static final String DELIVERY_DEFERRED = "deferred";
 public static final String DELIVERY_IMMEDIATE = "immediate";
+public static final String DIAGNOSE = "diagnose";
 private static final String METRICS_QUERY_TYPE = "metrics";
 
+
 private static final HashMap runningTestServers = 
new HashMap<>();
 private static Map ncEndPoints;
 private static Map replicationAddress;
@@ -213,7 +215,7 @@ public class TestExecutor {
 if (lineExpected.isEmpty()) {
 continue;
 }
-throw createLineChangedException(scriptFile, lineExpected, 
" after " + num + " lines", num);
+throw createLineChangedException(scriptFile, lineExpected, 
"", num);
 }
 
 // Comparing result equality but ignore "Time"-prefixed 
fields. (for metadata tests.)
@@ -248,7 +250,7 @@ public class TestExecutor {
 }
 lineActual = readerActual.readLine();
 if (lineActual != null) {
-throw createLineChangedException(scriptFile, "expected ", 
lineActual, num);
+throw createLineChangedException(scriptFile, "", 
lineActual, num);
 }
 } catch (Exception e) {
 System.err.println("Actual results file: " + 
actualFile.toString());
@@ -262,8 +264,8 @@ public class TestExecutor {
 
 private ComparisonException createLineChangedException(File scriptFile, 
String lineExpected, String lineActual,
 int num) {
-return new ComparisonException("Result for " + scriptFile + " changed 
at line " + num + ":\n< "
-+ truncateIfLong(lineExpected) + "\n> " + 
truncateIfLong(lineActual));
+return new ComparisonException("Result for " + scriptFile + " changed 
at line " + num + ":\nexpected < "
++ truncateIfLong(lineExpected) + "\nactual   > " + 
truncateIfLong(lineActual));
 }
 
 private String truncateIfLong(String string) {
@@ -888,8 +890,9 @@ public class TestExecutor {
 executeScript(pb, pb.environment().get("SCRIPT_HOME") + 
File.separator + "dml_recovery"
 + File.separator + "stop_and_start.sh");
 }
-expectedResultFile = (queryCount.intValue() >= 
expectedResultFileCtxs.size()) ? null
-: 
expectedResultFileCtxs.get(queryCount.intValue()).getFile();
+expectedResultFile =
+(queryCount.intValue() < 0 ||

[1/2] asterixdb git commit: [ASTERIXDB-2194][COMP] Introduce datasource functions

2017-12-17 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 26cc90802 -> b4d166b3c


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
new file mode 100644
index 000..c143a63
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.util;
+
+import org.apache.asterix.app.function.DatasetResourcesRewriter;
+import org.apache.asterix.app.function.DatasetRewriter;
+import org.apache.asterix.app.function.FeedRewriter;
+import org.apache.asterix.app.function.StorageComponentsRewriter;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.utils.RecordUtil;
+
+public class MetadataBuiltinFunctions {
+
+static {
+// Dataset function
+BuiltinFunctions.addFunction(BuiltinFunctions.DATASET, 
DatasetRewriter.INSTANCE, true);
+BuiltinFunctions.addUnnestFun(BuiltinFunctions.DATASET, false);
+BuiltinFunctions.addDatasourceFunction(BuiltinFunctions.DATASET, 
DatasetRewriter.INSTANCE);
+// Feed collect function
+BuiltinFunctions.addPrivateFunction(BuiltinFunctions.FEED_COLLECT, 
FeedRewriter.INSTANCE, true);
+BuiltinFunctions.addUnnestFun(BuiltinFunctions.FEED_COLLECT, false);
+BuiltinFunctions.addDatasourceFunction(BuiltinFunctions.FEED_COLLECT, 
FeedRewriter.INSTANCE);
+// Dataset resources function
+
BuiltinFunctions.addPrivateFunction(DatasetResourcesRewriter.DATASET_RESOURCES,
+(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, 
true);
+
BuiltinFunctions.addUnnestFun(DatasetResourcesRewriter.DATASET_RESOURCES, 
false);
+
BuiltinFunctions.addDatasourceFunction(DatasetResourcesRewriter.DATASET_RESOURCES,
+DatasetResourcesRewriter.INSTANCE);
+// Dataset components function
+
BuiltinFunctions.addPrivateFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
+(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, 
true);
+
BuiltinFunctions.addUnnestFun(StorageComponentsRewriter.STORAGE_COMPONENTS, 
false);
+
BuiltinFunctions.addDatasourceFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
+StorageComponentsRewriter.INSTANCE);
+
+}
+
+private MetadataBuiltinFunctions() {
+}
+
+public static void init() {
+// Only execute the static block
+}
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index 9a4..ac249b7 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -54,7 +54,6 @@ import 
org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
 import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable;
 import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
 import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.FeedConstants;
 import org.apache.asterix.external.util.FeedUtils;
 import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
 import org.apache.asterix.file.StorageComponentProvider;
@@ -87,6 +86,7 @@ import org.apache.asterix.metadata.entities.FeedConnection;
 import org.apache.asterix.metadata.entities.FeedPolicyEntity;
 import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
 import

asterixdb git commit: [NO ISSUE][CLUS] Allow Extension of cc application context

2017-12-05 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 8d868fbdd -> 3180d8702


[NO ISSUE][CLUS] Allow Extension of cc application context

- user model changes: no
- storage format changes: no
- interface changes: yes
  + ICcApplicationContext.setExtensionManager
  + ICcApplicationContext.getExtensionProperties

details:
- This change allows better extension of cc application context.
  While this was possible before this change, the cc can end up
  with multiple app context which can lead to bad situations.

Change-Id: I924716b037bf94b68dd9ffa2ec0043c75f4e81a6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2192
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Till Westmann 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3180d870
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3180d870
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3180d870

Branch: refs/heads/master
Commit: 3180d87021f7d94119e5de2890da96560122504b
Parents: 8d868fb
Author: Abdullah Alamoudi 
Authored: Mon Dec 4 17:41:05 2017 -0800
Committer: abdullah alamoudi 
Committed: Tue Dec 5 11:11:54 2017 -0800

--
 .../extension/IAlgebraExtensionManager.java | 23 ---
 .../api/http/server/ClusterApiServlet.java  |  3 +-
 .../asterix/app/cc/CCExtensionManager.java  | 18 +++---
 .../hyracks/bootstrap/CCApplication.java| 28 
 .../bootstrap/ClusterLifecycleListener.java |  7 +-
 .../asterix/test/common/TestExecutor.java   | 67 ++--
 .../common/dataflow/ICcApplicationContext.java  | 13 
 .../runtime/utils/CcApplicationContext.java |  3 +-
 .../hyracks/http/test/HttpServerTest.java   |  7 +-
 9 files changed, 85 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3180d870/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
--
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
deleted file mode 100644
index d9641df..000
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.algebra.extension;
-
-public interface IAlgebraExtensionManager {
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3180d870/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index 165c104..6826b26 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -28,7 +28,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.runtime.utils.CcApplicationContext;
 import org.apache.hyracks.api.config.IOption;
 import org.apache.hyracks.api.config.Section;
 import org.apache.hyracks.control.common.config.ConfigUtils;
@@ -97,7 +96,7 @@ public class ClusterApiServlet extends AbstractServlet {
 
 protected ObjectNode getClusterStateJSON(IServletRequest request, String 
pathToNode) {

[2/2] asterixdb git commit: [ASTERIXDB-2161][TEST] Add indexes to MultiPartitionTest

2017-11-27 Thread amoudi

[ASTERIXDB-2161][TEST] Add indexes to MultiPartitionTest

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- This change adds secondary indexes to multi partition
  LSM indexes tests.
- This enables testing of specific concurrency scenarios
  and ensuring properties stored in primary and secondary
  indexes are consistent.
- In addition, the call for flushDataset in
  DatasetLifecycleManager now throws an
  IllegalStateException if the number of active
  operations is not 0. Some tests used to call this
  function when there are ongoing operations and that
  is expected to never be the case in the actual system.

Change-Id: I5aea71a87f149b01f6c7310867fc15b5a340b93c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2173
Tested-by: Jenkins 
Reviewed-by: Luo Chen 
Integration-Tests: Jenkins 
Contrib: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c5a0a197
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c5a0a197
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c5a0a197

Branch: refs/heads/master
Commit: c5a0a1974d36d647a22d606d53bdaafd85f641df
Parents: f9e6bae
Author: Abdullah Alamoudi 
Authored: Mon Nov 27 11:53:30 2017 -0800
Committer: abdullah alamoudi 
Committed: Mon Nov 27 13:11:06 2017 -0800

--
 .../app/bootstrap/TestNodeController.java   | 213 ++-
 .../test/dataflow/ComponentRollbackTest.java| 139 +++--
 .../asterix/test/dataflow/LogMarkerTest.java|   2 +-
 .../dataflow/MultiPartitionLSMIndexTest.java| 561 +--
 .../SearchCursorComponentSwitchTest.java|  21 +-
 .../TestLsmBTreeResourceFactoryProvider.java|   4 +
 .../TestLsmBtreeIoOpCallbackFactory.java|  40 ++
 .../TestPrimaryIndexOperationTracker.java   |  68 +++
 ...TestPrimaryIndexOperationTrackerFactory.java |  68 +++
 .../asterix/test/logging/CheckpointingTest.java |   2 +-
 .../asterix/test/storage/DiskIsFullTest.java|   2 +-
 .../multi-partition-test-configuration.xml  | 112 
 .../common/context/DatasetLifecycleManager.java |  25 +-
 .../lsm/common/api/ILSMIOOperationCallback.java |   1 +
 .../am/lsm/btree/impl/ITestOpCallback.java  |   6 +-
 .../btree/impl/IVirtualBufferCacheCallback.java |  23 +
 .../storage/am/lsm/btree/impl/TestLsmBtree.java | 182 +-
 .../btree/impl/TestLsmBtreeLocalResource.java   |   7 +
 .../btree/impl/TestLsmBtreeSearchCursor.java|   5 +-
 .../lsm/btree/impl/TestVirtualBufferCache.java  | 215 +++
 20 files changed, 1539 insertions(+), 157 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c5a0a197/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index 72a9b44..352a5f8 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -69,6 +69,8 @@ import 
org.apache.asterix.transaction.management.service.logging.LogReader;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.runtime.base.IPushRuntime;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import 
org.apache.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
 import 
org.apache.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
 import org.apache.hyracks.api.comm.IFrameWriter;
 import org.apache.hyracks.api.comm.VSizeFrame;
@@ -89,6 +91,7 @@ import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.api.util.HyracksConstants;
 import org.apache.hyracks.dataflow.common.utils.TaskUtil;
 import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
 import 
org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import 
org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorNodePushable;
 import org.apache.hyracks.storage.am.common.api.IIndexBuilder;
@@ -166,33 +169,93 @@ public class TestNodeController {
 public Pair 
getInsertPipeline(IHyracksTaskContext ctx,
 Dataset dataset, IAType[] primaryKeyTypes,

[1/2] asterixdb git commit: [ASTERIXDB-2161][TEST] Add indexes to MultiPartitionTest

2017-11-27 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master f9e6bae98 -> c5a0a1974


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c5a0a197/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
new file mode 100644
index 000..e376ff9
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.asterix.common.context.PrimaryIndexOperationTracker;
+import org.apache.asterix.common.transactions.ILogManager;
+import org.apache.asterix.common.transactions.LogRecord;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
+
+public class TestPrimaryIndexOperationTracker extends 
PrimaryIndexOperationTracker {
+
+private final List callbacks = new ArrayList<>();
+
+public TestPrimaryIndexOperationTracker(int datasetID, ILogManager 
logManager, DatasetInfo dsInfo,
+ILSMComponentIdGenerator idGenerator) {
+super(datasetID, logManager, dsInfo, idGenerator);
+}
+
+public void addCallback(ITestOpCallback callback) {
+synchronized (callbacks) {
+callbacks.add(callback);
+}
+}
+
+public void clearCallbacks() {
+synchronized (callbacks) {
+callbacks.clear();
+}
+}
+
+@Override
+public void triggerScheduleFlush(LogRecord logRecord) throws 
HyracksDataException {
+synchronized (callbacks) {
+for (ITestOpCallback callback : callbacks) {
+callback.before(null);
+}
+}
+super.triggerScheduleFlush(logRecord);
+synchronized (callbacks) {
+for (ITestOpCallback callback : callbacks) {
+callback.after();
+}
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c5a0a197/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
--
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
new file mode 100644
index 000..5d7a7c6
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import

[2/3] asterixdb git commit: [ASTERIXDB-2169][STO][TX] Unblock modifications during full scan

2017-11-20 Thread amoudi

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e5a65429/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
--
diff --git 
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
 
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
index f6ebf19..25a4aea 100644
--- 
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
+++ 
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
@@ -78,33 +78,48 @@ public class AlgebricksMetaOperatorDescriptor extends 
AbstractSingleActivityOper
 public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext 
ctx,
 final IRecordDescriptorProvider recordDescProvider, int partition, 
int nPartitions) {
 if (inputArity == 0) {
-return createSourceInputPushRuntime(ctx);
+return new SourcePushRuntime(ctx);
 } else {
 return createOneInputOneOutputPushRuntime(ctx, recordDescProvider);
 }
 }
 
-private IOperatorNodePushable createSourceInputPushRuntime(final 
IHyracksTaskContext ctx) {
-return new AbstractUnaryOutputSourceOperatorNodePushable() {
+private class SourcePushRuntime extends 
AbstractUnaryOutputSourceOperatorNodePushable {
+private final IHyracksTaskContext ctx;
 
-@Override
-public void initialize() throws HyracksDataException {
-IFrameWriter startOfPipeline;
-RecordDescriptor pipelineOutputRecordDescriptor =
-outputArity > 0 ? 
AlgebricksMetaOperatorDescriptor.this.outRecDescs[0] : null;
-PipelineAssembler pa =
-new PipelineAssembler(pipeline, inputArity, 
outputArity, null, pipelineOutputRecordDescriptor);
-startOfPipeline = pa.assemblePipeline(writer, ctx);
+public SourcePushRuntime(IHyracksTaskContext ctx) {
+this.ctx = ctx;
+}
+
+@Override
+public void initialize() throws HyracksDataException {
+IFrameWriter startOfPipeline;
+RecordDescriptor pipelineOutputRecordDescriptor =
+outputArity > 0 ? 
AlgebricksMetaOperatorDescriptor.this.outRecDescs[0] : null;
+PipelineAssembler pa =
+new PipelineAssembler(pipeline, inputArity, outputArity, 
null, pipelineOutputRecordDescriptor);
+startOfPipeline = pa.assemblePipeline(writer, ctx);
+HyracksDataException exception = null;
+try {
+startOfPipeline.open();
+} catch (Exception e) {
+startOfPipeline.fail();
+exception = HyracksDataException.create(e);
+} finally {
 try {
-startOfPipeline.open();
-} catch (Exception e) {
-startOfPipeline.fail();
-throw e;
-} finally {
 startOfPipeline.close();
+} catch (Exception e) {
+if (exception == null) {
+exception = HyracksDataException.create(e);
+} else {
+exception.addSuppressed(e);
+}
 }
 }
-};
+if (exception != null) {
+throw exception;
+}
+}
 }
 
 private IOperatorNodePushable createOneInputOneOutputPushRuntime(final 
IHyracksTaskContext ctx,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e5a65429/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index d64858b..71ddbc0 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -49,8 +49,8 @@ public class HttpServer {
 // Constants
 private static final int LOW_WRITE_BUFFER_WATER_MARK = 8 * 1024;
 private static final int HIGH_WRITE_BUFFER_WATER_MARK = 32 * 1024;
-protected static final WriteBufferWaterMark WRITE_BUFFER_WATER_MARK = new

[3/3] asterixdb git commit: [ASTERIXDB-2169][STO][TX] Unblock modifications during full scan

2017-11-20 Thread amoudi

[ASTERIXDB-2169][STO][TX] Unblock modifications during full scan

- user model changes: no
- storage format changes: no
- interface changes: yes
  - added ILSMHarness.replaceMemoryComponentsWithDiskComponents

details:
- During a long running query aka full scan, two things block
  incoming modifications:
  1) Memory component gets full, is flushed but can't be recycled
 because of the search operation inside the component.
  2) Read latches on the memory component not being released and
 the memory component search cursor is not advancing.
  The two cases are addressed in this change for the LSMBTree but
  not yet addressed for other indexes.
  The proposed solution for case (1) is to poll memory components
  states every n records during the search operation. If a memory
  component was found to have been flushed, its cursor is moved
  to the corresponding disk component allowing the memory
  component to be recycled.
  The proposed solution for case (2) is to check memory component
  cursor every n records. If the cursor has not advanced and the
  component has writers, then the latches over the leaf page are
  released, and the cursor re-do the operation entering from the
  tree root.
- Added a test case.
- Added performance traces for enter and exit components.

Change-Id: I37ba52f6324ed1c5a78465c3a8cbcd351f1ed5bc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2166
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Luo Chen 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e5a65429
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e5a65429
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e5a65429

Branch: refs/heads/master
Commit: e5a65429d94c34b3079a3fe6bad206089c2c193c
Parents: 76ecc4b
Author: Abdullah Alamoudi 
Authored: Mon Nov 20 17:14:01 2017 -0800
Committer: abdullah alamoudi 
Committed: Mon Nov 20 21:32:04 2017 -0800

--
 .../app/bootstrap/TestNodeController.java   |  58 +--
 .../test/dataflow/ComponentRollbackTest.java|  94 +++--
 .../asterix/test/dataflow/LogMarkerTest.java|   7 +-
 .../dataflow/MultiPartitionLSMIndexTest.java| 196 ++
 .../SearchCursorComponentSwitchTest.java| 266 +
 .../asterix/test/dataflow/TestDataset.java  |   3 +-
 .../TestLsmBtreeIoOpCallbackFactory.java|   8 +-
 .../asterix/test/logging/CheckpointingTest.java |   7 +-
 .../asterix/test/storage/DeallocatableTest.java |   4 +-
 .../asterix/test/storage/DiskIsFullTest.java|   8 +-
 .../IndexDropOperatorNodePushableTest.java  |  28 +-
 .../AbstractLSMIOOperationCallback.java |  10 +-
 ...actOneInputOneOutputOneFramePushRuntime.java |  16 +-
 .../meta/AlgebricksMetaOperatorDescriptor.java  |  49 ++-
 .../apache/hyracks/http/server/HttpServer.java  |  10 +-
 .../dataflow/ExternalBTreeLocalResource.java|   2 +-
 .../ExternalBTreeWithBuddyLocalResource.java|   2 +-
 .../am/lsm/btree/impls/ExternalBTree.java   |  27 +-
 .../lsm/btree/impls/ExternalBTreeOpContext.java |   8 +-
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  26 +-
 .../impls/ExternalBTreeWithBuddyOpContext.java  |   9 +-
 .../storage/am/lsm/btree/impls/LSMBTree.java|  14 +-
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |  13 +-
 .../btree/impls/LSMBTreeRangeSearchCursor.java  | 152 +++-
 .../am/lsm/btree/utils/LSMBTreeUtil.java|   9 +-
 .../am/lsm/common/api/ILSMComponent.java|   5 +
 .../storage/am/lsm/common/api/ILSMHarness.java  |  13 +
 .../storage/am/lsm/common/api/ILSMIndex.java|   5 +
 .../common/api/ILSMIndexOperationContext.java   |  26 ++
 .../lsm/common/impls/AbstractLSMComponent.java  |   7 +-
 .../common/impls/AbstractLSMDiskComponent.java  |   9 +-
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  39 +-
 .../impls/AbstractLSMIndexOperationContext.java |  42 ++-
 .../impls/AbstractLSMMemoryComponent.java   |   6 +
 .../impls/ComponentReplacementContext.java  | 215 +++
 .../am/lsm/common/impls/LSMComponentId.java |   8 +-
 .../storage/am/lsm/common/impls/LSMHarness.java | 373 +++
 .../impls/LSMIndexDiskComponentBulkLoader.java  |   2 +-
 .../common/impls/LSMIndexReplicationJob.java|   2 +-
 .../lsm/common/impls/LSMIndexSearchCursor.java  |  19 +-
 .../invertedindex/impls/LSMInvertedIndex.java   |  14 +-
 .../impls/LSMInvertedIndexOpContext.java|  11 +-
 .../ondisk/OnDiskInvertedIndexOpContext.java|   1 +
 .../dataflow/ExternalRTreeLocalResource.java|   2 +-
 .../am/lsm/rtree/impls/AbstractLSMRTree.java|  10 +-
 .../am/lsm/rtree/impls/ExternalRTree.java

[1/3] asterixdb git commit: [ASTERIXDB-2169][STO][TX] Unblock modifications during full scan

2017-11-20 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/master 76ecc4b19 -> e5a65429d


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e5a65429/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index b0abeb1..e1d5114 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -63,6 +63,7 @@ public class LSMHarness implements ILSMHarness {
 private static final Logger LOGGER = 
Logger.getLogger(LSMHarness.class.getName());
 
 protected final ILSMIndex lsmIndex;
+protected final ComponentReplacementContext componentReplacementCtx;
 protected final ILSMMergePolicy mergePolicy;
 protected final ILSMOperationTracker opTracker;
 protected final AtomicBoolean fullMergeIsRequested;
@@ -84,83 +85,97 @@ public class LSMHarness implements ILSMHarness {
 if (replicationEnabled) {
 this.componentsToBeReplicated = new ArrayList<>();
 }
+componentReplacementCtx = new ComponentReplacementContext(lsmIndex);
 }
 
 protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, 
LSMOperationType opType,
 boolean isTryOperation) throws HyracksDataException {
-validateOperationEnterComponentsState(ctx);
-synchronized (opTracker) {
-while (true) {
-lsmIndex.getOperationalComponents(ctx);
-// Before entering the components, prune those corner cases 
that indeed should not proceed.
-switch (opType) {
-case FLUSH:
-// if the lsm index does not have memory components 
allocated, then nothing to flush
-if (!lsmIndex.isMemoryComponentsAllocated()) {
-return false;
-}
-ILSMMemoryComponent flushingComponent = 
(ILSMMemoryComponent) ctx.getComponentHolder().get(0);
-if (!flushingComponent.isModified()) {
-if (flushingComponent.getState() == 
ComponentState.READABLE_UNWRITABLE) {
-//The mutable component has not been modified 
by any writer. There is nothing to flush.
-//since the component is empty, set its state 
back to READABLE_WRITABLE only when it's
-//state has been set to READABLE_UNWRITABLE
-
flushingComponent.setState(ComponentState.READABLE_WRITABLE);
-opTracker.notifyAll();
-
-// Call recycled only when we change it's 
state is reset back to READABLE_WRITABLE
-// Otherwise, if the component is in other 
state, e.g., INACTIVE, or
-// READABLE_UNWRITABLE_FLUSHING, it's not 
considered as being recycled here.
-
lsmIndex.getIOOperationCallback().recycled(flushingComponent);
+long before = 0L;
+if (ctx.isTracingEnabled()) {
+before = System.nanoTime();
+}
+try {
+validateOperationEnterComponentsState(ctx);
+synchronized (opTracker) {
+while (true) {
+lsmIndex.getOperationalComponents(ctx);
+// Before entering the components, prune those corner 
cases that indeed should not proceed.
+switch (opType) {
+case FLUSH:
+// if the lsm index does not have memory 
components allocated, then nothing to flush
+if (!lsmIndex.isMemoryComponentsAllocated()) {
+return false;
 }
-return false;
-}
-if (flushingComponent.getWriterCount() > 0) {
-/*
- * This case is a case where even though FLUSH log 
was flushed to disk and scheduleFlush is triggered,
- * the current in-memory component (whose state 
was changed to READABLE_WRITABLE (RW)
- * from READABLE_UNWRITABLE(RU) before FLUSH log 
was written to log tail (which is memory buffer of log file)
- * and then the

asterixdb git commit: [NO ISSUE][TX] Fix DatasetLock for Multiple Index Builds

2017-11-12 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/release-0.9.3-pre-rc ed4b04680 -> 19a2b58c3


[NO ISSUE][TX] Fix DatasetLock for Multiple Index Builds

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- The mechanism used for allowing multiple concurrent index
  builds does not work if the first index build finishes
  before other index builds. It relied on a write lock
  obtained by the first index builder and released by
  the last index builder. This is not allowed when using
  ReentrantReadWriteLock and will lead to an
  IllegalMonitorStateException since the last thread to exit
  did not hold the lock.
- Potential Deadlock between modify and exclusive modify can
  happen when an exclusive modify attempts to upgrade the
  lock to write lock while the modify lock waits for the
  exclusive lock to release. This has been fixed too.
- Test cases were added.

Change-Id: I3bea3ff2075d952ab13402b0c445c464b431c0f5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2144
Integration-Tests: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Reviewed-by: Michael Blow 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/19a2b58c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/19a2b58c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/19a2b58c

Branch: refs/heads/release-0.9.3-pre-rc
Commit: 19a2b58c32d56060a8167a2ad69bb942083ba4c1
Parents: ed4b046
Author: Abdullah Alamoudi 
Authored: Sun Nov 12 17:21:34 2017 -0800
Committer: abdullah alamoudi 
Committed: Sun Nov 12 22:19:43 2017 -0800

--
 .../apache/asterix/app/active/RecoveryTask.java |   5 +-
 .../asterix/metadata/lock/DatasetLock.java  | 111 ---
 .../metadata/lock/MetadataLockManagerTest.java  | 307 +++
 3 files changed, 384 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19a2b58c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
--
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
index 29b38ba..0321ae4 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
@@ -134,8 +134,9 @@ public class RecoveryTask {
 
lockManager.acquireActiveEntityWriteLock(metadataProvider.getLocks(),
 listener.getEntityId().getDataverse() + '.' + 
listener.getEntityId().getEntityName());
 for (Dataset dataset : listener.getDatasets()) {
-MetadataLockUtil.modifyDatasetBegin(lockManager, 
metadataProvider.getLocks(),
-dataset.getDataverseName(), 
DatasetUtil.getFullyQualifiedName(dataset));
+
lockManager.acquireDataverseReadLock(metadataProvider.getLocks(), 
dataset.getDataverseName());
+
lockManager.acquireDatasetExclusiveModificationLock(metadataProvider.getLocks(),
+DatasetUtil.getFullyQualifiedName(dataset));
 }
 synchronized (listener) {
 try {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/19a2b58c/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java
--
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java
index 31f2089..bc302c2 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java
@@ -24,23 +24,31 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.MetadataException;
 import org.apache.asterix.common.metadata.IMetadataLock;
+import org.apache.asterix.common.utils.InterruptUtil;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 
 public class DatasetLock implements IMetadataLock {
 
 private final String key;
+// The lock
 private final ReentrantReadWriteLock lock;
-private final ReentrantReadWriteLock dsReadLock;
-private final

asterixdb git commit: [NO ISSUE][HYR] Heartbeat processing / NC exceptions

2017-11-08 Thread amoudi

Repository: asterixdb
Updated Branches:
  refs/heads/release-0.9.3-pre-rc e32cb6765 -> ed4b04680


[NO ISSUE][HYR] Heartbeat processing / NC exceptions

- Run heartbeat processing off of Worker thread, to prevent starvation
- Preserve stacktrace when creating node-scoped HyracksDataException
  clones

Change-Id: If8fd35a7fd488bed5f1d5e2146dd48892cb0a7a4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2138
Reviewed-by: abdullah alamoudi 
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Contrib: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ed4b0468
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ed4b0468
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ed4b0468

Branch: refs/heads/release-0.9.3-pre-rc
Commit: ed4b04680224ce9209bf417247d6c0424c4dfc3d
Parents: e32cb67
Author: Michael Blow 
Authored: Wed Nov 8 21:12:12 2017 -0500
Committer: abdullah alamoudi 
Committed: Wed Nov 8 21:31:19 2017 -0800

--
 .../hyracks/api/exceptions/HyracksDataException.java  | 10 --
 .../apache/hyracks/control/cc/ClusterControllerIPCI.java  |  3 +--
 2 files changed, 9 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ed4b0468/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksDataException.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksDataException.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksDataException.java
index 4517730..7ae7cbf 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksDataException.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksDataException.java
@@ -38,7 +38,7 @@ public class HyracksDataException extends HyracksException {
 return (HyracksDataException) cause;
 } else if (cause instanceof Error) {
 // don't wrap errors, allow them to propagate
-throw (Error)cause;
+throw (Error) cause;
 } else if (cause instanceof InterruptedException && 
!Thread.currentThread().isInterrupted()) {
 // TODO(mblow): why not force interrupt on current thread?
 LOGGER.log(Level.WARNING,
@@ -78,6 +78,12 @@ public class HyracksDataException extends HyracksException {
 super(component, errorCode, message, cause, nodeId, params);
 }
 
+public HyracksDataException(String component, int errorCode, String 
message, Throwable cause, String nodeId,
+StackTraceElement[] stackTrace, Serializable... params) {
+super(component, errorCode, message, cause, nodeId, params);
+setStackTrace(stackTrace);
+}
+
 /**
  * @deprecated Error code is needed.
  */
@@ -141,6 +147,6 @@ public class HyracksDataException extends HyracksException {
 
 public static HyracksDataException create(HyracksDataException e, String 
nodeId) {
 return new HyracksDataException(e.getComponent(), e.getErrorCode(), 
e.getMessage(), e.getCause(), nodeId,
-e.getParams());
+e.getStackTrace(), e.getParams());
 }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ed4b0468/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
index 350984c..af5c102 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
@@ -72,8 +72,7 @@ class ClusterControllerIPCI implements IIPCI {
 break;
 case NODE_HEARTBEAT:
 CCNCFunctions.NodeHeartbeatFunction nhf = 
(CCNCFunctions.NodeHeartbeatFunction) fn;
-ccs.getWorkQueue().schedule(new NodeHeartbeatWork(ccs, 
nhf.getNodeId(),
-nhf.getHeartbeatData()));
+

[5/5] asterixdb git commit: Merge pre-rc changes to master

2017-11-06 Thread amoudi

Merge pre-rc changes to master

Change-Id: I578ffd7ef17784034b14f3c0d23cd5094e39f6e2


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a0490ba6
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a0490ba6
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a0490ba6

Branch: refs/heads/master
Commit: a0490ba6f46f28470052b3cadeec1a60be0f394e
Parents: 40cff7b e32cb67
Author: Ian Maxon 
Authored: Mon Nov 6 15:41:03 2017 -0800
Committer: Ian Maxon 
Committed: Mon Nov 6 15:41:03 2017 -0800

--
 .../TestLsmBtreeIoOpCallbackFactory.java|  9 +++--
 .../common/context/DatasetLifecycleManager.java | 15 
 .../asterix/common/context/DatasetResource.java | 11 +++---
 .../AbstractLSMIOOperationCallback.java | 36 +---
 .../LSMBTreeIOOperationCallback.java|  5 +--
 .../LSMBTreeIOOperationCallbackFactory.java |  5 +--
 .../LSMBTreeWithBuddyIOOperationCallback.java   |  5 +++
 ...TreeWithBuddyIOOperationCallbackFactory.java |  5 +--
 .../LSMInvertedIndexIOOperationCallback.java|  5 +--
 ...InvertedIndexIOOperationCallbackFactory.java |  5 +--
 .../LSMRTreeIOOperationCallback.java|  5 +--
 .../LSMRTreeIOOperationCallbackFactory.java |  5 +--
 .../LSMBTreeIOOperationCallbackTest.java| 11 +++---
 ...SMBTreeWithBuddyIOOperationCallbackTest.java | 11 +++---
 ...LSMInvertedIndexIOOperationCallbackTest.java | 11 +++---
 .../LSMRTreeIOOperationCallbackTest.java| 11 +++---
 .../am/common/impls/AbstractTreeIndex.java  |  5 ---
 .../dataflow/ExternalBTreeLocalResource.java|  2 +-
 .../ExternalBTreeWithBuddyLocalResource.java|  2 +-
 .../btree/dataflow/LSMBTreeLocalResource.java   |  4 +--
 .../am/lsm/btree/impls/ExternalBTree.java   | 12 +++
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 10 ++
 .../storage/am/lsm/btree/impls/LSMBTree.java| 20 +--
 .../btree/impls/LSMBTreeMemoryComponent.java|  5 +--
 .../impls/LSMBTreeWithBuddyMemoryComponent.java |  9 ++---
 .../am/lsm/btree/utils/LSMBTreeUtil.java| 19 ++-
 .../AbstractLSMWithBuddyMemoryComponent.java|  6 ++--
 .../lsm/common/api/ILSMIOOperationCallback.java | 16 -
 .../api/ILSMIOOperationCallbackFactory.java |  2 +-
 .../storage/am/lsm/common/api/ILSMIndex.java|  5 +++
 .../lsm/common/impls/AbstractLSMComponent.java  |  4 ++-
 .../common/impls/AbstractLSMDiskComponent.java  |  4 +--
 .../am/lsm/common/impls/AbstractLSMIndex.java   | 17 -
 .../impls/AbstractLSMMemoryComponent.java   |  6 ++--
 .../BlockingIOOperationCallbackWrapper.java | 10 --
 .../storage/am/lsm/common/impls/LSMHarness.java | 13 ---
 .../impls/NoOpIOOperationCallbackFactory.java   | 12 +--
 .../common/impls/StubIOOperationCallback.java   | 20 +++
 .../lsm/common/impls/ThreadCountingTracker.java |  2 +-
 .../dataflow/LSMInvertedIndexLocalResource.java | 12 +++
 .../invertedindex/impls/LSMInvertedIndex.java   | 11 +++---
 .../impls/LSMInvertedIndexMemoryComponent.java  |  8 ++---
 .../impls/PartitionedLSMInvertedIndex.java  | 14 
 .../inmemory/InMemoryInvertedIndex.java |  5 ---
 .../ondisk/OnDiskInvertedIndex.java |  5 ---
 .../invertedindex/util/InvertedIndexUtils.java  | 22 ++--
 .../dataflow/ExternalRTreeLocalResource.java|  2 +-
 .../rtree/dataflow/LSMRTreeLocalResource.java   |  4 +--
 .../LSMRTreeWithAntiMatterLocalResource.java|  4 +--
 .../am/lsm/rtree/impls/AbstractLSMRTree.java| 14 
 .../am/lsm/rtree/impls/ExternalRTree.java   | 13 +++
 .../storage/am/lsm/rtree/impls/LSMRTree.java| 16 +
 .../rtree/impls/LSMRTreeMemoryComponent.java|  8 ++---
 .../impls/LSMRTreeWithAntiMatterTuples.java |  7 ++--
 .../am/lsm/rtree/utils/LSMRTreeUtils.java   | 27 ---
 .../apache/hyracks/storage/common/IIndex.java   |  7 ++--
 .../am/lsm/btree/LSMBTreeBulkLoadTest.java  |  4 +--
 .../am/lsm/btree/LSMBTreeDeleteTest.java|  4 +--
 .../am/lsm/btree/LSMBTreeExamplesTest.java  |  2 +-
 .../am/lsm/btree/LSMBTreeFileManagerTest.java   |  8 ++---
 .../am/lsm/btree/LSMBTreeFilterMergeTest.java   |  4 +--
 .../btree/LSMBTreeFilterMergeTestDriver.java|  3 +-
 .../am/lsm/btree/LSMBTreeInsertTest.java|  4 +--
 .../am/lsm/btree/LSMBTreeLifecycleTest.java |  4 +--
 .../storage/am/lsm/btree/LSMBTreeMergeTest.java |  4 +--
 .../am/lsm/btree/LSMBTreeMergeTestDriver.java   |  3 +-
 ...MBTreeModificationOperationCallbackTest.java |  9 +++--
 .../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java |  4 +--
 .../btree/LSMBTreeScanDiskComponentsTest.java   | 15 
 .../LSMBTreeSearchOperationCallbackTest.java|  2 +-
 ...TreeUpdateInPlaceScanDiskComponentsTest.java | 11 +++---

1 2 3 4 >

1 - 100 of 355 matches

Mail list logo