[2/5] asterixdb git commit: Change the API for writing and reading metadata pages

2017-01-06 Thread amoudi 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
index 7a75534..999f72b 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
@@ -20,21 +20,19 @@
 package org.apache.hyracks.storage.am.lsm.common.frames;
 
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
 import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterReference;
 
 public class LSMComponentFilterFrameFactory implements 
ILSMComponentFilterFrameFactory {
 
 private final ITreeIndexTupleWriterFactory tupleWriterFactory;
-private final int pageSize;
 
-public LSMComponentFilterFrameFactory(ITreeIndexTupleWriterFactory 
tupleWriterFactory, int pageSize) {
+public LSMComponentFilterFrameFactory(ITreeIndexTupleWriterFactory 
tupleWriterFactory) {
 this.tupleWriterFactory = tupleWriterFactory;
-this.pageSize = pageSize;
 }
 
 @Override
-public ILSMComponentFilterFrame createFrame() {
-return new 
LSMComponentFilterFrame(tupleWriterFactory.createTupleWriter(), pageSize);
+public ILSMComponentFilterReference createFrame() {
+return new 
LSMComponentFilterReference(tupleWriterFactory.createTupleWriter());
 }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
new file mode 100644
index 000..f04cbb0
--- /dev/null
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/frames/LSMComponentFilterReference.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.frames;
+
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import 
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterReference;
+
+/**
+ * When written to disk:
+ * [min set?][max set?][min length][min][max length][max]
+ */
+public class LSMComponentFilterReference implements 
ILSMComponentFilterReference {
+
+private static final int MIN_SET_INDICATOR_OFFSET = 0;
+private static final int MAX_SET_INDICATOR_OFFSET = 1;
+
+private final ArrayBackedValueStorage min;
+private final ArrayBackedValueStorage max;
+private ArrayBackedValueStorage binaryFilter;
+private final ITreeIndexTupleWriter tupleWriter;
+

[4/5] asterixdb git commit: Change the API for writing and reading metadata pages

2017-01-06 Thread amoudi 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index 9505f8a..e59523c 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -45,8 +45,8 @@ import 
org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
 
 public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements 
IBTreeInteriorFrame {
 
-private static final int rightLeafOff = flagOff + 1; // 22
-private static final int childPtrSize = 4;
+private static final int RIGHT_LEAF_OFFSET = 
TreeIndexNSMFrame.RESERVED_HEADER_SIZE;
+private static final int CHILD_PTR_SIZE = 4;
 
 private final ITreeIndexTupleReference cmpFrameTuple;
 private final ITreeIndexTupleReference previousFt;
@@ -61,13 +61,13 @@ public class BTreeNSMInteriorFrame extends 
TreeIndexNSMFrame implements IBTreeIn
 
 @Override
 public int getBytesRequiredToWriteTuple(ITupleReference tuple) {
-return tupleWriter.bytesRequired(tuple) + childPtrSize + 
slotManager.getSlotSize();
+return tupleWriter.bytesRequired(tuple) + CHILD_PTR_SIZE + 
slotManager.getSlotSize();
 }
 
 @Override
 public void initBuffer(byte level) {
 super.initBuffer(level);
-buf.putInt(rightLeafOff, -1);
+buf.putInt(RIGHT_LEAF_OFFSET, -1);
 }
 
 @Override
@@ -82,7 +82,7 @@ public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame 
implements IBTreeIn
 
 @Override
 public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) throws 
HyracksDataException {
-int tupleSize = tupleWriter.bytesRequired(tuple) + childPtrSize;
+int tupleSize = tupleWriter.bytesRequired(tuple) + CHILD_PTR_SIZE;
 if (tupleSize > getMaxTupleSize(buf.capacity())) {
 return FrameOpSpaceStatus.TOO_LARGE;
 }
@@ -99,28 +99,29 @@ public class BTreeNSMInteriorFrame extends 
TreeIndexNSMFrame implements IBTreeIn
 
 @Override
 public void insert(ITupleReference tuple, int tupleIndex) {
-int slotOff = slotManager.insertSlot(tupleIndex, 
buf.getInt(freeSpaceOff));
-int freeSpace = buf.getInt(freeSpaceOff);
+int slotOff = slotManager.insertSlot(tupleIndex, 
buf.getInt(Constants.FREE_SPACE_OFFSET));
+int freeSpace = buf.getInt(Constants.FREE_SPACE_OFFSET);
 int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, 
tuple.getFieldCount(), buf.array(), freeSpace);
 System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), 
getLeftChildPageOff(tuple), buf.array(),
-freeSpace + bytesWritten, childPtrSize);
-int tupleSize = bytesWritten + childPtrSize;
-buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
-buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
-buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - 
tupleSize - slotManager.getSlotSize());
+freeSpace + bytesWritten, CHILD_PTR_SIZE);
+int tupleSize = bytesWritten + CHILD_PTR_SIZE;
+buf.putInt(Constants.TUPLE_COUNT_OFFSET, 
buf.getInt(Constants.TUPLE_COUNT_OFFSET) + 1);
+buf.putInt(Constants.FREE_SPACE_OFFSET, 
buf.getInt(Constants.FREE_SPACE_OFFSET) + tupleSize);
+buf.putInt(TOTAL_FREE_SPACE_OFFSET, 
buf.getInt(TOTAL_FREE_SPACE_OFFSET) - tupleSize - slotManager
+.getSlotSize());
 // Did we insert into the rightmost slot?
 if (slotOff == slotManager.getSlotEndOff()) {
-System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), 
getLeftChildPageOff(tuple) + childPtrSize,
-buf.array(), rightLeafOff, childPtrSize);
+System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), 
getLeftChildPageOff(tuple) + CHILD_PTR_SIZE,
+buf.array(), RIGHT_LEAF_OFFSET, CHILD_PTR_SIZE);
 } else {
 // If slotOff has a right (slot-)neighbor then update its child 
pointer.
 // The only time when this is NOT the case, is when this is the 
very first tuple
 // (or when the splitkey goes into the rightmost slot but that 
case is handled in the if above).
-if (buf.getInt(tupleCountOff) > 1) {
+if (buf.getInt(Constants.TUPLE_COUNT_OFFSET) > 1) {
 int

[5/5] asterixdb git commit: Change the API for writing and reading metadata pages

2017-01-06 Thread amoudi 
Change the API for writing and reading metadata pages

Change-Id: Iadad522ab5568677aa816c74fc1d63acad505380
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1396
Sonar-Qube: Jenkins 
Reviewed-by: Ian Maxon 
Tested-by: Jenkins 
BAD: Jenkins 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/90cdbac7
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/90cdbac7
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/90cdbac7

Branch: refs/heads/master
Commit: 90cdbac7f118d85deb5377520ea99e3187d21725
Parents: 69ab91a
Author: Abdullah Alamoudi 
Authored: Thu Jan 5 18:18:48 2017 -0800
Committer: abdullah alamoudi 
Committed: Fri Jan 6 21:48:02 2017 -0800

--
 .../AbstractLSMIOOperationCallback.java |  19 +-
 .../LSMBTreeIOOperationCallback.java|   8 +-
 .../LSMBTreeWithBuddyIOOperationCallback.java   |   6 +-
 .../LSMInvertedIndexIOOperationCallback.java|   6 +-
 .../LSMRTreeIOOperationCallback.java|   8 +-
 .../asterix/common/transactions/LogRecord.java  |   2 +-
 .../asterix/common/utils/StorageConstants.java  |   4 +-
 .../apache/asterix/test/aql/TestExecutor.java   |   1 -
 .../metadata/bootstrap/MetadataBootstrap.java   |   4 +-
 .../TupleCopyValueExtractor.java|   2 +-
 .../management/ReplicationChannel.java  |   8 +-
 .../management/ReplicationManager.java  |  18 +-
 .../storage/LSMComponentLSNSyncTask.java|   6 +-
 .../storage/LSMComponentProperties.java |   8 +-
 .../storage/LSMIndexFileProperties.java |   4 +-
 .../PersistentLocalResourceRepository.java  |   7 +-
 .../data/std/primitive/IntegerPointable.java|   2 +-
 .../data/std/primitive/LongPointable.java   |   6 +
 .../data/std/util/ArrayBackedValueStorage.java  |   4 +
 .../util/ByteArrayAccessibleOutputStream.java   |  14 +-
 .../hyracks/data/std/util/GrowableArray.java|   4 +
 .../hdfs/lib/RawBinaryComparatorFactory.java|  26 +-
 .../frames/BTreeFieldPrefixNSMLeafFrame.java| 193 +++--
 .../am/btree/frames/BTreeNSMInteriorFrame.java  | 147 +-
 .../am/btree/frames/BTreeNSMLeafFrame.java  |  34 ++-
 .../hyracks/storage/am/btree/impls/BTree.java   |   4 +-
 .../storage/am/btree/impls/BTreeOpContext.java  |   4 +-
 .../storage/am/btree/impls/BTreeSplitKey.java   |  14 +-
 .../impls/FieldPrefixPrefixTupleReference.java  |   2 +-
 .../btree/impls/FieldPrefixTupleReference.java  |  19 +-
 .../hyracks/hyracks-storage-am-common/pom.xml   |   5 +
 .../am/common/api/IMetadataPageManager.java |  53 ++--
 .../storage/am/common/api/IPageManager.java |  12 +-
 .../storage/am/common/api/ITreeIndexFrame.java  |  17 +-
 .../am/common/api/ITreeIndexMetaDataFrame.java  |  85 --
 .../api/ITreeIndexMetaDataFrameFactory.java |  24 --
 .../am/common/api/ITreeIndexMetadataFrame.java  | 165 +++
 .../api/ITreeIndexMetadataFrameFactory.java |  25 ++
 .../am/common/api/ITreeIndexTupleReference.java |   4 +-
 .../am/common/dataflow/IndexDataflowHelper.java |   4 +-
 .../am/common/frames/LIFOMetaDataFrame.java | 281 +--
 .../common/frames/LIFOMetaDataFrameFactory.java |   8 +-
 .../am/common/frames/TreeIndexNSMFrame.java | 117 
 .../AppendOnlyLinkedMetadataPageManager.java| 203 +-
 .../freepage/LinkedMetaDataPageManager.java | 192 -
 .../freepage/MutableArrayValueReference.java|  53 
 .../am/common/impls/AbstractTreeIndex.java  |   4 +-
 .../am/common/tuples/SimpleTupleReference.java  |  22 +-
 .../common/tuples/TypeAwareTupleReference.java  |  12 +-
 .../common/util/TreeIndexBufferCacheWarmup.java |   4 +-
 .../storage/am/common/util/TreeIndexStats.java  |   4 +-
 .../am/common/util/TreeIndexStatsGatherer.java  |   4 +-
 .../am/common/frames/LIFOMetadataFrameTest.java |  68 +
 .../storage/am/lsm/btree/impls/LSMBTree.java|   8 +-
 .../lsm/btree/impls/LSMBTreeDiskComponent.java  |   3 +-
 .../btree/tuples/LSMBTreeTupleReference.java|   9 +-
 .../am/lsm/btree/util/LSMBTreeUtils.java|   5 +-
 .../hyracks-storage-am-lsm-common/pom.xml   |  12 +-
 .../common/api/ILSMComponentFilterFrame.java|  44 ---
 .../api/ILSMComponentFilterFrameFactory.java|   2 +-
 .../common/api/ILSMComponentFilterManager.java  |   2 -
 .../api/ILSMComponentFilterReference.java   |  40 +++
 .../common/frames/LSMComponentFilterFrame.java  | 108 ---
 .../frames/LSMComponentFilterFrameFactory.java  |  10 +-
 .../frames/LSMComponentFilterReference.java | 163 +++
 .../common/freepage/VirtualFreePageManager.java |  14 +-

[1/5] asterixdb git commit: Change the API for writing and reading metadata pages

2017-01-06 Thread amoudi 
Repository: asterixdb
Updated Branches:
  refs/heads/master 69ab91aae -> 90cdbac7f


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 6dfeffd..b880a01 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -65,7 +65,7 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest {
 private final int fieldCount = 2;
 private final ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
 private final TypeAwareTupleWriterFactory tupleWriterFactory = new 
TypeAwareTupleWriterFactory(typeTraits);
-private final ITreeIndexMetaDataFrameFactory metaFrameFactory = new 
LIFOMetaDataFrameFactory();
+private final ITreeIndexMetadataFrameFactory metaFrameFactory = new 
LIFOMetaDataFrameFactory();
 private final Random rnd = new Random(50);
 
 @Before

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
index 2c82fab..797ee49 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
@@ -22,7 +22,7 @@ import org.apache.hyracks.dataflow.common.util.SerdeUtils;
 import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
 import org.apache.hyracks.storage.am.btree.util.BTreeTestHarness;
 import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrameFactory;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
 import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 
@@ -35,7 +35,7 @@ public class BTreeSearchOperationCallbackTest extends 
AbstractSearchOperationCal
 
 @Override
 protected void createIndexInstance() throws Exception {
-ITreeIndexMetaDataFrameFactory metaFrameFactory = new 
LIFOMetaDataFrameFactory();
+ITreeIndexMetadataFrameFactory metaFrameFactory = new 
LIFOMetaDataFrameFactory();
 LinkedMetaDataPageManager freePageManager = new 
LinkedMetaDataPageManager(harness.getBufferCache(),
 metaFrameFactory);
 index = BTreeUtils.createBTree(harness.getBufferCache(), 
harness.getFileMapProvider(),

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
index 489c2c7..e4c1449 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -89,11 +89,11 @@ public class BTreeStatsTest extends AbstractBTreeTest {
 TypeAwareTupleWriterFactory tupleWriterFactory = new 
TypeAwareTupleWriterFactory(typeTraits);
 ITreeIndexFrameFactory leafFrameFactory = new 
BTreeNSMLeafFrameFactory(tupleWriterFactory);
 ITreeIndexFrameFactory

[3/5] asterixdb git commit: Change the API for writing and reading metadata pages

2017-01-06 Thread amoudi 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/90cdbac7/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
 
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
index 47530a8..0948704 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
@@ -34,15 +34,12 @@ import 
org.apache.hyracks.storage.common.buffercache.ICachedPage;
 
 public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
 
-protected static final int pageLsnOff = 0; // 0
-protected static final int tupleCountOff = pageLsnOff + 8; // 8
-protected static final int freeSpaceOff = tupleCountOff + 4; // 12
-protected static final int totalFreeSpaceOff = freeSpaceOff + 4; // 16
-protected static final int levelOff = totalFreeSpaceOff + 4; // 20
-protected static final int flagOff = levelOff + 1; // 21
-
-protected static final byte smFlagBit   = 0x1;
-protected static final byte largeFlagBit= 0x2;
+protected static final int PAGE_LSN_OFFSET = 
Constants.RESERVED_HEADER_SIZE;
+protected static final int TOTAL_FREE_SPACE_OFFSET = PAGE_LSN_OFFSET + 8;
+protected static final int FLAG_OFFSET = TOTAL_FREE_SPACE_OFFSET + 4;
+protected static final int RESERVED_HEADER_SIZE = FLAG_OFFSET + 1;
+protected static final byte SMALL_FLAG_BIT = 0x1;
+protected static final byte LARGE_FLAG_BIT = 0x2;
 
 protected ICachedPage page = null;
 protected ByteBuffer buf = null;
@@ -60,12 +57,12 @@ public abstract class TreeIndexNSMFrame implements 
ITreeIndexFrame {
 
 @Override
 public void initBuffer(byte level) {
-buf.putLong(pageLsnOff, 0); // TODO: might to set to a different lsn
+buf.putLong(PAGE_LSN_OFFSET, 0); // TODO: might to set to a different 
lsn
 // during creation
-buf.putInt(tupleCountOff, 0);
+buf.putInt(Constants.TUPLE_COUNT_OFFSET, 0);
 resetSpaceParams();
-buf.put(levelOff, level);
-buf.put(flagOff, (byte) 0);
+buf.put(Constants.LEVEL_OFFSET, level);
+buf.put(FLAG_OFFSET, (byte) 0);
 }
 
 @Override
@@ -75,56 +72,56 @@ public abstract class TreeIndexNSMFrame implements 
ITreeIndexFrame {
 
 @Override
 public boolean isLeaf() {
-return buf.get(levelOff) == 0;
+return buf.get(Constants.LEVEL_OFFSET) == 0;
 }
 
 public boolean getSmFlag() {
-return (buf.get(flagOff) & smFlagBit) != 0;
+return (buf.get(FLAG_OFFSET) & SMALL_FLAG_BIT) != 0;
 }
 
 public void setSmFlag(boolean smFlag) {
 if (smFlag) {
-buf.put(flagOff, (byte) (buf.get(flagOff) | smFlagBit));
+buf.put(FLAG_OFFSET, (byte) (buf.get(FLAG_OFFSET) | 
SMALL_FLAG_BIT));
 } else {
-buf.put(flagOff, (byte) (buf.get(flagOff) & ~smFlagBit));
+buf.put(FLAG_OFFSET, (byte) (buf.get(FLAG_OFFSET) & 
~SMALL_FLAG_BIT));
 }
 }
 
 public void setLargeFlag(boolean largeFlag) {
 if (largeFlag) {
-buf.put(flagOff, (byte) (buf.get(flagOff) | largeFlagBit));
+buf.put(FLAG_OFFSET, (byte) (buf.get(FLAG_OFFSET) | 
LARGE_FLAG_BIT));
 } else {
-buf.put(flagOff, (byte) (buf.get(flagOff) & ~largeFlagBit));
+buf.put(FLAG_OFFSET, (byte) (buf.get(FLAG_OFFSET) & 
~LARGE_FLAG_BIT));
 }
 }
 
 public boolean getLargeFlag() {
-return (buf.get(flagOff) & largeFlagBit) != 0;
+return (buf.get(FLAG_OFFSET) & LARGE_FLAG_BIT) != 0;
 }
 
 @Override
 public boolean isInterior() {
-return buf.get(levelOff) > 0;
+return buf.get(Constants.LEVEL_OFFSET) > 0;
 }
 
 @Override
 public byte getLevel() {
-return buf.get(levelOff);
+return buf.get(Constants.LEVEL_OFFSET);
 }
 
 @Override
 public void setLevel(byte level) {
-buf.put(levelOff, level);
+buf.put(Constants.LEVEL_OFFSET, level);
 }
 
 @Override
 public int getFreeSpaceOff() {
-return buf.getInt(freeSpaceOff);
+return buf.getInt(Constants.FREE_SPACE_OFFSET);
 }
 
 @Override
 public void setFreeSpaceOff(int freeSpace) {
-buf.putInt(freeSpaceOff, freeSpace);
+buf.putInt(Constants.FREE_SPACE_OFFSET, freeSpace);
 }
 
 @Override
@@ -146,8 +143,8 @@ public abstract class TreeIndexNSMFrame implements 
ITreeIndexFrame {

[2/4] asterixdb git commit: Full-text implementation step 3

2017-01-06 Thread wangsaeu 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-01/fulltext-01.8.query.sqlpp
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-01/fulltext-01.8.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-01/fulltext-01.8.query.sqlpp
new file mode 100644
index 000..895b73e
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-01/fulltext-01.8.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select element {"id":ftval.id}
+from MyData as ftval
+where test.ftcontains(ftval.title, {{"database"}})
+order by ftval.id;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-02/fulltext-02.1.ddl.sqlpp
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-02/fulltext-02.1.ddl.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-02/fulltext-02.1.ddl.sqlpp
new file mode 100644
index 000..6bff4a1
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-02/fulltext-02.1.ddl.sqlpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ *  Description : Full-text search non-index test
+ *  : This test is intended to verify that the full-text 
search works as expected.
+ *  : query #3 - two string values in [an ordered list] query 
with "any" option
+ *  :in this case, "any" option that enforces a 
disjunctive search will be applied.
+ *  : query #4 - the same as query #3, but with a different 
option - "all"
+ *  :in this case, we explicitly specify "all" 
option that enforces a conjunctive search.
+ *  : query #5 - two string values in {{an unordered list}} 
query with "any" option
+ *  :in this case, "any" option that enforces a 
disjunctive search will be applied.
+ *  : query #6 - the same as query #6, but with a different 
option - "all"
+ *  :in this case, we explicitly specify "all" 
option that enforces a conjunctive search.
+ *  : query #7 - the same as query #4, but without any option 
that is equivalent to "all".
+ *  : query #8 - the same as query #6, but without any option 
that is equivalent to "all".
+ *  Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+  id: int64,
+  docid: int64,
+  val1: int64,
+  title: string,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+create dataset MyData(MyRecord)
+  primary key id;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-02/fulltext-02.2.update.sqlpp

[1/4] asterixdb git commit: Full-text implementation step 3

2017-01-06 Thread wangsaeu 
Repository: asterixdb
Updated Branches:
  refs/heads/master d49bc6eb1 -> c49405aaf


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-doc/src/site/markdown/aql/fulltext.md
--
diff --git a/asterixdb/asterix-doc/src/site/markdown/aql/fulltext.md 
b/asterixdb/asterix-doc/src/site/markdown/aql/fulltext.md
index 921f0b3..4fe17ac 100644
--- a/asterixdb/asterix-doc/src/site/markdown/aql/fulltext.md
+++ b/asterixdb/asterix-doc/src/site/markdown/aql/fulltext.md
@@ -39,9 +39,10 @@ returned as well.
 ## Syntax [Back to 
TOC] ##
 
 The syntax of AsterixDB FTS follows a portion of the XQuery FullText Search 
syntax.
-A basic form is as follows:
+Two basic forms are as follows:
 
 ftcontains(Expression1, Expression2, {FullTextOption})
+ftcontains(Expression1, Expression2)
 
 For example, we can execute the following query to find tweet messages where 
the `message-text` field includes
 “voice” as a word. Please note that an FTS search is case-insensitive.
@@ -62,6 +63,7 @@ into one of the first two types, i.e., into a string value or 
an (un)ordered lis
 
 The following examples are all valid expressions.
 
+   ... where ftcontains($msg.message-text, "sound")
... where ftcontains($msg.message-text, "sound", {"mode":"any"})
... where ftcontains($msg.message-text, ["sound", "system"], 
{"mode":"any"})
... where ftcontains($msg.message-text, {{"speed", "stand", 
"customization"}}, {"mode":"all"})
@@ -70,30 +72,34 @@ The following examples are all valid expressions.
 
 In the last example above, `$keyword_list` should evaluate to a string or an 
(un)ordered list of string value(s).
 
-The last `FullTextOption` parameter clarifies the given FTS request. 
Currently, we only have one option named `mode`.
+The last `FullTextOption` parameter clarifies the given FTS request. If you 
omit the `FullTextOption` parameter,
+then the default value will be set for each possible option. Currently, we 
only have one option named `mode`.
 And as we extend the FTS feature, more options will be added. Please note that 
the format of `FullTextOption`
 is a record, thus you need to put the option(s) in a record `{}`.
 The `mode` option indicates whether the given FTS query is a conjunctive (AND) 
or disjunctive (OR) search request.
-This option can be either `“any”` or `“all”`. If one specifies 
`“any”`, a disjunctive search will be conducted.
-For example, the following query will find documents whose `message-text` 
field contains “sound” or “system”,
-so a document will be returned if it contains either “sound”, 
“system”, or both of the keywords.
+This option can be either `“any”` or `“all”`. The default value for 
`mode` is `“all”`. If one specifies `“any”`,
+a disjunctive search will be conducted. For example, the following query will 
find documents whose `message-text`
+field contains “sound” or “system”, so a document will be returned if 
it contains either “sound”, “system”,
+or both of the keywords.
 
... where ftcontains($msg.message-text, ["sound", "system"], 
{"mode":"any"})
 
-The other option parameter,`“all”`, specifies a conjunctive search. The 
following example will find the documents whose
+The other option parameter,`“all”`, specifies a conjunctive search. The 
following examples will find the documents whose
 `message-text` field contains both “sound” and “system”. If a document 
contains only “sound” or “system” but
 not both, it will not be returned.
 
... where ftcontains($msg.message-text, ["sound", "system"], 
{"mode":"all"})
+   ... where ftcontains($msg.message-text, ["sound", "system"])
 
 Currently AsterixDB doesn’t (yet) support phrase searches, so the following 
query will not work.
 
... where ftcontains($msg.message-text, "sound system", {"mode":"any"})
 
 As a workaround solution, the following query can be used to achieve a roughly 
similar goal. The difference is that
-the following query will find documents where `$msg.message-text` contains 
both “sound” and “system”, but the order
+the following queries will find documents where `$msg.message-text` contains 
both “sound” and “system”, but the order
 and adjacency of “sound” and “system” are not checked, unlike in a 
phrase search. As a result, the query below would
 also return documents with “sound system can be installed.”, “system 
sound is perfect.”,
 or “sound is not clear. You may need to install a new system.”
 
... where ftcontains($msg.message-text, ["sound", "system"], 
{"mode":"all"})
+   ... where ftcontains($msg.message-text, ["sound", "system"])

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-doc/src/site/markdown/aql/manual.md
--

[3/4] asterixdb git commit: Full-text implementation step 3

2017-01-06 Thread wangsaeu 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.1.ddl.aql
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.1.ddl.aql
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.1.ddl.aql
new file mode 100644
index 000..bfef89a
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.1.ddl.aql
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ *  Description : Full-text search index test
+ *  : This test is intended to verify that the full-text 
search works as expected.
+ *  : query #3 - single string value query
+ *  : query #4 - single string value in an ordered list query
+ *  : query #5 - single string value in an unordered list query
+ *  : query #6 - the same as #3, but without any option
+ *  : query #7 - the same as #4, but without any option
+ *  : query #8 - the same as #5, but without any option
+ *  Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+  id: int64,
+  docid: int64,
+  val1: int64,
+  title: string,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+create dataset MyData(MyRecord)
+  primary key id;
+
+create index fulltext_index_title on MyData(title) type fulltext;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.2.update.aql
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.2.update.aql
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.2.update.aql
new file mode 100644
index 000..c627cf1
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.2.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+load dataset MyData
+using localfs
+(("path"="asterix_nc1://data/spatial/spatialData2.json"),("format"="adm"));

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c49405aa/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.3.query.aql
--
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.3.query.aql
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.3.query.aql
new file mode 100644
index 000..bc47bb6
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/fulltext/fulltext-index-01/fulltext-index-01.3.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache

[6/6] asterixdb git commit: ASTERIXDB-1714: Eliminate dependency on org.json

2017-01-06 Thread imaxon 
ASTERIXDB-1714: Eliminate dependency on org.json

Change-Id: Ie9c5400fd134ae75d43385255af7794e968b1c7e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1392
Tested-by: Jenkins 
BAD: Jenkins 
Reviewed-by: Till Westmann 
Integration-Tests: Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d49bc6eb
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d49bc6eb
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d49bc6eb

Branch: refs/heads/master
Commit: d49bc6eb1bf8dc3ef7966466521d28a0054ecc63
Parents: 0aeebae
Author: Ian Maxon 
Authored: Thu Jan 5 18:50:57 2017 -0800
Committer: Ian Maxon 
Committed: Fri Jan 6 03:42:33 2017 -0800

--
 asterixdb/asterix-algebra/pom.xml   |   4 -
 asterixdb/asterix-app/pom.xml   |  18 +-
 .../apache/asterix/api/common/APIFramework.java |  25 +-
 .../api/http/servlet/ClusterAPIServlet.java |  52 +--
 .../servlet/ClusterCCDetailsAPIServlet.java |  24 +-
 .../servlet/ClusterNodeDetailsAPIServlet.java   |  91 +++--
 .../api/http/servlet/ConnectorAPIServlet.java   |  30 +-
 .../api/http/servlet/DiagnosticsAPIServlet.java |  59 +--
 .../api/http/servlet/QueryResultAPIServlet.java |  15 +-
 .../api/http/servlet/QueryServiceServlet.java   |  28 +-
 .../api/http/servlet/QueryStatusAPIServlet.java |  16 +-
 .../http/servlet/QueryWebInterfaceServlet.java  |   6 +-
 .../api/http/servlet/RESTAPIServlet.java|   7 +-
 .../api/http/servlet/ShutdownAPIServlet.java|  34 +-
 .../api/http/servlet/VersionAPIServlet.java |   9 +-
 .../asterix/app/result/ResultPrinter.java   |  22 +-
 .../apache/asterix/app/result/ResultUtil.java   |  39 +-
 .../asterix/app/translator/QueryTranslator.java |  18 +-
 .../http/servlet/ConnectorAPIServletTest.java   |  41 +-
 .../api/http/servlet/VersionAPIServletTest.java |  15 +-
 .../types/any-object/any-object.2.query.sqlpp   |   4 +-
 .../api/cluster_state_1/cluster_state_1.1.adm   | 283 +++--
 .../cluster_state_1_full.1.adm  | 283 +++--
 .../cluster_state_1_less.1.adm  | 283 +++--
 .../api/cluster_state_3/cluster_state_3.1.adm   |  65 ++-
 .../api/cluster_state_4/cluster_state_4.1.adm   |  29 +-
 .../cluster_state_cc_1/cluster_state_cc_1.1.adm |   8 +-
 .../cluster_state_cc_stats_1.1.regexadm |  49 ++-
 .../cluster_state_cc_threaddump_1.1.regex   |  18 +-
 .../api/diagnostics_1/diagnostics_1.1.regexadm  |  17 +-
 .../results/api/replication/replication.1.adm   |  20 +-
 asterixdb/asterix-client-helper/pom.xml |   8 +-
 .../commands/GetClusterStateCommand.java|  13 +-
 .../commands/WaitForClusterCommand.java |  12 +-
 asterixdb/asterix-common/pom.xml|  17 +-
 .../common/config/AbstractProperties.java   |  11 +-
 .../common/config/ExternalProperties.java   |  41 +-
 .../apache/asterix/common/utils/JSONUtil.java   |  70 +++-
 .../asterix/test/aql/ResultExtractor.java   | 218 --
 .../test/aql/SingleLinePrettyPrinter.java   | 404 +++
 .../apache/asterix/test/aql/TestExecutor.java   |  66 +--
 .../external/feed/message/EndFeedMessage.java   |  10 +-
 .../asterix/external/parser/TweetParser.java| 160 +++-
 .../node_failback.cluster_state.10.adm  |  38 +-
 .../node_failback.cluster_state.5.adm   |  38 +-
 .../pom.xml |   9 +-
 .../RecordManagerGeneratorMojo.java |   3 -
 .../recordmanagergenerator/RecordType.java  | 177 
 .../org/apache/asterix/om/base/ABinary.java |  17 +-
 .../org/apache/asterix/om/base/ABitArray.java   |  17 +-
 .../org/apache/asterix/om/base/ABoolean.java|   8 +-
 .../org/apache/asterix/om/base/ACircle.java |  15 +-
 .../java/org/apache/asterix/om/base/ADate.java  |   9 +-
 .../org/apache/asterix/om/base/ADateTime.java   |   9 +-
 .../asterix/om/base/ADayTimeDuration.java   |  14 +-
 .../org/apache/asterix/om/base/ADouble.java |  10 +-
 .../org/apache/asterix/om/base/ADuration.java   |  14 +-
 .../java/org/apache/asterix/om/base/AFloat.java |  10 +-
 .../java/org/apache/asterix/om/base/AInt16.java |  11 +-
 .../java/org/apache/asterix/om/base/AInt32.java |  10 +-
 .../java/org/apache/asterix/om/base/AInt64.java |  10 +-
 .../java/org/apache/asterix/om/base/AInt8.java  |  10 +-
 .../org/apache/asterix/om/base/AInterval.java   |   5 +-
 .../java/org/apache/asterix/om/base/ALine.java  |  17 +-
 .../org/apache/asterix/om/base/AMissing.java|   9 +-
 .../java/org/apache/asterix/om/base/ANull.java  |   9 +-
 .../apache/asterix/om/base/AOrderedList.java|  17 +-
 .../java/org/apache/asterix/om/base/APoint.java |  14 +-

[1/6] asterixdb git commit: ASTERIXDB-1714: Eliminate dependency on org.json

2017-01-06 Thread imaxon 
Repository: asterixdb
Updated Branches:
  refs/heads/master 0aeebae06 -> d49bc6eb1


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d49bc6eb/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
index 890ab0a..4163e46 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -45,7 +46,7 @@ import org.apache.hyracks.control.nc.NodeControllerService;
 import org.apache.hyracks.control.nc.resources.memory.FrameManager;
 import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
 import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.json.JSONArray;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -123,7 +124,7 @@ public abstract class AbstractMultiNCIntegrationTest {
 
 protected void runTest(JobSpecification spec) throws Exception {
 if (LOGGER.isLoggable(Level.INFO)) {
-LOGGER.info(spec.toJSON().toString(2));
+LOGGER.info(spec.toJSON().asText());
 }
 JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
 if (LOGGER.isLoggable(Level.INFO)) {
@@ -141,7 +142,8 @@ public abstract class AbstractMultiNCIntegrationTest {
 IHyracksDataset hyracksDataset = new HyracksDataset(hcc, 
spec.getFrameSize(), nReaders);
 IHyracksDatasetReader reader = hyracksDataset.createReader(jobId, 
spec.getResultSetIds().get(0));
 
-JSONArray resultRecords = new JSONArray();
+ObjectMapper om = new ObjectMapper();
+ArrayNode resultRecords = om.createArrayNode();
 ByteBufferInputStream bbis = new ByteBufferInputStream();
 
 int readSize = reader.read(resultFrame);
@@ -156,7 +158,7 @@ public abstract class AbstractMultiNCIntegrationTest {
 bbis.setByteBuffer(resultFrame.getBuffer(), start);
 byte[] recordBytes = new byte[length];
 bbis.read(recordBytes, 0, length);
-resultRecords.put(new String(recordBytes, 0, length));
+resultRecords.add(new String(recordBytes, 0, length));
 }
 } finally {
 try {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d49bc6eb/hyracks-fullstack/hyracks/hyracks-server/pom.xml
--
diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml 
b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
index 70e47fe..3667951 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
@@ -164,11 +164,6 @@
   test
 
 
-  org.json
-  json
-  20090211
-
-
   org.apache.httpcomponents
   httpcore
   4.4.5
@@ -183,5 +178,9 @@
   hyracks-control-nc
   ${project.version}
 
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
   
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d49bc6eb/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
 
b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
index 8d1246b..2185826 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.util.logging.Logger;
 
+import

[4/6] asterixdb git commit: ASTERIXDB-1714: Eliminate dependency on org.json

2017-01-06 Thread imaxon 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d49bc6eb/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/SingleLinePrettyPrinter.java
--
diff --git 
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/SingleLinePrettyPrinter.java
 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/SingleLinePrettyPrinter.java
new file mode 100644
index 000..52f68a0
--- /dev/null
+++ 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/SingleLinePrettyPrinter.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.aql;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.io.SerializedString;
+import com.fasterxml.jackson.core.util.DefaultIndenter;
+import com.fasterxml.jackson.core.util.Instantiatable;
+
+/**
+ * Default {@link PrettyPrinter} implementation that uses 2-space
+ * indentation with platform-default linefeeds.
+ * Usually this class is not instantiated directly, but instead
+ * method {@link JsonGenerator#useSingleLinePrettyPrinter} is
+ * used, which will use an instance of this class for operation.
+ */
+@SuppressWarnings("serial")
+public class SingleLinePrettyPrinter
+implements PrettyPrinter, Instantiatable, 
java.io.Serializable {
+private static final long serialVersionUID = 1;
+
+/**
+ * Constant that specifies default "root-level" separator to use between
+ * root values: a single space character.
+ *
+ * @since 2.1
+ */
+public final static SerializedString DEFAULT_ROOT_VALUE_SEPARATOR = new 
SerializedString(" ");
+
+/**
+ * Interface that defines objects that can produce indentation used
+ * to separate object entries and array values. Indentation in this
+ * context just means insertion of white space, independent of whether
+ * linefeeds are output.
+ */
+public interface Indenter {
+void writeIndentation(JsonGenerator jg, int level) throws IOException;
+
+/**
+ * @return True if indenter is considered inline (does not add 
linefeeds),
+ * false otherwise
+ */
+boolean isInline();
+}
+
+// // // Config, indentation
+
+/**
+ * By default, let's use only spaces to separate array values.
+ */
+protected Indenter _arrayIndenter = FixedSpaceIndenter.instance;
+
+/**
+ * By default, let's use linefeed-adding indenter for separate
+ * object entries. We'll further configure indenter to use
+ * system-specific linefeeds, and 2 spaces per level (as opposed to,
+ * say, single tabs)
+ */
+protected Indenter _objectIndenter = new FixedSpaceIndenter();
+
+/**
+ * String printed between root-level values, if any.
+ */
+protected final SerializableString _rootSeparator;
+
+// // // Config, other white space configuration
+
+/**
+ * By default we will add spaces around colons used to
+ * separate object fields and values.
+ * If disabled, will not use spaces around colon.
+ */
+protected boolean _spacesInObjectEntries = true;
+
+// // // State:
+
+/**
+ * Number of open levels of nesting. Used to determine amount of
+ * indentation to use.
+ */
+protected transient int _nesting;
+
+/*
+/**
+/* Life-cycle (construct, configure)
+/**
+*/
+
+public SingleLinePrettyPrinter() {
+this(DEFAULT_ROOT_VALUE_SEPARATOR);
+}
+
+/**
+ * Constructor that specifies separator String to use between root values;
+ * if null, no separator is printed.
+ * 
+ * Note: simply constructs a {@link SerializedString} out of parameter,
+ * calls {@link #SingleLinePrettyPrinter(SerializableString)}
+ *
+ * @param rootSeparator
+ * @since 2.1
+ */
+public SingleLinePrettyPrinter(String rootSeparator) {
+this((rootSeparator == null) ? null : new 
SerializedString(rootSeparator));
+}
+
+/**
+ *

[2/6] asterixdb git commit: ASTERIXDB-1714: Eliminate dependency on org.json

2017-01-06 Thread imaxon 
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d49bc6eb/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/adminconsole/pages/IndexPage.java
--
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/adminconsole/pages/IndexPage.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/adminconsole/pages/IndexPage.java
index 32ac3f5..680c2a7 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/adminconsole/pages/IndexPage.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/adminconsole/pages/IndexPage.java
@@ -18,22 +18,23 @@
  */
 package org.apache.hyracks.control.cc.adminconsole.pages;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
 import org.apache.wicket.markup.html.basic.Label;
 import org.apache.wicket.markup.html.link.BookmarkablePageLink;
 import org.apache.wicket.markup.html.list.ListItem;
 import org.apache.wicket.markup.html.list.ListView;
 import org.apache.wicket.request.mapper.parameter.PageParameters;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
 
 import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.cc.web.util.JSONUtils;
 import org.apache.hyracks.control.cc.work.GetJobSummariesJSONWork;
 import org.apache.hyracks.control.cc.work.GetNodeSummariesJSONWork;
 
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.List;
 
 public class IndexPage extends AbstractPage {
 private static final long serialVersionUID = 1L;
@@ -43,49 +44,42 @@ public class IndexPage extends AbstractPage {
 
 GetNodeSummariesJSONWork gnse = new GetNodeSummariesJSONWork(ccs);
 ccs.getWorkQueue().scheduleAndSync(gnse);
-JSONArray nodeSummaries = gnse.getSummaries();
-add(new Label("node-count", String.valueOf(nodeSummaries.length(;
-ListView nodeList = new ListView("node-list", 
JSONUtils.toList(nodeSummaries)) {
+ArrayNode nodeSummaries = gnse.getSummaries();
+add(new Label("node-count", String.valueOf(nodeSummaries.size(;
+ListView nodeList = new ListView("node-list",
+Lists.newArrayList(nodeSummaries.iterator())) {
 private static final long serialVersionUID = 1L;
 
 @Override
-protected void populateItem(ListItem item) {
-JSONObject o = item.getModelObject();
-try {
-item.add(new Label("node-id", o.getString("node-id")));
-item.add(new Label("heap-used", o.getString("heap-used")));
-item.add(new Label("system-load-average", 
o.getString("system-load-average")));
-PageParameters params = new PageParameters();
-params.add("node-id", o.getString("node-id"));
-item.add(new BookmarkablePageLink("node-details", 
NodeDetailsPage.class, params));
-} catch (JSONException e) {
-throw new RuntimeException(e);
-}
+protected void populateItem(ListItem item) {
+JsonNode o = item.getModelObject();
+item.add(new Label("node-id", o.get("node-id").asText()));
+item.add(new Label("heap-used", o.get("heap-used").asText()));
+item.add(new Label("system-load-average", 
o.get("system-load-average").asText()));
+PageParameters params = new PageParameters();
+params.add("node-id", o.get("node-id").asText());
+item.add(new BookmarkablePageLink("node-details", 
NodeDetailsPage.class, params));
 }
 };
 add(nodeList);
 
 GetJobSummariesJSONWork gjse = new GetJobSummariesJSONWork(ccs);
 ccs.getWorkQueue().scheduleAndSync(gjse);
-JSONArray jobSummaries = gjse.getSummaries();
-ListView jobList = new ListView("jobs-list", 
JSONUtils.toList(jobSummaries)) {
+ArrayNode jobSummaries = gjse.getSummaries();
+ListView jobList = new ListView("jobs-list", 
Lists.newArrayList(jobSummaries.iterator())) {
 private static final long serialVersionUID = 1L;
 
 @Override
-protected void populateItem(ListItem item) {
-JSONObject o = item.getModelObject();
-try {
-item.add(new Label("job-id", o.getString("job-id")));
-item.add(new Label("status", o.getString("status")));
-