[22/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
new file mode 100644
index 000..00116c5
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * It reads the data vector batch format
+ */
+public class VectorDetailQueryResultIterator extends 
AbstractDetailQueryResultIterator {
+
+  private final Object lock = new Object();
+
+  public VectorDetailQueryResultIterator(List infos, 
QueryModel queryModel,
+  ExecutorService execService) {
+super(infos, queryModel, execService);
+  }
+
+  @Override public Object next() {
+throw new UnsupportedOperationException("call processNextBatch instaed");
+  }
+
+  public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+synchronized (lock) {
+  updateDataBlockIterator();
+  if (dataBlockIterator != null) {
+dataBlockIterator.processNextBatch(columnarBatch);
+  }
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
new file mode 100644
index 000..7d29b0f
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result.vector;
+
+import org.apache.spark.sql.types.Decimal;
+
+public interface CarbonColumnVector {
+
+  void putShort(int rowId, short value);
+
+  void putInt(int rowId, int value);
+
+  void putLong(int rowId, long value);
+
+  void putDecimal(int rowId, Decimal value, int precision);
+
+  void putDouble(int rowId, double value);
+
+  void putBytes(int rowId, byte[] value);
+
+  void putBytes(int rowId, int offset, int length, byte[] value);
+
+  void putNull(int rowId);
+
+  boolean isNull(int rowId);
+
+  void putObject(int rowId, Object obj);
+
+  Object getData(int rowId);
+
+  void reset();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
 

[41/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
deleted file mode 100644
index 9ff0f59..000
--- 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-
-
-public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
-  /**
-   * LOGGER
-   */
-  private static final LogService LOGGER =
-  LogServiceFactory.getLogService(AlluxioCarbonFile.class.getName());
-
-  public AlluxioCarbonFile(String filePath) {
-super(filePath);
-  }
-
-  public AlluxioCarbonFile(Path path) {
-super(path);
-  }
-
-  public AlluxioCarbonFile(FileStatus fileStatus) {
-super(fileStatus);
-  }
-
-  /**
-   * @param listStatus
-   * @return
-   */
-  private CarbonFile[] getFiles(FileStatus[] listStatus) {
-if (listStatus == null) {
-  return new CarbonFile[0];
-}
-CarbonFile[] files = new CarbonFile[listStatus.length];
-for (int i = 0; i < files.length; i++) {
-  files[i] = new AlluxioCarbonFile(listStatus[i]);
-}
-return files;
-  }
-
-  @Override
-  public CarbonFile[] listFiles() {
-FileStatus[] listStatus = null;
-try {
-  if (null != fileStatus && fileStatus.isDirectory()) {
-Path path = fileStatus.getPath();
-listStatus = 
path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
-  } else {
-return null;
-  }
-} catch (IOException e) {
-  LOGGER.error("Exception occured: " + e.getMessage());
-  return new CarbonFile[0];
-}
-return getFiles(listStatus);
-  }
-
-  @Override
-  public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
-CarbonFile[] files = listFiles();
-if (files != null && files.length >= 1) {
-  List fileList = new ArrayList(files.length);
-  for (int i = 0; i < files.length; i++) {
-if (fileFilter.accept(files[i])) {
-  fileList.add(files[i]);
-}
-  }
-  if (fileList.size() >= 1) {
-return fileList.toArray(new CarbonFile[fileList.size()]);
-  } else {
-return new CarbonFile[0];
-  }
-}
-return files;
-  }
-
-  @Override
-  public CarbonFile getParentFile() {
-Path parent = fileStatus.getPath().getParent();
-return null == parent ? null : new AlluxioCarbonFile(parent);
-  }
-
-  @Override
-  public boolean renameForce(String changetoName) {
-FileSystem fs;
-try {
-  fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
-  if (fs instanceof DistributedFileSystem) {
-((DistributedFileSystem) fs).rename(fileStatus.getPath(), new 
Path(changetoName),
-org.apache.hadoop.fs.Options.Rename.OVERWRITE);
-return true;
-  } else {
-return false;
-  }
-} catch (IOException e) {
-  LOGGER.error("Exception occured: " + e.getMessage());
-  return false;
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java
 

[13/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
deleted file mode 100644
index 0de0a4d..000
--- 
a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import 
org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class IncludeFilterExecuterImpl implements FilterExecuter {
-
-  protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
-  protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
-  protected SegmentProperties segmentProperties;
-
-  public IncludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColumnEvaluatorInfo,
-  SegmentProperties segmentProperties) {
-this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
-this.segmentProperties = segmentProperties;
-dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
-
FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
-segmentProperties, dimColumnEvaluatorInfo.getDimension(), 
dimColumnExecuterInfo);
-
-  }
-
-  @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) 
throws IOException {
-int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-.get(dimColumnEvaluatorInfo.getColumnIndex());
-if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
-  blockChunkHolder.getDimensionDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
-  .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
-}
-return 
getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
-blockChunkHolder.getDataBlock().nodeSize());
-  }
-
-  protected BitSet getFilteredIndexes(DimensionColumnDataChunk 
dimensionColumnDataChunk,
-  int numerOfRows) {
-if (dimensionColumnDataChunk.isNoDicitionaryColumn()
-&& dimensionColumnDataChunk instanceof 
VariableLengthDimensionDataChunk) {
-  return setDirectKeyFilterIndexToBitSet(
-  (VariableLengthDimensionDataChunk) dimensionColumnDataChunk, 
numerOfRows);
-} else if (dimensionColumnDataChunk.isExplicitSorted()
-&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
-  return setFilterdIndexToBitSetWithColumnIndex(
-  (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, 
numerOfRows);
-}
-
-return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
-  }
-
-  private BitSet setDirectKeyFilterIndexToBitSet(
-  VariableLengthDimensionDataChunk dimensionColumnDataChunk, int 
numerOfRows) {
-BitSet bitSet = new BitSet(numerOfRows);
-byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
-for (int i = 0; i < filterValues.length; i++) {
-  byte[] filterVal = filterValues[i];
-  if (dimensionColumnDataChunk.isExplicitSorted()) {
-for (int index = 0; index < numerOfRows; index++) {
-  if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
-bitSet.set(dimensionColumnDataChunk.getInvertedIndex(index));
-  }

[40/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/BtreeBuilder.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/BtreeBuilder.java 
b/core/src/main/java/org/apache/carbondata/core/datastore/BtreeBuilder.java
new file mode 100644
index 000..3e2b013
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BtreeBuilder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore;
+
+/**
+ * Below interface will be used to build the index
+ * in some data structure
+ */
+public interface BtreeBuilder {
+
+  /**
+   * Below method will be used to store the leaf collection in some data 
structure
+   */
+  void build(BTreeBuilderInfo blocksBuilderInfos);
+
+  /**
+   * below method to get the first data block
+   *
+   * @return data block
+   */
+  DataRefNode get();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java 
b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
new file mode 100644
index 000..9b27c3d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+
+import 
org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+
+/**
+ * Interface data block reference
+ */
+public interface DataRefNode {
+
+  /**
+   * Method to get the next block this can be used while scanning when
+   * iterator of this class can be used iterate over blocks
+   *
+   * @return next block
+   */
+  DataRefNode getNextDataRefNode();
+
+  /**
+   * to get the number of keys tuples present in the block
+   *
+   * @return number of keys in the block
+   */
+  int nodeSize();
+
+  /**
+   * Method can be used to get the block index .This can be used when multiple
+   * thread can be used scan group of blocks in that can we can assign the
+   * some of the blocks to one thread and some to other
+   *
+   * @return block number
+   */
+  long nodeNumber();
+
+  /**
+   * This method will be used to get the max value of all the columns this can
+   * be used in case of filter query
+   *
+   */
+  byte[][] getColumnsMaxValue();
+
+  /**
+   * This method will be used to get the min value of all the columns this can
+   * be used in case of filter query
+   *
+   */
+  byte[][] getColumnsMinValue();
+
+  /**
+   * Below method will be used to get the dimension chunks
+   *
+   * @param fileReader   file reader to read the chunks from file
+   * @param blockIndexes range indexes of the blocks need to be read
+   * value can be {{0,10},{11,12},{13,13}}
+   * here 0 to 10 and 11 to 12 column blocks will be read 
in one
+   * IO operation 13th column block will be read 

[17/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
 
b/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
deleted file mode 100644
index ac717a9..000
--- 
a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.complextypes;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import 
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import 
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.keygenerator.mdkey.Bits;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-import org.apache.spark.sql.types.*;
-
-public class PrimitiveQueryType extends ComplexQueryType implements 
GenericQueryType {
-
-  private String name;
-  private String parentname;
-
-  private int keySize;
-
-  private int blockIndex;
-
-  private Dictionary dictionary;
-
-  private org.apache.carbondata.core.carbon.metadata.datatype.DataType 
dataType;
-
-  private boolean isDirectDictionary;
-
-  public PrimitiveQueryType(String name, String parentname, int blockIndex,
-  org.apache.carbondata.core.carbon.metadata.datatype.DataType dataType, 
int keySize,
-  Dictionary dictionary, boolean isDirectDictionary) {
-super(name, parentname, blockIndex);
-this.dataType = dataType;
-this.keySize = keySize;
-this.dictionary = dictionary;
-this.name = name;
-this.parentname = parentname;
-this.blockIndex = blockIndex;
-this.isDirectDictionary = isDirectDictionary;
-  }
-
-  @Override public void addChildren(GenericQueryType children) {
-
-  }
-
-  @Override public String getName() {
-return name;
-  }
-
-  @Override public void setName(String name) {
-this.name = name;
-  }
-
-  @Override public String getParentname() {
-return parentname;
-  }
-
-  @Override public void setParentname(String parentname) {
-this.parentname = parentname;
-
-  }
-
-  @Override public int getColsCount() {
-return 1;
-  }
-
-  @Override public void parseBlocksAndReturnComplexColumnByteArray(
-  DimensionColumnDataChunk[] dimensionDataChunks, int rowNumber,
-  DataOutputStream dataOutputStream) throws IOException {
-byte[] currentVal =
-new byte[dimensionDataChunks[blockIndex].getColumnValueSize()];
-copyBlockDataChunk(dimensionDataChunks, rowNumber, currentVal);
-dataOutputStream.write(currentVal);
-  }
-
-  @Override public DataType getSchemaType() {
-switch (dataType) {
-  case INT:
-return IntegerType$.MODULE$;
-  case DOUBLE:
-return DoubleType$.MODULE$;
-  case LONG:
-return LongType$.MODULE$;
-  case BOOLEAN:
-return BooleanType$.MODULE$;
-  case TIMESTAMP:
-return TimestampType$.MODULE$;
-  case DATE:
-return DateType$.MODULE$;
-  default:
-return IntegerType$.MODULE$;
-}
-  }
-
-  @Override public void fillRequiredBlockData(BlocksChunkHolder 
blockChunkHolder)
-  throws IOException {
-readBlockDataChunk(blockChunkHolder);
-  }
-
-  @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer 
surrogateData) {
-byte[] data = new byte[keySize];
-surrogateData.get(data);
-Bits bit = new Bits(new int[]{keySize * 8});
-int surrgateValue = (int)bit.getKeyArray(data, 0)[0];
-Object actualData = null;
-if (isDirectDictionary) {
-  DirectDictionaryGenerator directDictionaryGenerator = 

[12/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
 
b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
deleted file mode 100644
index d22bf38..000
--- 
a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import 
org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import 
org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import 
org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import 
org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import 
org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
-
-public class ConditionalFilterResolverImpl implements FilterResolverIntf {
-
-  private static final long serialVersionUID = 1838955268462201691L;
-  protected Expression exp;
-  protected boolean isExpressionResolve;
-  protected boolean isIncludeFilter;
-  private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
-
-  public ConditionalFilterResolverImpl(Expression exp, boolean 
isExpressionResolve,
-  boolean isIncludeFilter) {
-this.exp = exp;
-this.isExpressionResolve = isExpressionResolve;
-this.isIncludeFilter = isIncludeFilter;
-this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
-  }
-
-  /**
-   * This API will resolve the filter expression and generates the
-   * dictionaries for executing/evaluating the filter expressions in the
-   * executer layer.
-   *
-   * @throws FilterUnsupportedException
-   */
-  @Override public void resolve(AbsoluteTableIdentifier 
absoluteTableIdentifier)
-  throws FilterUnsupportedException, IOException {
-FilterResolverMetadata metadata = new FilterResolverMetadata();
-metadata.setTableIdentifier(absoluteTableIdentifier);
-if ((!isExpressionResolve) && exp instanceof BinaryConditionalExpression) {
-  BinaryConditionalExpression binaryConditionalExpression = 
(BinaryConditionalExpression) exp;
-  Expression leftExp = binaryConditionalExpression.getLeft();
-  Expression rightExp = binaryConditionalExpression.getRight();
-  if (leftExp instanceof ColumnExpression) {
-ColumnExpression columnExpression = (ColumnExpression) leftExp;
-metadata.setColumnExpression(columnExpression);
-metadata.setExpression(rightExp);
-metadata.setIncludeFilter(isIncludeFilter);
-// If imei=imei comes in filter condition then we need to
-// skip processing of right expression.
-// This flow has reached here assuming that this is a single
-// column expression.
-// we need to check if the other expression contains column
-// expression or not in depth.
-CarbonDimension dimension = columnExpression.getDimension();
-if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
-|| 

[35/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
new file mode 100644
index 000..04bab54
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import 
org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneLong extends ValueCompressionHolder {
+  /**
+   * Attribute for Carbon LOGGER
+   */
+  private static final LogService LOGGER =
+  LogServiceFactory.getLogService(CompressionNoneLong.class.getName());
+  /**
+   * longCompressor.
+   */
+  private static Compressor compressor = 
CompressorFactory.getInstance().getCompressor();
+  /**
+   * value.
+   */
+  protected long[] value;
+
+  private DataType actualDataType;
+
+  private MeasureDataChunkStore measureChunkStore;
+
+  public CompressionNoneLong(DataType actualDataType) {
+this.actualDataType = actualDataType;
+  }
+
+  @Override public void setValue(long[] value) { this.value = value;  }
+
+  @Override public long[] getValue() { return this.value; }
+
+  @Override public void compress() {
+compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+  }
+
+  @Override
+  public void uncompress(DataType dataType, byte[] data, int offset, int 
length,
+  int decimalPlaces, Object maxValueObject) {
+super.unCompress(compressor, dataType, data, offset, length);
+setUncompressedValues(value);
+  }
+
+  @Override public void setValueInBytes(byte[] byteValue) {
+ByteBuffer buffer = ByteBuffer.wrap(byteValue);
+this.value = ValueCompressionUtil.convertToLongArray(buffer, 
byteValue.length);
+  }
+
+  @Override public long getLongValue(int index) {
+return measureChunkStore.getLong(index);
+  }
+
+  @Override public double getDoubleValue(int index) {
+return measureChunkStore.getLong(index);
+  }
+
+  @Override public BigDecimal getBigDecimalValue(int index) {
+throw new UnsupportedOperationException("Get big decimal is not 
supported");
+  }
+
+  private void setUncompressedValues(long[] data) {
+this.measureChunkStore =
+  
MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, 
data.length);
+this.measureChunkStore.putData(data);
+  }
+
+  @Override public void freeMemory() {
+this.measureChunkStore.freeMemory();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
new file mode 100644
index 000..b84c562
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one

[39/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
new file mode 100644
index 000..5fa6722
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.block;
+
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
+import org.apache.carbondata.core.datastore.BtreeBuilder;
+import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeBuilder;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+
+/**
+ * Class which is responsible for loading the b+ tree block. This class will
+ * persist all the detail of a table segment
+ */
+public class SegmentTaskIndex extends AbstractIndex {
+
+  /**
+   * Below method is store the blocks in some data structure
+   *
+   */
+  public void buildIndex(List footerList) {
+// create a metadata details
+// this will be useful in query handling
+// all the data file metadata will have common segment properties we
+// can use first one to get create the segment properties
+segmentProperties = new 
SegmentProperties(footerList.get(0).getColumnInTable(),
+footerList.get(0).getSegmentInfo().getColumnCardinality());
+// create a segment builder info
+// in case of segment create we do not need any file path and each column 
value size
+// as Btree will be build as per min max and start key
+BTreeBuilderInfo btreeBuilderInfo = new BTreeBuilderInfo(footerList, null);
+BtreeBuilder blocksBuilder = new BlockBTreeBuilder();
+// load the metadata
+blocksBuilder.build(btreeBuilderInfo);
+dataRefNode = blocksBuilder.get();
+for (DataFileFooter footer : footerList) {
+  totalNumberOfRows += footer.getNumberOfRows();
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
new file mode 100644
index 000..1467a99
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.block;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.mutate.UpdateVO;
+
+/**
+ * SegmentTaskIndexWrapper class holds the  taskIdToTableSegmentMap
+ */
+public class SegmentTaskIndexWrapper implements Cacheable {
+
+  /**
+   * task_id to table segment index map
+   */
+  

[28/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
new file mode 100644
index 000..276e5db
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -0,0 +1,882 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.scan.complextypes.ArrayQueryType;
+import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType;
+import org.apache.carbondata.core.scan.complextypes.StructQueryType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * Utility class for query execution
+ */
+public class QueryUtil {
+
+  /**
+   * Below method will be used to get the masked byte range based on the query
+   * dimension. It will give the range in the mdkey. This will be used to get
+   * the actual key array from masked mdkey
+   *
+   * @param queryDimensions query dimension selected in query
+   * @param keyGeneratorkey generator
+   * @return masked key
+   */
+  public static int[] getMaskedByteRange(List queryDimensions,
+  KeyGenerator keyGenerator) {
+Set byteRangeSet = new TreeSet();
+int[] byteRange = null;
+for (int i = 0; i < queryDimensions.size(); i++) {
+
+  // as no dictionary column and complex type columns
+  // are not selected in the mdkey
+  // so we will not select the those dimension for calculating the
+  // range
+  if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
+continue;
+  }
+  // get the offset of the dimension in the mdkey
+  byteRange =
+  
keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal());
+  for 

[51/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

move core package

change update package name

fix style

fix compile


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ce09aaaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ce09aaaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ce09aaaf

Branch: refs/heads/master
Commit: ce09aaaf76c313b21f535eca9b53e99fb88668fa
Parents: 1d5b5fc
Author: jackylk 
Authored: Mon Jan 16 21:18:23 2017 +0800
Committer: chenliang613 
Committed: Mon Jan 16 22:41:59 2017 +0800

--
 .../common/ext/ColumnUniqueIdGenerator.java |   41 -
 .../common/ext/DictionaryFactory.java   |  119 --
 .../carbondata/common/ext/PathFactory.java  |   46 -
 .../common/factory/CarbonCommonFactory.java |   54 -
 .../BlockletLevelDeleteDeltaDataCache.java  |   46 -
 .../iudprocessor/iuddata/BlockMappingVO.java|   54 -
 .../iuddata/BlockletDeleteDeltaCacheLoader.java |   80 -
 .../iuddata/DeleteDeltaCacheLoaderIntf.java |   27 -
 .../iudprocessor/iuddata/RowCountDetailsVO.java |   68 -
 .../carbondata/core/cache/CacheProvider.java|8 +-
 .../apache/carbondata/core/cache/CacheType.java |8 +-
 .../dictionary/AbstractDictionaryCache.java |8 +-
 .../cache/dictionary/ColumnDictionaryInfo.java  |2 +-
 .../cache/dictionary/DictionaryCacheLoader.java |2 +-
 .../dictionary/DictionaryCacheLoaderImpl.java   |6 +-
 .../DictionaryColumnUniqueIdentifier.java   |6 +-
 .../dictionary/DoubleArrayTrieDictionary.java   |5 +-
 .../BlockletLevelDeleteDeltaDataCache.java  |   46 +
 .../core/carbon/AbsoluteTableIdentifier.java|  141 --
 .../core/carbon/CarbonDataLoadSchema.java   |  184 --
 .../core/carbon/CarbonTableIdentifier.java  |  139 --
 .../core/carbon/ColumnIdentifier.java   |  114 --
 .../core/carbon/ColumnarFormatVersion.java  |   52 -
 .../datastore/AbstractBlockIndexStoreCache.java |  115 --
 .../core/carbon/datastore/BTreeBuilderInfo.java |   61 -
 .../core/carbon/datastore/BlockIndexStore.java  |  381 
 .../core/carbon/datastore/BtreeBuilder.java |   38 -
 .../core/carbon/datastore/DataRefNode.java  |  128 --
 .../carbon/datastore/DataRefNodeFinder.java |   43 -
 .../core/carbon/datastore/IndexKey.java |   62 -
 .../carbon/datastore/SegmentTaskIndexStore.java |  401 
 .../datastore/TableSegmentUniqueIdentifier.java |  144 --
 .../carbon/datastore/block/AbstractIndex.java   |  129 --
 .../core/carbon/datastore/block/BlockIndex.java |   52 -
 .../core/carbon/datastore/block/BlockInfo.java  |  134 --
 .../carbon/datastore/block/BlockletInfos.java   |  112 --
 .../carbon/datastore/block/Distributable.java   |   27 -
 .../datastore/block/SegmentProperties.java  |  754 ---
 .../datastore/block/SegmentTaskIndex.java   |   57 -
 .../block/SegmentTaskIndexWrapper.java  |  148 --
 .../carbon/datastore/block/TableBlockInfo.java  |  311 ---
 .../block/TableBlockUniqueIdentifier.java   |   72 -
 .../carbon/datastore/block/TableTaskInfo.java   |  115 --
 .../carbon/datastore/block/TaskBlockInfo.java   |   68 -
 .../chunk/DimensionColumnDataChunk.java |  111 --
 .../datastore/chunk/MeasureColumnDataChunk.java |   74 -
 .../chunk/impl/AbstractDimensionDataChunk.java  |   90 -
 .../impl/ColumnGroupDimensionDataChunk.java |  186 --
 .../impl/FixedLengthDimensionDataChunk.java |  161 --
 .../impl/VariableLengthDimensionDataChunk.java  |  153 --
 .../chunk/reader/CarbonDataReaderFactory.java   |   94 -
 .../reader/DimensionColumnChunkReader.java  |   52 -
 .../chunk/reader/MeasureColumnChunkReader.java  |   50 -
 .../reader/dimension/AbstractChunkReader.java   |  100 -
 ...mpressedDimensionChunkFileBasedReaderV1.java |  145 --
 ...mpressedDimensionChunkFileBasedReaderV2.java |  288 ---
 .../measure/AbstractMeasureChunkReader.java |   41 -
 ...CompressedMeasureChunkFileBasedReaderV1.java |  108 -
 ...CompressedMeasureChunkFileBasedReaderV2.java |  236 ---
 .../chunk/store/DimensionChunkStoreFactory.java |   88 -
 .../chunk/store/DimensionDataChunkStore.java|   95 -
 .../chunk/store/MeasureChunkStoreFactory.java   |   97 -
 .../chunk/store/MeasureDataChunkStore.java  |   86 -
 .../SafeAbsractDimensionDataChunkStore.java |  126 --
 .../safe/SafeAbstractMeasureDataChunkStore.java |  114 --
 .../impl/safe/SafeByteMeasureChunkStore.java|   55 -
 .../impl/safe/SafeDoubleMeasureChunkStore.java  |   54 -
 .../SafeFixedLengthDimensionDataChunkStore.java |  114 --
 .../impl/safe/SafeIntMeasureChunkStore.java |   54 -
 .../impl/safe/SafeLongMeasureChunkStore.java|   55 -
 .../impl/safe/SafeShortMeasureChunkStore.java   |   56 -
 ...feVariableLengthDimensionDataChunkStore.java |  139 --
 

[49/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
 
b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
deleted file mode 100644
index 4dd76c6..000
--- 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastore.block;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.carbondata.core.cache.Cacheable;
-import org.apache.carbondata.core.carbon.datastore.DataRefNode;
-import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-
-public abstract class AbstractIndex implements Cacheable {
-
-  /**
-   * vo class which will hold the RS information of the block
-   */
-  protected SegmentProperties segmentProperties;
-
-  /**
-   * data block
-   */
-  protected DataRefNode dataRefNode;
-
-  /**
-   * total number of row present in the block
-   */
-  protected long totalNumberOfRows;
-
-  /**
-   * atomic integer to maintain the access count for a column access
-   */
-  protected AtomicInteger accessCount = new AtomicInteger();
-
-  /**
-   * Table block meta size.
-   */
-  protected long memorySize;
-
-  /**
-   * @return the segmentProperties
-   */
-  public SegmentProperties getSegmentProperties() {
-return segmentProperties;
-  }
-
-  /**
-   * @return the dataBlock
-   */
-  public DataRefNode getDataRefNode() {
-return dataRefNode;
-  }
-
-  @Override public long getFileTimeStamp() {
-return 0;
-  }
-
-  /**
-   * Below method will be used to load the data block
-   *
-   * @param footerList footer list
-   */
-  public abstract void buildIndex(List footerList);
-
-  /**
-   * the method returns the access count
-   *
-   * @return
-   */
-  @Override public int getAccessCount() {
-return accessCount.get();
-  }
-
-  /**
-   * The method returns table block size
-   *
-   * @return
-   */
-  @Override public long getMemorySize() {
-return this.memorySize;
-  }
-
-  /**
-   * The method is used to set the access count
-   */
-  public void incrementAccessCount() {
-accessCount.incrementAndGet();
-  }
-
-  /**
-   * This method will release the objects and set default value for primitive 
types
-   */
-  public void clear() {
-decrementAccessCount();
-  }
-
-  /**
-   * This method will decrement the access count for a column by 1
-   * whenever a column usage is complete
-   */
-  private void decrementAccessCount() {
-if (accessCount.get() > 0) {
-  accessCount.decrementAndGet();
-}
-  }
-
-  /**
-   * the method is used to set the memory size of the b-tree
-   * @param memorySize
-   */
-  public void setMemorySize(long memorySize) {
-this.memorySize = memorySize;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
 
b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
deleted file mode 100644
index 75ebb84..000
--- 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless 

[37/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
new file mode 100644
index 000..6100dae
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing Byte array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeByteMeasureChunkStore extends 
UnsafeAbstractMeasureDataChunkStore {
+
+  public UnsafeByteMeasureChunkStore(int numberOfRows) {
+super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put byte array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(byte[] data) {
+assert (!this.isMemoryOccupied);
+this.dataPageMemoryBlock =
+
MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length);
+// copy the data to memory
+CarbonUnsafe.unsafe
+.copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, 
dataPageMemoryBlock.getBaseObject(),
+dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override public byte getByte(int index) {
+return CarbonUnsafe.unsafe
+.getByte(dataPageMemoryBlock.getBaseObject(), 
dataPageMemoryBlock.getBaseOffset() + index);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
new file mode 100644
index 000..5b888ed
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing double array data to memory. memory can be on heap 
or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeDoubleMeasureChunkStore extends 
UnsafeAbstractMeasureDataChunkStore {
+
+  public 

[09/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
deleted file mode 100644
index 72d47df..000
--- 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastore.block;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-import junit.framework.TestCase;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class SegmentPropertiesTest extends TestCase {
-
-  private SegmentProperties blockMetadataInfos;
-
-  @BeforeClass public void setUp() {
-List columnSchema = new ArrayList();
-columnSchema.add(getDimensionColumn1());
-columnSchema.add(getDimensionColumn2());
-columnSchema.add(getDimensionColumn3());
-columnSchema.add(getDimensionColumn4());
-columnSchema.add(getDimensionColumn5());
-columnSchema.add(getDimensionColumn9());
-columnSchema.add(getDimensionColumn10());
-columnSchema.add(getDimensionColumn11());
-columnSchema.add(getDimensionColumn6());
-columnSchema.add(getDimensionColumn7());
-columnSchema.add(getMeasureColumn());
-columnSchema.add(getMeasureColumn1());
-int[] cardinality = new int[columnSchema.size()];
-int x = 100;
-for (int i = 0; i < columnSchema.size(); i++) {
-  cardinality[i] = x;
-  x++;
-}
-blockMetadataInfos = new SegmentProperties(columnSchema, cardinality);
-  }
-
-  @Test public void testBlockMetadataHasProperDimensionCardinality() {
-int[] cardinality = { 100, 102, 103, 105, 106, 107 };
-boolean isProper = true;
-for (int i = 0; i < cardinality.length; i++) {
-  isProper = cardinality[i] == 
blockMetadataInfos.getDimColumnsCardinality()[i];
-  if (!isProper) {
-assertTrue(false);
-  }
-}
-assertTrue(true);
-  }
-
-  @Test public void testBlockMetadataHasProperComplesDimensionCardinality() {
-int[] cardinality = { 108, 109 };
-boolean isProper = true;
-for (int i = 0; i < cardinality.length; i++) {
-  isProper = cardinality[i] == 
blockMetadataInfos.getComplexDimColumnCardinality()[i];
-  if (!isProper) {
-assertTrue(false);
-  }
-}
-assertTrue(true);
-  }
-
-  @Test public void testBlockMetadataHasProperDimensionBlockMapping() {
-Map dimensionOrdinalToBlockMapping = new 
HashMap();
-dimensionOrdinalToBlockMapping.put(0, 0);
-dimensionOrdinalToBlockMapping.put(1, 1);
-dimensionOrdinalToBlockMapping.put(2, 2);
-dimensionOrdinalToBlockMapping.put(3, 2);
-dimensionOrdinalToBlockMapping.put(4, 3);
-dimensionOrdinalToBlockMapping.put(5, 4);
-dimensionOrdinalToBlockMapping.put(6, 4);
-dimensionOrdinalToBlockMapping.put(7, 4);
-dimensionOrdinalToBlockMapping.put(8, 5);
-dimensionOrdinalToBlockMapping.put(9, 6);
-Map dimensionOrdinalToBlockMappingActual =
-blockMetadataInfos.getDimensionOrdinalToBlockMapping();
-assertEquals(dimensionOrdinalToBlockMapping.size(),
-dimensionOrdinalToBlockMappingActual.size());
-Iterator> iterator =
-dimensionOrdinalToBlockMapping.entrySet().iterator();
-while (iterator.hasNext()) {
-  Entry next = iterator.next();
-  Integer integer = 

[06/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
 
b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
new file mode 100644
index 000..72b9c01
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.block;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+public class SegmentPropertiesTestUtil {
+
+  public static SegmentProperties getSegmentProperties() {
+List columnSchema = new ArrayList();
+columnSchema.add(getDimensionColumn1());
+columnSchema.add(getDimensionColumn2());
+columnSchema.add(getDimensionColumn3());
+columnSchema.add(getDimensionColumn4());
+columnSchema.add(getDimensionColumn5());
+columnSchema.add(getDimensionColumn9());
+columnSchema.add(getDimensionColumn10());
+columnSchema.add(getDimensionColumn11());
+columnSchema.add(getDimensionColumn6());
+columnSchema.add(getDimensionColumn7());
+columnSchema.add(getMeasureColumn());
+columnSchema.add(getMeasureColumn1());
+int[] cardinality = new int[columnSchema.size()];
+int x = 100;
+for (int i = 0; i < columnSchema.size(); i++) {
+  cardinality[i] = x;
+  x++;
+}
+SegmentProperties segmentProperties = new SegmentProperties(columnSchema, 
cardinality);
+return segmentProperties;
+  }
+
+  public static ColumnSchema getDimensionColumn1() {
+ColumnSchema dimColumn = new ColumnSchema();
+dimColumn.setColumnar(true);
+dimColumn.setColumnName("IMEI");
+dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+dimColumn.setDataType(DataType.STRING);
+dimColumn.setDimensionColumn(true);
+List encodeList =
+new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+encodeList.add(Encoding.DICTIONARY);
+dimColumn.setEncodingList(encodeList);
+dimColumn.setNumberOfChild(0);
+return dimColumn;
+  }
+
+  public static ColumnSchema getDimensionColumn2() {
+ColumnSchema dimColumn = new ColumnSchema();
+dimColumn.setColumnar(true);
+dimColumn.setColumnName("IMEI1");
+dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+dimColumn.setDataType(DataType.STRING);
+dimColumn.setDimensionColumn(true);
+List encodeList =
+new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+dimColumn.setEncodingList(encodeList);
+dimColumn.setNumberOfChild(0);
+return dimColumn;
+  }
+
+  public static ColumnSchema getDimensionColumn3() {
+ColumnSchema dimColumn = new ColumnSchema();
+dimColumn.setColumnar(false);
+dimColumn.setColumnName("IMEI2");
+dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+dimColumn.setDataType(DataType.STRING);
+dimColumn.setDimensionColumn(true);
+List encodeList =
+new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+encodeList.add(Encoding.DICTIONARY);
+dimColumn.setEncodingList(encodeList);
+dimColumn.setColumnGroup(0);
+dimColumn.setNumberOfChild(0);
+return dimColumn;
+  }
+
+  public static ColumnSchema getDimensionColumn4() {
+ColumnSchema dimColumn = new ColumnSchema();
+dimColumn.setColumnar(false);
+dimColumn.setColumnName("IMEI3");
+dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+dimColumn.setDataType(DataType.STRING);
+dimColumn.setDimensionColumn(true);
+List encodeList =
+new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+

[31/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
new file mode 100644
index 000..b24eaee
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table.column;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+
+public class CarbonColumn implements Serializable {
+
+  /**
+   * serialization version
+   */
+  private static final long serialVersionUID = 3648269871256322681L;
+
+  /**
+   * column schema
+   */
+  protected ColumnSchema columnSchema;
+
+  /**
+   * table ordinal
+   */
+  protected int ordinal;
+
+  /**
+   * order in which user has created table
+   */
+  protected int schemaOrdinal;
+  /**
+   * default value for in case of restructuring will be used when older
+   * segment does not have particular column
+   */
+  protected byte[] defaultValue;
+
+  /**
+   * Column identifier
+   */
+  protected ColumnIdentifier columnIdentifier;
+
+  public CarbonColumn(ColumnSchema columnSchema, int ordinal, int 
schemaOrdinal) {
+this.columnSchema = columnSchema;
+this.ordinal = ordinal;
+this.schemaOrdinal = schemaOrdinal;
+this.columnIdentifier =
+ new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType());
+  }
+  /**
+   * @return columnar or row based
+   */
+  public boolean isColumnar() {
+return columnSchema.isColumnar();
+  }
+
+  /**
+   * @return column unique id
+   */
+  public String getColumnId() {
+return columnSchema.getColumnUniqueId();
+  }
+
+  /**
+   * @return the dataType
+   */
+  public DataType getDataType() {
+return columnSchema.getDataType();
+  }
+
+  /**
+   * @return the colName
+   */
+  public String getColName() {
+return columnSchema.getColumnName();
+  }
+
+  /**
+   * @return the ordinal
+   */
+  public int getOrdinal() {
+return ordinal;
+  }
+
+  /**
+   * @return the list of encoder used in dimension
+   */
+  public List getEncoder() {
+return columnSchema.getEncodingList();
+  }
+
+  /**
+   * @return row group id if it is row based
+   */
+  public int columnGroupId() {
+return columnSchema.getColumnGroupId();
+  }
+
+  /**
+   * @return the defaultValue
+   */
+  public byte[] getDefaultValue() {
+return defaultValue;
+  }
+
+  /**
+   * @param defaultValue the defaultValue to set
+   */
+  public void setDefaultValue(byte[] defaultValue) {
+this.defaultValue = defaultValue;
+  }
+
+  /**
+   * @param encoding
+   * @return true if contains the passing encoding
+   */
+  public boolean hasEncoding(Encoding encoding) {
+return columnSchema.hasEncoding(encoding);
+  }
+
+  /**
+   * @return if DataType is ARRAY or STRUCT, this method return true, else
+   * false.
+   */
+  public Boolean isComplex() {
+return columnSchema.isComplex();
+  }
+
+  /**
+   * @return if column is dimension return true, else false.
+   */
+  public Boolean isDimesion() {
+return columnSchema.isDimensionColumn();
+  }
+
+  /**
+   * return the visibility
+   * @return
+   */
+  public boolean isInvisible() {
+return columnSchema.isInvisible();
+  }
+
+  /**
+   * @return if column use inverted index return true, else false.
+   */
+  public Boolean isUseInvertedIndex() {
+return columnSchema.isUseInvertedIndex();
+  }
+  public ColumnSchema getColumnSchema() {
+return this.columnSchema;
+  }
+
+  /**
+   * @return columnproperty
+   */
+  public Map getColumnProperties() {
+return this.columnSchema.getColumnProperties();
+  

[33/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index f6e8e90..1de4702 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -24,9 +24,9 @@ import java.util.Date;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import 
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CarbonProperties;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index ec1a710..bb5edee 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -24,15 +24,19 @@ import java.util.Date;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import 
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CarbonProperties;
 
-import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_DAY;
-import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_HOUR;
-import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_MIN;
-import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_SEC;
+import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+.TimeStampGranularityConstants.TIME_GRAN_DAY;
+import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+.TimeStampGranularityConstants.TIME_GRAN_HOUR;
+import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+.TimeStampGranularityConstants.TIME_GRAN_MIN;
+import static 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+.TimeStampGranularityConstants.TIME_GRAN_SEC;
 
 /**
  * The class provides the method to generate dictionary key and getting the 
actual value from

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java 
b/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
deleted file mode 100644
index 69c7cf5..000
--- a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS 

[43/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
 
b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
deleted file mode 100644
index 013d873..000
--- 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class BlockIndexerStorageForInt implements IndexStorage {
-  private boolean alreadySorted;
-
-  private int[] dataAfterComp;
-
-  private int[] indexMap;
-
-  private byte[][] keyBlock;
-
-  private int[] dataIndexMap;
-
-  private int totalSize;
-
-  public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, 
boolean isNoDictionary,
-  boolean isSortRequired) {
-ColumnWithIntIndex[] columnWithIndexs = 
createColumnWithIndexArray(keyBlock, isNoDictionary);
-if (isSortRequired) {
-  Arrays.sort(columnWithIndexs);
-}
-compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
-if (compressData) {
-  compressDataMyOwnWay(columnWithIndexs);
-}
-  }
-
-  /**
-   * Create an object with each column array and respective index
-   *
-   * @return
-   */
-  private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
-  boolean isNoDictionary) {
-ColumnWithIntIndex[] columnWithIndexs;
-if (isNoDictionary) {
-  columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
-  for (int i = 0; i < columnWithIndexs.length; i++) {
-columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], 
i);
-  }
-
-} else {
-  columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
-  for (int i = 0; i < columnWithIndexs.length; i++) {
-columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
-  }
-}
-
-return columnWithIndexs;
-  }
-
-  private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] 
columnWithIndexs,
-  byte[][] keyBlock) {
-int[] indexes = new int[columnWithIndexs.length];
-for (int i = 0; i < indexes.length; i++) {
-  indexes[i] = columnWithIndexs[i].getIndex();
-  keyBlock[i] = columnWithIndexs[i].getColumn();
-}
-this.keyBlock = keyBlock;
-return indexes;
-  }
-
-  /**
-   * It compresses depends up on the sequence numbers.
-   * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
-   * first array the start and end of sequential numbers and second array
-   * keeps the indexes of where sequential numbers starts. If there is no
-   * sequential numbers then the same array it returns with empty second
-   * array.
-   *
-   * @param indexes
-   */
-  public void compressMyOwnWay(int[] indexes) {
-List list = new 
ArrayList(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-List map = new 
ArrayList(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-int k = 0;
-int i = 1;
-for (; i < indexes.length; i++) {
-  if (indexes[i] - indexes[i - 1] == 1) {
-k++;
-  } else {
-if (k > 0) {
-  map.add((list.size()));
-  list.add(indexes[i - k - 1]);
-  list.add(indexes[i - 1]);
-} else {
-  list.add(indexes[i - 1]);
-}
-k = 0;
-  }
-}
-if (k > 0) {
-  map.add((list.size()));
-  list.add(indexes[i - k - 1]);
-  list.add(indexes[i - 1]);
-} else {
-  list.add(indexes[i - 1]);
-}
-dataAfterComp = convertToArray(list);
-if (indexes.length == dataAfterComp.length) {
-  indexMap = new int[0];
-} else {
-  indexMap = 

[47/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
deleted file mode 100644
index fed4ab4..000
--- 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.carbon.datastore.chunk.store;
-
-import java.math.BigDecimal;
-
-/**
- * Responsibility is store the measure data in memory,
- * memory can be on heap or offheap based on the user configuration
- */
-public interface MeasureDataChunkStore {
-
-  /**
-   * Below method will be used to put the data to memory
-   *
-   * @param data
-   */
-  void putData(T data);
-
-  /**
-   * to get byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  byte getByte(int index);
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return short value based on index
-   */
-  short getShort(int index);
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  int getInt(int index);
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  long getLong(int index);
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  double getDouble(int index);
-
-  /**
-   * To get the bigdecimal value
-   *
-   * @param index
-   * @return bigdecimal value based on index
-   */
-  BigDecimal getBigDecimal(int index);
-
-  /**
-   * To free the occupied memory
-   */
-  void freeMemory();
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
deleted file mode 100644
index 0ed2218..000
--- 
a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
-
-import 
org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore;
-
-/**
- * Responsibility is to store dimension data
- */
-public abstract class SafeAbsractDimensionDataChunkStore implements 
DimensionDataChunkStore {
-
-  /**
-   * data chunk for dimension column
-   */
-  protected byte[] data;
-
-  /**
-   * inverted index
-   */
-  protected int[] invertedIndex;
-
-  /**
-   * inverted index reverser
-   */
-  protected int[] invertedIndexReverse;
-
-  /**
-   * to check whether dimension column was 

[42/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
 
b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
deleted file mode 100644
index 1961d3f..000
--- 
a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
-import 
org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import 
org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import 
org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalDefault extends 
ValueCompressionHolder {
-  /**
-   * Attribute for Carbon LOGGER
-   */
-  private static final LogService LOGGER =
-  
LogServiceFactory.getLogService(CompressionNonDecimalDefault.class.getName());
-  /**
-   * doubleCompressor.
-   */
-  private static Compressor compressor = 
CompressorFactory.getInstance().getCompressor();
-  /**
-   * value.
-   */
-  private double[] value;
-
-  private MeasureDataChunkStore measureChunkStore;
-
-  private double divisionFactory;
-
-  @Override
-  public void uncompress(DataType dataType, byte[] compressedData, int offset, 
int length,
-  int decimalPlaces, Object maxValueObject) {
-super.unCompress(compressor, dataType, compressedData, offset, length);
-setUncompressedValues(value, decimalPlaces);
-  }
-
-  @Override public void compress() {
-compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
-  }
-
-  @Override public void setValue(double[] value) {
-this.value = value;
-
-  }
-
-  @Override public double[] getValue() {return this.value; }
-
-  @Override public void setValueInBytes(byte[] value) {
-ByteBuffer buffer = ByteBuffer.wrap(value);
-this.value = ValueCompressionUtil.convertToDoubleArray(buffer, 
value.length);
-  }
-
-  @Override public long getLongValue(int index) {
-throw new UnsupportedOperationException(
-  "Long value is not defined for CompressionNonDecimalDefault");
-  }
-
-  @Override public double getDoubleValue(int index) {
-return (measureChunkStore.getDouble(index) / divisionFactory);
-  }
-
-  @Override public BigDecimal getBigDecimalValue(int index) {
-throw new UnsupportedOperationException(
-  "Big decimal value is not defined for CompressionNonDecimalDefault");
-  }
-
-  private void setUncompressedValues(double[] data, int decimalPlaces) {
-this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-.getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
-this.measureChunkStore.putData(data);
-this.divisionFactory = Math.pow(10, decimalPlaces);
-  }
-
-  @Override public void freeMemory() {
-this.measureChunkStore.freeMemory();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalInt.java
--
diff --git 

[25/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
new file mode 100644
index 000..5fc23b7
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+public class IncludeFilterExecuterImpl implements FilterExecuter {
+
+  protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+  protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected SegmentProperties segmentProperties;
+
+  public IncludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColumnEvaluatorInfo,
+  SegmentProperties segmentProperties) {
+this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+this.segmentProperties = segmentProperties;
+dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+
FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
+segmentProperties, dimColumnEvaluatorInfo.getDimension(), 
dimColumnExecuterInfo);
+
+  }
+
+  @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) 
throws IOException {
+int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+.get(dimColumnEvaluatorInfo.getColumnIndex());
+if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+  blockChunkHolder.getDimensionDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
+  .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+}
+return 
getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+blockChunkHolder.getDataBlock().nodeSize());
+  }
+
+  protected BitSet getFilteredIndexes(DimensionColumnDataChunk 
dimensionColumnDataChunk,
+  int numerOfRows) {
+if (dimensionColumnDataChunk.isNoDicitionaryColumn()
+&& dimensionColumnDataChunk instanceof 
VariableLengthDimensionDataChunk) {
+  return setDirectKeyFilterIndexToBitSet(
+  (VariableLengthDimensionDataChunk) dimensionColumnDataChunk, 
numerOfRows);
+} else if (dimensionColumnDataChunk.isExplicitSorted()
+&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+  return setFilterdIndexToBitSetWithColumnIndex(
+  (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, 
numerOfRows);
+}
+
+return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+  }
+
+  private BitSet setDirectKeyFilterIndexToBitSet(
+  VariableLengthDimensionDataChunk dimensionColumnDataChunk, int 
numerOfRows) {
+BitSet bitSet = new BitSet(numerOfRows);
+byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+for (int i = 0; i < filterValues.length; i++) {
+  byte[] filterVal = filterValues[i];
+  if (dimensionColumnDataChunk.isExplicitSorted()) {
+for (int index = 0; index < numerOfRows; index++) {
+  if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
+bitSet.set(dimensionColumnDataChunk.getInvertedIndex(index));
+ 

[16/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java 
b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
deleted file mode 100644
index 88088ee..000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
+++ /dev/null
@@ -1,882 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.complextypes.ArrayQueryType;
-import org.apache.carbondata.scan.complextypes.PrimitiveQueryType;
-import org.apache.carbondata.scan.complextypes.StructQueryType;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import 
org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
-
-import org.apache.commons.lang3.ArrayUtils;
-
-/**
- * Utility class for query execution
- */
-public class QueryUtil {
-
-  /**
-   * Below method will be used to get the masked byte range based on the query
-   * dimension. It will give the range in the mdkey. This will be used to get
-   * the actual key array from masked mdkey
-   *
-   * @param queryDimensions query dimension selected in query
-   * @param keyGeneratorkey generator
-   * @return masked key
-   */
-  public static int[] getMaskedByteRange(List queryDimensions,
-  KeyGenerator keyGenerator) {
-Set byteRangeSet = new TreeSet();
-int[] byteRange = null;
-for (int i = 0; i < queryDimensions.size(); i++) {
-
-  // as no dictionary column and complex type columns
-  // are not selected in the mdkey
-  // so we will not select the those dimension for calculating the
-  // range
-  if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
-continue;
-  }
-  // get the offset of the dimension in the mdkey
-  byteRange =
-  
keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal());
-  for (int j = byteRange[0]; j <= 

[23/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java 
b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
new file mode 100644
index 000..f4b1325
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.mutate.UpdateVO;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.UnknownExpression;
+import 
org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Query model which will have all the detail
+ * about the query, This will be sent from driver to executor '
+ * This will be refereed to executing the query.
+ */
+public class QueryModel implements Serializable {
+
+  /**
+   * serialization version
+   */
+  private static final long serialVersionUID = -4674677234007089052L;
+  /**
+   * this will hold the information about the dictionary dimension
+   * which to
+   */
+  public transient Map columnToDictionaryMapping;
+  /**
+   * list of dimension selected for in query
+   */
+  private List queryDimension;
+  /**
+   * list of measure selected in query
+   */
+  private List queryMeasures;
+  /**
+   * query id
+   */
+  private String queryId;
+  /**
+   * filter tree
+   */
+  private FilterResolverIntf filterExpressionResolverTree;
+
+  /**
+   * table block information in which query will be executed
+   */
+  private List tableBlockInfos;
+  /**
+   * absolute table identifier
+   */
+  private AbsoluteTableIdentifier absoluteTableIdentifier;
+  /**
+   * To handle most of the computation in query engines like spark and hive, 
carbon should give
+   * raw detailed records to it.
+   */
+  private boolean forcedDetailRawQuery;
+  /**
+   * table on which query will be executed
+   * TODO need to remove this ad pass only the path
+   * and carbon metadata will load the table from metadata file
+   */
+  private CarbonTable table;
+
+  private QueryStatisticsRecorder statisticsRecorder;
+
+  private boolean vectorReader;
+
+  /**
+   * Invalid table blocks, which need to be removed from
+   * memory, invalid blocks can be segment which are deleted
+   * or compacted
+   */
+  private List invalidSegmentIds;
+  private Map invalidSegmentBlockIdMap =
+  new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+  public QueryModel() {
+tableBlockInfos = new ArrayList();
+queryDimension = new ArrayList();
+queryMeasures = new ArrayList();
+invalidSegmentIds = new ArrayList<>();
+  }
+
+  public static QueryModel createModel(AbsoluteTableIdentifier 
absoluteTableIdentifier,
+  CarbonQueryPlan queryPlan, CarbonTable carbonTable) {
+QueryModel queryModel = new QueryModel();
+String factTableName = carbonTable.getFactTableName();
+

[05/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
new file mode 100644
index 000..7569dc6
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class ViewFsCarbonFileTest {
+
+private static ViewFSCarbonFile viewFSCarbonFile;
+private static FileStatus fileStatus;
+private static FileStatus fileStatusWithOutDirectoryPermission;
+private static String fileName;
+private static File file;
+
+
+@BeforeClass
+static public void setUp() {
+file = new File("Test.carbondata");
+if (!file.exists())
+try {
+file.createNewFile();
+} catch (IOException e) {
+e.printStackTrace();
+}
+try {
+FileOutputStream oFile = new FileOutputStream(file, true);
+} catch (FileNotFoundException e) {
+e.printStackTrace();
+}
+
+fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new 
Path(file.getAbsolutePath()));
+fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 
120l, 180L, new Path(file.getAbsolutePath()));
+fileName = file.getAbsolutePath();
+viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+}
+
+@AfterClass
+static public void cleanUp() {
+file.delete();
+}
+
+@Test
+public void testRenameForceForException() throws IOException {
+
+new MockUp() {
+@Mock
+public FileSystem getFileSystem(Configuration conf) throws 
IOException {
+throw new IOException();
+}
+
+};
+viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+viewFSCarbonFile.renameForce(fileName);
+}
+
+@Test
+public void testListFilesWithOutDirectoryPermission() {
+viewFSCarbonFile = new 
ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+assertTrue(viewFSCarbonFile.listFiles() == null);
+}
+
+@Test
+public void testConstructorWithFilePath() {
+viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath());
+assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile);
+}
+
+@Test
+public void testListFilesForNullListStatus() {
+viewFSCarbonFile = new 
ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+new MockUp() {
+@Mock
+public FileSystem getFileSystem(Configuration conf) throws 
IOException {
+return new ViewFileSystem();
+}
+
+};
+new MockUp() {
+@Mock
+public FileStatus[] listStatus(Path var1) throws IOException {
+
+return null;
+}
+
+};
+new MockUp() {
+@Mock
+public boolean delete(Path var1, boolean var2) throws IOException {
+
+return true;
+}
+
+};
+//public boolean delete(Path var1, boolean var2) throws IOException;
+

[30/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
deleted file mode 100644
index a702a6b..000
--- 
a/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.partition.impl;
-
-import java.util.List;
-
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.partition.Partitioner;
-
-/**
- * Hash partitioner implementation
- */
-public class HashPartitionerImpl implements Partitioner {
-
-  private int numberOfBuckets;
-
-  private Hash[] hashes;
-
-  public HashPartitionerImpl(List indexes, List 
columnSchemas,
-  int numberOfBuckets) {
-this.numberOfBuckets = numberOfBuckets;
-hashes = new Hash[indexes.size()];
-for (int i = 0; i < indexes.size(); i++) {
-  switch(columnSchemas.get(i).getDataType()) {
-case SHORT:
-case INT:
-case LONG:
-  hashes[i] = new IntegralHash(indexes.get(i));
-  break;
-case DOUBLE:
-case FLOAT:
-case DECIMAL:
-  hashes[i] = new DecimalHash(indexes.get(i));
-  break;
-default:
-  hashes[i] = new StringHash(indexes.get(i));
-  }
-}
-  }
-
-  @Override public int getPartition(Object[] objects) {
-int hashCode = 0;
-for (Hash hash : hashes) {
-  hashCode += hash.getHash(objects);
-}
-return (hashCode & Integer.MAX_VALUE) % numberOfBuckets;
-  }
-
-  private interface Hash {
-int getHash(Object[] value);
-  }
-
-  private static class IntegralHash implements Hash {
-
-private int index;
-
-private IntegralHash(int index) {
-  this.index = index;
-}
-
-public int getHash(Object[] value) {
-  return value[index] != null ? 
Long.valueOf(value[index].toString()).hashCode() : 0;
-}
-  }
-
-  private static class DecimalHash implements Hash {
-
-private int index;
-
-private DecimalHash(int index) {
-  this.index = index;
-}
-
-public int getHash(Object[] value) {
-  return value[index] != null ? 
Double.valueOf(value[index].toString()).hashCode() : 0;
-}
-  }
-
-  private static class StringHash implements Hash {
-
-private int index;
-
-private StringHash(int index) {
-  this.index = index;
-}
-
-@Override public int getHash(Object[] value) {
-  return value[index] != null ? value[index].hashCode() : 0;
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReader.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReader.java
 
b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReader.java
index 10f7029..db5f068 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReader.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReader.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.reader;
 
 import java.io.IOException;
 
-import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
+import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index dac2d43..854a23b 

[21/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorder.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorder.java
 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorder.java
new file mode 100644
index 000..fd9c0ef
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorder.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+/**
+ * interface will be used to record and log the query statistics
+ */
+public interface QueryStatisticsRecorder {
+
+  void recordStatistics(QueryStatistic statistic);
+
+  void logStatistics();
+
+  void logStatisticsAsTableExecutor();
+
+  void recordStatisticsForDriver(QueryStatistic statistic, String queryId);
+
+  void logStatisticsAsTableDriver();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderDummy.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderDummy.java
 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderDummy.java
new file mode 100644
index 000..a751936
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderDummy.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+import java.io.Serializable;
+
+/**
+ * Class will be used to record and log the query statistics
+ */
+public class QueryStatisticsRecorderDummy implements 
QueryStatisticsRecorder,Serializable {
+
+  /**
+   * serialization version
+   */
+  private static final long serialVersionUID = -5719752001674467864L;
+
+  public QueryStatisticsRecorderDummy() {
+
+  }
+
+  /**
+   * Below method will be used to add the statistics
+   *
+   * @param statistic
+   */
+  public synchronized void recordStatistics(QueryStatistic statistic) {
+
+  }
+
+  /**
+   * Below method will be used to log the statistic
+   */
+  public void logStatistics() {
+
+  }
+
+  /**
+   * Below method will be used to show statistic log as table
+   */
+  public void logStatisticsAsTableExecutor() {
+
+  }
+
+  public void recordStatisticsForDriver(QueryStatistic statistic, String 
queryId) {
+
+  }
+
+  public void logStatisticsAsTableDriver() {
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
new file mode 100644
index 000..9cfeab4
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * 

[18/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 1683f24..6037dc5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -27,8 +27,8 @@ import java.io.OutputStreamWriter;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
 import org.apache.carbondata.core.util.CarbonUtil;
 
 import com.google.gson.Gson;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index f8e8a65..f0b39c5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -25,20 +25,20 @@ import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
-import org.apache.carbondata.core.carbon.ColumnIdentifier;
-import org.apache.carbondata.core.carbon.path.CarbonTablePath;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
 import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
 import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
 import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
 import org.apache.carbondata.core.service.PathService;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.format.ColumnDictionaryChunk;
 import org.apache.carbondata.format.ColumnDictionaryChunkMeta;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java 
b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
index 93b17f3..3f446f5 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.writer;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.util.CarbonUtil;
 
 import org.apache.hadoop.fs.FSDataOutputStream;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index cb66214..15a8df8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ 

[34/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
new file mode 100644
index 000..48928c6
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileHolder;
+
+public class FileHolderImpl implements FileHolder {
+  /**
+   * cache to hold filename and its stream
+   */
+  private Map fileNameAndStreamCache;
+
+  /**
+   * FileHolderImpl Constructor
+   * It will create the cache
+   */
+  public FileHolderImpl() {
+this.fileNameAndStreamCache =
+new HashMap(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+  }
+
+  public FileHolderImpl(int capacity) {
+this.fileNameAndStreamCache = new HashMap(capacity);
+  }
+
+  /**
+   * This method will be used to read the byte array from file based on offset
+   * and length(number of bytes) need to read
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @param length   number of bytes to be read
+   * @return read byte array
+   */
+  @Override public byte[] readByteArray(String filePath, long offset, int 
length)
+  throws IOException {
+FileChannel fileChannel = updateCache(filePath);
+ByteBuffer byteBffer = read(fileChannel, length, offset);
+return byteBffer.array();
+  }
+
+  /**
+   * This method will be used to close all the streams currently present in 
the cache
+   */
+  @Override public void finish() throws IOException {
+for (Entry entry : fileNameAndStreamCache.entrySet()) 
{
+  FileChannel channel = entry.getValue();
+  if (null != channel) {
+channel.close();
+  }
+}
+  }
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read int
+   */
+  @Override public int readInt(String filePath, long offset) throws 
IOException {
+FileChannel fileChannel = updateCache(filePath);
+ByteBuffer byteBffer = read(fileChannel, 
CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
+return byteBffer.getInt();
+  }
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @return read int
+   */
+  @Override public int readInt(String filePath) throws IOException {
+FileChannel fileChannel = updateCache(filePath);
+ByteBuffer byteBffer = read(fileChannel, 
CarbonCommonConstants.INT_SIZE_IN_BYTE);
+return byteBffer.getInt();
+  }
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read int
+   */
+  @Override public long readDouble(String filePath, long offset) throws 
IOException {
+FileChannel fileChannel = updateCache(filePath);
+ByteBuffer byteBffer = read(fileChannel, 
CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+return byteBffer.getLong();
+  }
+
+  /**
+   * This method will be used to check whether stream is already present in
+   * cache 

[52/52] incubator-carbondata git commit: [CARBONDATA-638] Move and refine package in carbon-core module This closes #532

[chenliang613]

[CARBONDATA-638] Move and refine package in carbon-core module This closes #532


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3b6b1e83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3b6b1e83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3b6b1e83

Branch: refs/heads/master
Commit: 3b6b1e837cf2638306155671fcd369af3ce31bab
Parents: 1d5b5fc ce09aaa
Author: chenliang613 
Authored: Mon Jan 16 22:43:11 2017 +0800
Committer: chenliang613 
Committed: Mon Jan 16 22:43:11 2017 +0800

--
 .../common/ext/ColumnUniqueIdGenerator.java |   41 -
 .../common/ext/DictionaryFactory.java   |  119 --
 .../carbondata/common/ext/PathFactory.java  |   46 -
 .../common/factory/CarbonCommonFactory.java |   54 -
 .../BlockletLevelDeleteDeltaDataCache.java  |   46 -
 .../iudprocessor/iuddata/BlockMappingVO.java|   54 -
 .../iuddata/BlockletDeleteDeltaCacheLoader.java |   80 -
 .../iuddata/DeleteDeltaCacheLoaderIntf.java |   27 -
 .../iudprocessor/iuddata/RowCountDetailsVO.java |   68 -
 .../carbondata/core/cache/CacheProvider.java|8 +-
 .../apache/carbondata/core/cache/CacheType.java |8 +-
 .../dictionary/AbstractDictionaryCache.java |8 +-
 .../cache/dictionary/ColumnDictionaryInfo.java  |2 +-
 .../cache/dictionary/DictionaryCacheLoader.java |2 +-
 .../dictionary/DictionaryCacheLoaderImpl.java   |6 +-
 .../DictionaryColumnUniqueIdentifier.java   |6 +-
 .../dictionary/DoubleArrayTrieDictionary.java   |5 +-
 .../BlockletLevelDeleteDeltaDataCache.java  |   46 +
 .../core/carbon/AbsoluteTableIdentifier.java|  141 --
 .../core/carbon/CarbonDataLoadSchema.java   |  184 --
 .../core/carbon/CarbonTableIdentifier.java  |  139 --
 .../core/carbon/ColumnIdentifier.java   |  114 --
 .../core/carbon/ColumnarFormatVersion.java  |   52 -
 .../datastore/AbstractBlockIndexStoreCache.java |  115 --
 .../core/carbon/datastore/BTreeBuilderInfo.java |   61 -
 .../core/carbon/datastore/BlockIndexStore.java  |  381 
 .../core/carbon/datastore/BtreeBuilder.java |   38 -
 .../core/carbon/datastore/DataRefNode.java  |  128 --
 .../carbon/datastore/DataRefNodeFinder.java |   43 -
 .../core/carbon/datastore/IndexKey.java |   62 -
 .../carbon/datastore/SegmentTaskIndexStore.java |  401 
 .../datastore/TableSegmentUniqueIdentifier.java |  144 --
 .../carbon/datastore/block/AbstractIndex.java   |  129 --
 .../core/carbon/datastore/block/BlockIndex.java |   52 -
 .../core/carbon/datastore/block/BlockInfo.java  |  134 --
 .../carbon/datastore/block/BlockletInfos.java   |  112 --
 .../carbon/datastore/block/Distributable.java   |   27 -
 .../datastore/block/SegmentProperties.java  |  754 ---
 .../datastore/block/SegmentTaskIndex.java   |   57 -
 .../block/SegmentTaskIndexWrapper.java  |  148 --
 .../carbon/datastore/block/TableBlockInfo.java  |  311 ---
 .../block/TableBlockUniqueIdentifier.java   |   72 -
 .../carbon/datastore/block/TableTaskInfo.java   |  115 --
 .../carbon/datastore/block/TaskBlockInfo.java   |   68 -
 .../chunk/DimensionColumnDataChunk.java |  111 --
 .../datastore/chunk/MeasureColumnDataChunk.java |   74 -
 .../chunk/impl/AbstractDimensionDataChunk.java  |   90 -
 .../impl/ColumnGroupDimensionDataChunk.java |  186 --
 .../impl/FixedLengthDimensionDataChunk.java |  161 --
 .../impl/VariableLengthDimensionDataChunk.java  |  153 --
 .../chunk/reader/CarbonDataReaderFactory.java   |   94 -
 .../reader/DimensionColumnChunkReader.java  |   52 -
 .../chunk/reader/MeasureColumnChunkReader.java  |   50 -
 .../reader/dimension/AbstractChunkReader.java   |  100 -
 ...mpressedDimensionChunkFileBasedReaderV1.java |  145 --
 ...mpressedDimensionChunkFileBasedReaderV2.java |  288 ---
 .../measure/AbstractMeasureChunkReader.java |   41 -
 ...CompressedMeasureChunkFileBasedReaderV1.java |  108 -
 ...CompressedMeasureChunkFileBasedReaderV2.java |  236 ---
 .../chunk/store/DimensionChunkStoreFactory.java |   88 -
 .../chunk/store/DimensionDataChunkStore.java|   95 -
 .../chunk/store/MeasureChunkStoreFactory.java   |   97 -
 .../chunk/store/MeasureDataChunkStore.java  |   86 -
 .../SafeAbsractDimensionDataChunkStore.java |  126 --
 .../safe/SafeAbstractMeasureDataChunkStore.java |  114 --
 .../impl/safe/SafeByteMeasureChunkStore.java|   55 -
 .../impl/safe/SafeDoubleMeasureChunkStore.java  |   54 -
 .../SafeFixedLengthDimensionDataChunkStore.java |  114 --
 .../impl/safe/SafeIntMeasureChunkStore.java |   54 -
 .../impl/safe/SafeLongMeasureChunkStore.java|   55 -
 .../impl/safe/SafeShortMeasureChunkStore.java   |   56 -
 ...feVariableLengthDimensionDataChunkStore.java |  139 --
 

[10/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
 
b/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
deleted file mode 100644
index 417f597..000
--- 
a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.iterator;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * It reads the data vector batch format
- */
-public class VectorDetailQueryResultIterator extends 
AbstractDetailQueryResultIterator {
-
-  private final Object lock = new Object();
-
-  public VectorDetailQueryResultIterator(List infos, 
QueryModel queryModel,
-  ExecutorService execService) {
-super(infos, queryModel, execService);
-  }
-
-  @Override public Object next() {
-throw new UnsupportedOperationException("call processNextBatch instaed");
-  }
-
-  public void processNextBatch(CarbonColumnarBatch columnarBatch) {
-synchronized (lock) {
-  updateDataBlockIterator();
-  if (dataBlockIterator != null) {
-dataBlockIterator.processNextBatch(columnarBatch);
-  }
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
 
b/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
deleted file mode 100644
index 0af1857..000
--- 
a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.scan.result.vector;
-
-import org.apache.spark.sql.types.Decimal;
-
-public interface CarbonColumnVector {
-
-  void putShort(int rowId, short value);
-
-  void putInt(int rowId, int value);
-
-  void putLong(int rowId, long value);
-
-  void putDecimal(int rowId, Decimal value, int precision);
-
-  void putDouble(int rowId, double value);
-
-  void putBytes(int rowId, byte[] value);
-
-  void putBytes(int rowId, int offset, int length, byte[] value);
-
-  void putNull(int rowId);
-
-  boolean isNull(int rowId);
-
-  void putObject(int rowId, Object obj);
-
-  Object getData(int rowId);
-
-  void reset();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java
 

[04/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
 
b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
new file mode 100644
index 000..7709042
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -0,0 +1,1846 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.metadata.converter;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
+import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.metadata.schema.table.TableSchema;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ThriftWrapperSchemaConverterImplTest {
+
+  private static ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter 
= null;
+  private static org.apache.carbondata.format.SchemaEvolutionEntry 
schemaEvolEntry = null;
+  private static org.apache.carbondata.format.ColumnSchema thriftColumnSchema 
= null;
+  private static List columnSchemas = null;
+  private static List schemaEvolutionEntries = null;
+  private static List 
thriftColumnSchemas = null;
+  private static List
+  thriftSchemaEvolutionEntries = null;
+  private static org.apache.carbondata.format.SchemaEvolution schemaEvol = 
null;
+  private static List encoders = null;
+  private static List encodings = null;
+  private static Map columnPropertyMap = null;
+  private static org.apache.carbondata.format.TableSchema tabSchema = null;
+
+  @BeforeClass public static void setUp() {
+
+thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
+schemaEvolEntry = new org.apache.carbondata.format.SchemaEvolutionEntry();
+schemaEvolutionEntries = new ArrayList();
+schemaEvolutionEntries.add(new SchemaEvolutionEntry());
+columnSchemas = new ArrayList();
+columnSchemas.add(new ColumnSchema());
+encodings = new ArrayList<>();
+encodings.add(Encoding.INVERTED_INDEX);
+encodings.add(Encoding.DELTA);
+encodings.add(Encoding.BIT_PACKED);
+encodings.add(Encoding.DICTIONARY);
+encodings.add(Encoding.RLE);
+encodings.add(Encoding.DIRECT_DICTIONARY);
+encoders = new ArrayList();
+encoders.add(org.apache.carbondata.format.Encoding.INVERTED_INDEX);
+encoders.add(org.apache.carbondata.format.Encoding.DELTA);
+encoders.add(org.apache.carbondata.format.Encoding.BIT_PACKED);
+encoders.add(org.apache.carbondata.format.Encoding.DICTIONARY);
+encoders.add(org.apache.carbondata.format.Encoding.RLE);
+encoders.add(org.apache.carbondata.format.Encoding.DIRECT_DICTIONARY);
+
+columnPropertyMap = new HashMap();
+columnPropertyMap.put("property", "value");
+thriftColumnSchema =
+new 
org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.STRING,
+"columnName", "1", true, encoders, true);
+thriftColumnSchema.setSchemaOrdinal(1);
+thriftColumnSchemas = new 
ArrayList();
+thriftColumnSchemas.add(thriftColumnSchema);
+thriftSchemaEvolutionEntries = new ArrayList<>();
+thriftSchemaEvolutionEntries.add(schemaEvolEntry);
+schemaEvol = new 
org.apache.carbondata.format.SchemaEvolution(thriftSchemaEvolutionEntries);
+
+new MockUp() {
+  @Mock public List getSchemaEvolutionEntryList() {
+return 

[08/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
deleted file mode 100644
index 4af55c0..000
--- 
a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ /dev/null
@@ -1,1846 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.metadata.converter;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.apache.carbondata.core.carbon.metadata.schema.SchemaEvolution;
-import org.apache.carbondata.core.carbon.metadata.schema.SchemaEvolutionEntry;
-import org.apache.carbondata.core.carbon.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.carbon.metadata.schema.table.TableSchema;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class ThriftWrapperSchemaConverterImplTest {
-
-  private static ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter 
= null;
-  private static org.apache.carbondata.format.SchemaEvolutionEntry 
schemaEvolEntry = null;
-  private static org.apache.carbondata.format.ColumnSchema thriftColumnSchema 
= null;
-  private static List columnSchemas = null;
-  private static List schemaEvolutionEntries = null;
-  private static List 
thriftColumnSchemas = null;
-  private static List
-  thriftSchemaEvolutionEntries = null;
-  private static org.apache.carbondata.format.SchemaEvolution schemaEvol = 
null;
-  private static List encoders = null;
-  private static List encodings = null;
-  private static Map columnPropertyMap = null;
-  private static org.apache.carbondata.format.TableSchema tabSchema = null;
-
-  @BeforeClass public static void setUp() {
-
-thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
-schemaEvolEntry = new org.apache.carbondata.format.SchemaEvolutionEntry();
-schemaEvolutionEntries = new ArrayList();
-schemaEvolutionEntries.add(new SchemaEvolutionEntry());
-columnSchemas = new ArrayList();
-columnSchemas.add(new ColumnSchema());
-encodings = new ArrayList<>();
-encodings.add(Encoding.INVERTED_INDEX);
-encodings.add(Encoding.DELTA);
-encodings.add(Encoding.BIT_PACKED);
-encodings.add(Encoding.DICTIONARY);
-encodings.add(Encoding.RLE);
-encodings.add(Encoding.DIRECT_DICTIONARY);
-encoders = new ArrayList();
-encoders.add(org.apache.carbondata.format.Encoding.INVERTED_INDEX);
-encoders.add(org.apache.carbondata.format.Encoding.DELTA);
-encoders.add(org.apache.carbondata.format.Encoding.BIT_PACKED);
-encoders.add(org.apache.carbondata.format.Encoding.DICTIONARY);
-encoders.add(org.apache.carbondata.format.Encoding.RLE);
-encoders.add(org.apache.carbondata.format.Encoding.DIRECT_DICTIONARY);
-
-columnPropertyMap = new HashMap();
-columnPropertyMap.put("property", "value");
-thriftColumnSchema =
-new 
org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.STRING,
-"columnName", "1", true, encoders, true);
-thriftColumnSchema.setSchemaOrdinal(1);
-thriftColumnSchemas = new 
ArrayList();
-thriftColumnSchemas.add(thriftColumnSchema);
-thriftSchemaEvolutionEntries = new ArrayList<>();
-thriftSchemaEvolutionEntries.add(schemaEvolEntry);
-schemaEvol = new 
org.apache.carbondata.format.SchemaEvolution(thriftSchemaEvolutionEntries);
-
-new 

[14/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java 
b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
deleted file mode 100644
index 90ef0bb..000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
+++ /dev/null
@@ -1,1353 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
-import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.IndexKey;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import 
org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.ListExpression;
-import 
org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import 
org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.executer.AndFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.DimColumnExecuterFilterInfo;
-import 
org.apache.carbondata.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.ExcludeFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import 
org.apache.carbondata.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.IncludeFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.OrFilterExecuterImpl;
-import 
org.apache.carbondata.scan.filter.executer.RestructureFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.RowLevelFilterExecuterImpl;
-import 
org.apache.carbondata.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import 

[01/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 1d5b5fcc4 -> 3b6b1e837


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java 
b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
new file mode 100644
index 000..c9a52dd
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import 
org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class FilterUtilTest extends AbstractDictionaryCacheTest {
+
+  private ColumnSchema columnSchema;
+
+  @Before public void setUp() throws Exception {
+init();
+this.databaseName = props.getProperty("database", "testSchema");
+this.tableName = props.getProperty("tableName", "carbon");
+this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+carbonTableIdentifier =
+new CarbonTableIdentifier(databaseName, tableName, 
UUID.randomUUID().toString());
+this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+columnSchema = new ColumnSchema();
+columnSchema.setColumnar(true);
+columnSchema.setColumnName("IMEI");
+columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
+columnSchema.setDataType(DataType.STRING);
+columnSchema.setDimensionColumn(true);
+  }
+
+  @Test public void testCheckIfLeftExpressionRequireEvaluation() {
+List children = new ArrayList<>();
+ListExpression expression = new ListExpression(children);
+boolean result = 
FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
+assertTrue(result);
+  }
+
+  @Test
+  public void 
testCheckIfLeftExpressionRequireEvaluationWithExpressionNotInstanceOfColumnExpression()
 {
+ColumnExpression expression = new ColumnExpression("test", 
DataType.STRING);
+boolean result = 
FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
+assertFalse(result);
+  }
+
+  @Test public void testNanSafeEqualsDoublesWithUnEqualValues() {
+Double d1 = new Double(60.67);
+Double d2 = new Double(60.69);
+boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
+assertFalse(result);
+  }
+
+  @Test public void testNanSafeEqualsDoublesWithEqualValues() {
+Double d1 = new Double(60.67);
+Double d2 = 

[29/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
new file mode 100644
index 000..b88d77b
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.common.logging.impl.StandardLogService;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import 
org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * This class provides a skeletal implementation of the {@link QueryExecutor}
+ * interface to minimize the effort required to implement this interface. This
+ * will be used to prepare all the properties required for query execution
+ */
+public abstract class AbstractQueryExecutor implements QueryExecutor {
+
+  private static final LogService LOGGER =
+  LogServiceFactory.getLogService(AbstractQueryExecutor.class.getName());
+  /**
+   * holder for query properties which will be used to execute the query
+   */
+  protected QueryExecutorProperties queryProperties;
+
+  /**
+   * query result iterator which will execute the query
+   * and give the result
+   */
+  protected CarbonIterator queryIterator;
+
+  public AbstractQueryExecutor() {
+queryProperties = new QueryExecutorProperties();
+  }
+
+  

[02/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
new file mode 100644
index 000..97e5e66
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import 
org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import 
org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class GreaterThanExpressionUnitTest {
+  static GreaterThanExpression greaterThanExpression;
+
+  @Test public void testEvaluateForGreaterThanExpressionWithStringDataType()
+  throws FilterUnsupportedException, FilterIllegalMemberException {
+ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+left.setColIndex(0);
+ColumnExpression right = new ColumnExpression("right_name", 
DataType.STRING);
+right.setColIndex(1);
+greaterThanExpression = new GreaterThanExpression(left, right);
+RowImpl value = new RowImpl();
+String[] row = { "string1" };
+String[] row1 = { "String's Value" };
+Object objectRow[] = { row, row1 };
+new MockUp() {
+  Boolean returnMockFlag = true;
+
+  @Mock public String getString() {
+if (returnMockFlag) {
+  returnMockFlag = false;
+  return "string1";
+
+} else {
+  return "String's Value";
+
+}
+
+  }
+};
+value.setValues(objectRow);
+ExpressionResult result = greaterThanExpression.evaluate(value);
+assertTrue(result.getBoolean());
+  }
+
+  @Test public void testEvaluateForGreaterThanExpressionWithShortDataType()
+  throws FilterUnsupportedException, FilterIllegalMemberException {
+ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+right.setColIndex(0);
+ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+left.setColIndex(1);
+greaterThanExpression = new GreaterThanExpression(left, right);
+RowImpl value = new RowImpl();
+Short[] row = { 170 };
+Short[] row1 = { 70 };
+Object objectRow[] = { row, row1 };
+value.setValues(objectRow);
+
+new MockUp() {
+  Boolean returnMockFlag = true;
+
+  @Mock public Short getShort() {
+if (returnMockFlag) {
+  returnMockFlag = false;
+  return 170;
+
+} else {
+  return 70;
+
+}
+
+  }
+};
+
+ExpressionResult result = greaterThanExpression.evaluate(value);
+assertTrue(result.getBoolean());
+
+  }
+
+  @Test public void testEvaluateForGreaterThanExpressionWithDoubleDataType()
+  throws FilterUnsupportedException, FilterIllegalMemberException {
+ColumnExpression right = new ColumnExpression("right_contact", 
DataType.DOUBLE);
+right.setColIndex(0);
+ColumnExpression left = new ColumnExpression("left_contact", 
DataType.DOUBLE);
+left.setColIndex(1);
+greaterThanExpression = new GreaterThanExpression(left, 

[07/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
deleted file mode 100644
index 9ff0677..000
--- 
a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.List;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class ColumnarKeyStoreDataHolderUnitTest {
-
-  private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder;
-  private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
-
-  @BeforeClass public static void setup() {
-byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
-int eachRowSize = 2;
-int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 };
-columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
-columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex);
-columnarKeyStoreDataHolder =
-new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
-  }
-
-  @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() {
-byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
-int eachRowSize = 1;
-ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new 
ColumnarKeyStoreMetadata(eachRowSize);
-ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
-new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
-int columnIndex = 5;
-int expected_result = 8;
-int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
-assertEquals(expected_result, result);
-  }
-
-  @Test public void 
testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() {
-byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
-int eachRowSize = 2;
-ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new 
ColumnarKeyStoreMetadata(eachRowSize);
-ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
-new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
-int columnIndex = 0;
-int expected_result = 4104;
-int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
-assertEquals(expected_result, result);
-  }
-
-  @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() {
-int columnIndex = 0;
-int expected_result = 8232;
-int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
-assertEquals(expected_result, result);
-  }
-
-  @Test(expected = ArrayIndexOutOfBoundsException.class)
-  public void testExceptionInGetSurrogateKey() {
-int columnIndex = 10;
-int expected_result = 8232;
-int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
-assertEquals(expected_result, result);
-  }
-
-  @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() {
-byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
-List noDictionaryValBasedKeyBlockData = new 
java.util.ArrayList<>();
-noDictionaryValBasedKeyBlockData.add(keyBlockData);
-new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
-int columnIndex = 0;
-int expected_result = 8232;
-int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
-assertEquals(expected_result, result);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFileTest.java
--
diff --git 

[44/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
 
b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
deleted file mode 100644
index ac1caca..000
--- 
a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
+++ /dev/null
@@ -1,405 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.metadata.schema.table.column;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-
-/**
- * Store the information about the column meta data present the table
- */
-public class ColumnSchema implements Serializable {
-
-  /**
-   * serialization version
-   */
-  private static final long serialVersionUID = 7676766554874863763L;
-
-  /**
-   * dataType
-   */
-  private DataType dataType;
-  /**
-   * Name of the column. If it is a complex data type, we follow a naming rule
-   * grand_parent_column.parent_column.child_column
-   * For Array types, two columns will be stored one for
-   * the array type and one for the primitive type with
-   * the name parent_column.value
-   */
-  private String columnName;
-
-  /**
-   * Unique ID for a column. if this is dimension,
-   * it is an unique ID that used in dictionary
-   */
-  private String columnUniqueId;
-
-  /**
-   * column reference id
-   */
-  private String columnReferenceId;
-
-  /**
-   * whether it is stored as columnar format or row format
-   */
-  private boolean isColumnar = true;
-
-  /**
-   * List of encoding that are chained to encode the data for this column
-   */
-  private List encodingList;
-
-  /**
-   * Whether the column is a dimension or measure
-   */
-  private boolean isDimensionColumn;
-
-  /**
-   * Whether the column should use inverted index
-   */
-  private boolean useInvertedIndex = true;
-
-  /**
-   * The group ID for column used for row format columns,
-   * where in columns in each group are chunked together.
-   */
-  private int columnGroupId = -1;
-
-  /**
-   * Used when this column contains decimal data.
-   */
-  private int scale;
-
-  private int precision;
-
-  private int schemaOrdinal;
-  /**
-   * Nested fields.  Since thrift does not support nested fields,
-   * the nesting is flattened to a single list by a depth-first traversal.
-   * The children count is used to construct the nested relationship.
-   * This field is not set when the element is a primitive type
-   */
-  private int numberOfChild;
-
-  /**
-   * used in case of schema restructuring
-   */
-  private byte[] defaultValue;
-
-  /**
-   * Column properties
-   */
-  private Map columnProperties;
-
-  /**
-   * used to define the column visibility of column default is false
-   */
-  private boolean invisible = false;
-
-  /**
-   * @return the columnName
-   */
-  public String getColumnName() {
-return columnName;
-  }
-
-  /**
-   * @param columnName the columnName to set
-   */
-  public void setColumnName(String columnName) {
-this.columnName = columnName;
-  }
-
-  /**
-   * @return the columnUniqueId
-   */
-  public String getColumnUniqueId() {
-return columnUniqueId;
-  }
-
-  /**
-   * @param columnUniqueId the columnUniqueId to set
-   */
-  public void setColumnUniqueId(String columnUniqueId) {
-this.columnUniqueId = columnUniqueId;
-  }
-
-  /**
-   * @return the isColumnar
-   */
-  public boolean isColumnar() {
-return isColumnar;
-  }
-
-  /**
-   * @param isColumnar the isColumnar to set
-   */
-  public void setColumnar(boolean isColumnar) {
-this.isColumnar = isColumnar;
-  }
-
-  /**
-   * @return the isDimensionColumn
-   */
-  public boolean isDimensionColumn() {
-return isDimensionColumn;
-  }
-
-  /**
-   * 

[26/52] [partial] incubator-carbondata git commit: move core package

[chenliang613]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
new file mode 100644
index 000..18eba0a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -0,0 +1,1353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
+import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import 
org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import 
org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.AndFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.DimColumnExecuterFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.ExcludeFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import 
org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.RestructureFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
+import 
org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import 
org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
+import 

[1/2] incubator-carbondata git commit: fixing no kettle issue for IUD. load count/ segment count should be string because in compaction case it will be 2.1

[jackylk]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master b5c20a80a -> 1d5b5fcc4


fixing no kettle issue for IUD.
load count/ segment count should be string because in compaction case it will 
be 2.1


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2cf2d193
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2cf2d193
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2cf2d193

Branch: refs/heads/master
Commit: 2cf2d1936237144a23c0a4393584a15b7beb600c
Parents: b5c20a8
Author: ravikiran 
Authored: Mon Jan 9 18:58:13 2017 +0530
Committer: jackylk 
Committed: Mon Jan 16 20:13:24 2017 +0800

--
 .../spark/rdd/CarbonDataLoadRDD.scala   |  6 +-
 .../spark/rdd/NewCarbonDataLoadRDD.scala|  4 +-
 .../carbondata/spark/rdd/UpdateDataLoad.scala   | 79 
 .../spark/rdd/CarbonDataRDDFactory.scala| 49 +++-
 .../iud/UpdateCarbonTableTestCase.scala | 13 
 5 files changed, 128 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2cf2d193/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index ff3a174..c8e3c67 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -83,7 +83,7 @@ class SparkPartitionLoader(model: CarbonLoadModel,
 splitIndex: Int,
 storePath: String,
 kettleHomePath: String,
-loadCount: Int,
+loadCount: String,
 loadMetadataDetails: LoadMetadataDetails) {
   private val LOGGER = 
LogServiceFactory.getLogService(this.getClass.getCanonicalName)
 
@@ -239,7 +239,7 @@ class DataFileLoaderRDD[K, V](
 carbonLoadModel.setSegmentId(String.valueOf(loadCount))
 setModelAndBlocksInfo()
 val loader = new SparkPartitionLoader(model, theSplit.index, storePath,
-  kettleHomePath, loadCount, loadMetadataDetails)
+  kettleHomePath, String.valueOf(loadCount), loadMetadataDetails)
 loader.initialize
 if (model.isRetentionRequest) {
   recreateAggregationTableForRetention
@@ -511,7 +511,7 @@ class DataFrameLoaderRDD[K, V](
 carbonLoadModel.setSegmentId(String.valueOf(loadCount))
 carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
 val loader = new SparkPartitionLoader(carbonLoadModel, theSplit.index, 
storePath,
-  kettleHomePath, loadCount, loadMetadataDetails)
+  kettleHomePath, String.valueOf(loadCount), loadMetadataDetails)
 loader.initialize
 val rddIteratorKey = UUID.randomUUID().toString
 try {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2cf2d193/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 46e83a5..9e13883 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -157,7 +157,7 @@ class NewCarbonDataLoadRDD[K, V](
   theSplit.index,
   null,
   null,
-  loadCount,
+  String.valueOf(loadCount),
   loadMetadataDetails)
 // Intialize to set carbon properties
 loader.initialize()
@@ -355,7 +355,7 @@ class NewDataFrameLoaderRDD[K, V](
   theSplit.index,
   null,
   null,
-  loadCount,
+  String.valueOf(loadCount),
   loadMetadataDetails)
 // Intialize to set carbon properties
 loader.initialize()

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2cf2d193/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
 

[1/2] incubator-carbondata git commit: support for float datatype

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 396d0d6b0 -> b5c20a80a


support for float datatype

remove style checks for float support

fix ordering imports for style checks

add testcases and example for float datatype


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/54e83045
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/54e83045
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/54e83045

Branch: refs/heads/master
Commit: 54e83045ee29ecbb67ab08eaaab480fa6e7b09a4
Parents: 396d0d6
Author: sandy 
Authored: Wed Jan 11 15:07:38 2017 +0530
Committer: ravipesala 
Committed: Mon Jan 16 16:56:00 2017 +0530

--
 examples/spark/src/main/resources/data.csv  | 22 
 .../examples/AllDictionaryExample.scala |  6 ++-
 examples/spark2/src/main/resources/data.csv | 20 +++
 .../examples/CarbonSessionExample.scala | 11 +++-
 .../src/test/resources/floatSample.csv  | 11 
 .../primitiveTypes/FloatDataTypeTestCase.scala  | 50 +
 .../spark/util/DataTypeConverterUtil.scala  |  3 ++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  9 +++-
 .../spark/sql/CarbonDatasourceRelation.scala|  2 +
 .../org/apache/spark/sql/CarbonSqlParser.scala  | 56 ++--
 .../spark/sql/parser/CarbonSparkSqlParser.scala | 14 +++--
 11 files changed, 149 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark/src/main/resources/data.csv
--
diff --git a/examples/spark/src/main/resources/data.csv 
b/examples/spark/src/main/resources/data.csv
index 76aec2d..5696978 100644
--- a/examples/spark/src/main/resources/data.csv
+++ b/examples/spark/src/main/resources/data.csv
@@ -1,11 +1,11 @@
-ID,date,country,name,phonetype,serialname,salary
-1,2015/7/23,china,aaa1,phone197,ASD69643,15000
-2,2015/7/24,china,aaa2,phone756,ASD42892,15001
-3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
-4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
-5,2015/7/27,china,aaa5,phone2441,ASD90633,15004
-6,2015/7/28,china,aaa6,phone294,ASD59961,15005
-7,2015/7/29,china,aaa7,phone610,ASD14875,15006
-8,2015/7/30,china,aaa8,phone1848,ASD57308,15007
-9,2015/7/18,china,aaa9,phone706,ASD86717,15008
-10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
\ No newline at end of file
+ID,date,country,name,phonetype,serialname,salary,floatField
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
--
diff --git 
a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
 
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index 4dcc868..558139a 100644
--- 
a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ 
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -40,7 +40,7 @@ object AllDictionaryExample {
 cc.sql("""
CREATE TABLE IF NOT EXISTS t3
(ID Int, date Timestamp, country String,
-   name String, phonetype String, serialname String, salary Int)
+   name String, phonetype String, serialname String, salary 
Int,floatField float)
STORED BY 'carbondata'
""")
 
@@ -53,6 +53,10 @@ object AllDictionaryExample {
SELECT * FROM t3
""").show()
 
+cc.sql("""
+   SELECT * FROM t3 where floatField=3.5
+   """).show()
+
 cc.sql("DROP TABLE IF EXISTS t3")
 
 // clean local dictionary files

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark2/src/main/resources/data.csv
--
diff --git a/examples/spark2/src/main/resources/data.csv 
b/examples/spark2/src/main/resources/data.csv
index 2722edd..a9cdf9e 100644
--- a/examples/spark2/src/main/resources/data.csv

[2/2] incubator-carbondata git commit: [CARBONDATA-390] Support for float datatype This closes #521

[ravipesala]

[CARBONDATA-390] Support for float datatype This closes #521


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b5c20a80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b5c20a80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b5c20a80

Branch: refs/heads/master
Commit: b5c20a80aced23b94177e8c93ea511bc68e92ac1
Parents: 396d0d6 54e8304
Author: ravipesala 
Authored: Mon Jan 16 16:56:48 2017 +0530
Committer: ravipesala 
Committed: Mon Jan 16 16:56:48 2017 +0530

--
 examples/spark/src/main/resources/data.csv  | 22 
 .../examples/AllDictionaryExample.scala |  6 ++-
 examples/spark2/src/main/resources/data.csv | 20 +++
 .../examples/CarbonSessionExample.scala | 11 +++-
 .../src/test/resources/floatSample.csv  | 11 
 .../primitiveTypes/FloatDataTypeTestCase.scala  | 50 +
 .../spark/util/DataTypeConverterUtil.scala  |  3 ++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  9 +++-
 .../spark/sql/CarbonDatasourceRelation.scala|  2 +
 .../org/apache/spark/sql/CarbonSqlParser.scala  | 56 ++--
 .../spark/sql/parser/CarbonSparkSqlParser.scala | 14 +++--
 11 files changed, 149 insertions(+), 55 deletions(-)
--