http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java new file mode 100644 index 0000000..7569dc6 --- /dev/null +++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.filesystem; + +import mockit.Mock; +import mockit.MockUp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +public class ViewFsCarbonFileTest { + + private static ViewFSCarbonFile viewFSCarbonFile; + private static FileStatus fileStatus; + private static FileStatus fileStatusWithOutDirectoryPermission; + private static String fileName; + private static File file; + + + @BeforeClass + static public void setUp() { + file = new File("Test.carbondata"); + if (!file.exists()) + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + try { + FileOutputStream oFile = new FileOutputStream(file, true); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); + fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath())); + fileName = file.getAbsolutePath(); + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + } + + @AfterClass + static public void cleanUp() { + file.delete(); + } + + @Test + public void testRenameForceForException() throws IOException { + + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + throw new IOException(); + } + + }; + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + viewFSCarbonFile.renameForce(fileName); + } + + @Test + public void testListFilesWithOutDirectoryPermission() { + viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission); + assertTrue(viewFSCarbonFile.listFiles() == null); + } + + @Test + public void testConstructorWithFilePath() { + viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath()); + assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile); + } + + @Test + public void testListFilesForNullListStatus() { + viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission); + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new ViewFileSystem(); + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public FileStatus[] listStatus(Path var1) throws IOException { + + return null; + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public boolean delete(Path var1, boolean var2) throws IOException { + + return true; + } + + }; + //public boolean delete(Path var1, boolean var2) throws IOException; + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + assertTrue(viewFSCarbonFile.listFiles().length == 0); + } + + @Test + public void testListDirectory() { + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new ViewFileSystem(); + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public FileStatus[] listStatus(Path var1) throws IOException { + + FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; + return fileStatus; + } + + }; + + assertTrue(viewFSCarbonFile.listFiles().length == 1); + } + + @Test + public void testListFilesForException() throws IOException { + viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission); + + new MockUp<FileStatus>() { + @Mock + public Path getPath() { + return new Path(file.getAbsolutePath()); + } + + }; + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + throw new IOException(); + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public FileStatus[] listStatus(Path var1) throws IOException { + + throw new IOException(); + } + + }; + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + viewFSCarbonFile.listFiles(); + } + + @Test + public void testListFilesWithCarbonFilter() { + CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { + + @Override + public boolean accept(CarbonFile file) { + return true; + } + }; + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 1); + } + + @Test + public void testlistFilesWithoutFilter() { + CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { + + @Override + public boolean accept(CarbonFile file) { + return false; + } + }; + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new ViewFileSystem(); + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public FileStatus[] listStatus(Path var1) throws IOException { + + FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; + return fileStatus; + } + + }; + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 0); + } + + @Test + public void testGetParentFIle() { + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new DistributedFileSystem(); + } + + }; + new MockUp<Path>() { + @Mock + public Path getParent() { + return new Path(file.getAbsolutePath() + ); + } + + }; + new MockUp<FileStatus>() { + @Mock + public Path getPath() { + return new Path(file.getAbsolutePath()); + } + + }; + new MockUp<DistributedFileSystem>() { + @Mock + public FileStatus getFileStatus(Path f) throws IOException { + + return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); + } + + }; + + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + assertFalse(viewFSCarbonFile.getParentFile().equals(null)); + } + + @Test + public void testForNonDisributedSystem() { + viewFSCarbonFile = new ViewFSCarbonFile(fileStatus); + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new WebHdfsFileSystem(); + } + + }; + assertFalse(viewFSCarbonFile.renameForce(fileName)); + } + + @Test + public void testrenameForceForViewFileSystem() { + new MockUp<Path>() { + @Mock + public FileSystem getFileSystem(Configuration conf) throws IOException { + return new ViewFileSystem(); + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public boolean delete(Path f, boolean recursive) throws + IOException { + return true; + + } + + }; + new MockUp<ViewFileSystem>() { + @Mock + public boolean rename(Path src, Path dst) throws IOException { + return true; + + } + + }; + + assertTrue(viewFSCarbonFile.renameForce(fileName)); + + } +}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java new file mode 100644 index 0000000..74e660f --- /dev/null +++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.datastore.impl.btree; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datastore.BTreeBuilderInfo; +import org.apache.carbondata.core.datastore.BtreeBuilder; +import org.apache.carbondata.core.datastore.DataRefNode; +import org.apache.carbondata.core.datastore.DataRefNodeFinder; +import org.apache.carbondata.core.datastore.IndexKey; +import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; +import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex; +import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex; +import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex; +import org.apache.carbondata.core.keygenerator.KeyGenException; +import org.apache.carbondata.core.keygenerator.KeyGenerator; +import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; +import org.apache.carbondata.core.util.CarbonUtil; + +import junit.framework.TestCase; +import org.junit.Test; + +public class BTreeBlockFinderTest extends TestCase { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(BTreeBlockFinderTest.class.getName()); + + @Test public void testBtreeBuldingIsPorper() { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getDataFileFooterList(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + + } + + @Test public void testBtreeBuilderGetMethodIsGivingNotNullRootNode() { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getDataFileFooterList(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + } + + @Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }); + ByteBuffer buffer = ByteBuffer.allocate(4 + 2); + buffer.rewind(); + buffer.putShort((short) 1); + buffer.putInt(12); + buffer.array(); + IndexKey key = new IndexKey(null, buffer.array()); + DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key); + assertEquals(1, findFirstBlock.nodeNumber()); + DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key); + assertEquals(1, findLastBlock.nodeNumber()); + } + + @Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }); + ByteBuffer buffer = ByteBuffer.allocate(4 + 1); + buffer.rewind(); + buffer.put((byte) 1); + buffer.putInt(0); + buffer.array(); + IndexKey key = new IndexKey(null, buffer.array()); + DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key); + assertEquals(0, findFirstBlock.nodeNumber()); + DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key); + assertEquals(0, findLastBlock.nodeNumber()); + } + + @Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1() + throws KeyGenException { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + + IndexKey key = + new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null); + DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key); + assertEquals(0, findFirstBlock.nodeNumber()); + DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key); + assertEquals(0, findLastBlock.nodeNumber()); + } + + @Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2() + throws KeyGenException { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + + IndexKey key = + new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null); + + DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key); + assertEquals(0, findFirstBlock.nodeNumber()); + DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key); + assertEquals(0, findLastBlock.nodeNumber()); + } + + @Test + /** + * Below method will test when key which is not present and key which is + * more than + * last node key is passes for searching it should give first block + */ + public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey() + throws KeyGenException { + BtreeBuilder builder = new BlockBTreeBuilder(); + List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey(); + BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null); + builder.build(infos); + DataRefNode dataBlock = builder.get(); + assertTrue(dataBlock != null); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + + IndexKey key = + new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 10001, 10001 }), null); + + DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key); + assertEquals(99, findFirstBlock.nodeNumber()); + DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key); + assertEquals(99, findLastBlock.nodeNumber()); + } + + private List<DataFileFooter> getDataFileFooterList() { + List<DataFileFooter> list = new ArrayList<DataFileFooter>(); + try { + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + int i = 1; + while (i < 1001) { + byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i }); + byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 }); + ByteBuffer buffer = ByteBuffer.allocate(4 + 1); + buffer.rewind(); + buffer.put((byte) 1); + buffer.putInt(i); + buffer.array(); + byte[] noDictionaryStartKey = buffer.array(); + + ByteBuffer buffer1 = ByteBuffer.allocate(4 + 1); + buffer1.rewind(); + buffer1.put((byte) 1); + buffer1.putInt(i + 10); + buffer1.array(); + byte[] noDictionaryEndKey = buffer.array(); + DataFileFooter footer = + getFileFooter(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey); + list.add(footer); + i = i + 10; + } + } catch (Exception e) { + LOGGER.error(e); + } + return list; + } + + private List<DataFileFooter> getFileFooterListWithOnlyNoDictionaryKey() { + List<DataFileFooter> list = new ArrayList<DataFileFooter>(); + try { + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + int i = 1; + while (i < 1001) { + byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i }); + byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 }); + ByteBuffer buffer = ByteBuffer.allocate(2 + 4); + buffer.rewind(); + buffer.putShort((short) 1); + buffer.putInt(i); + buffer.array(); + byte[] noDictionaryStartKey = buffer.array(); + + ByteBuffer buffer1 = ByteBuffer.allocate(2 + 4); + buffer1.rewind(); + buffer1.putShort((short) 2); + buffer1.putInt(i + 10); + buffer1.array(); + byte[] noDictionaryEndKey = buffer.array(); + DataFileFooter footer = + getFileMatadataWithOnlyNoDictionaryKey(startKey, endKey, noDictionaryStartKey, + noDictionaryEndKey); + list.add(footer); + i = i + 10; + } + } catch (Exception e) { + LOGGER.error(e); + } + return list; + } + + private List<DataFileFooter> getFileFooterListWithOnlyDictionaryKey() { + List<DataFileFooter> list = new ArrayList<DataFileFooter>(); + try { + int[] dimensionBitLength = + CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); + KeyGenerator multiDimKeyVarLengthGenerator = + new MultiDimKeyVarLengthGenerator(dimensionBitLength); + int i = 1; + while (i < 1001) { + byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i }); + byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 }); + ByteBuffer buffer = ByteBuffer.allocate(1 + 4); + buffer.rewind(); + buffer.put((byte) 1); + buffer.putInt(i); + buffer.array(); + byte[] noDictionaryStartKey = buffer.array(); + + ByteBuffer buffer1 = ByteBuffer.allocate(1 + 4); + buffer1.rewind(); + buffer1.put((byte) 1); + buffer1.putInt(i + 10); + buffer1.array(); + byte[] noDictionaryEndKey = buffer.array(); + DataFileFooter footer = + getFileFooterWithOnlyDictionaryKey(startKey, endKey, noDictionaryStartKey, + noDictionaryEndKey); + list.add(footer); + i = i + 10; + } + } catch (Exception e) { + LOGGER.error(e); + } + return list; + } + + private DataFileFooter getFileFooter(byte[] startKey, byte[] endKey, + byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) { + DataFileFooter footer = new DataFileFooter(); + BlockletIndex index = new BlockletIndex(); + BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex(); + ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + noDictionaryStartKey.length); + buffer.putInt(startKey.length); + buffer.putInt(noDictionaryStartKey.length); + buffer.put(startKey); + buffer.put(noDictionaryStartKey); + buffer.rewind(); + btreeIndex.setStartKey(buffer.array()); + ByteBuffer buffer1 = ByteBuffer.allocate(4 + startKey.length + 4 + noDictionaryEndKey.length); + buffer1.putInt(endKey.length); + buffer1.putInt(noDictionaryEndKey.length); + buffer1.put(endKey); + buffer1.put(noDictionaryEndKey); + buffer1.rewind(); + btreeIndex.setEndKey(buffer1.array()); + BlockletMinMaxIndex minMax = new BlockletMinMaxIndex(); + minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey }); + minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey }); + index.setBtreeIndex(btreeIndex); + index.setMinMaxIndex(minMax); + footer.setBlockletIndex(index); + return footer; + } + + private DataFileFooter getFileMatadataWithOnlyNoDictionaryKey(byte[] startKey, byte[] endKey, + byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) { + DataFileFooter footer = new DataFileFooter(); + BlockletIndex index = new BlockletIndex(); + BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex(); + ByteBuffer buffer = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryStartKey.length); + buffer.putInt(0); + buffer.putInt(noDictionaryStartKey.length); + buffer.put(noDictionaryStartKey); + buffer.rewind(); + btreeIndex.setStartKey(buffer.array()); + ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length); + buffer1.putInt(0); + buffer1.putInt(noDictionaryEndKey.length); + buffer1.put(noDictionaryEndKey); + buffer1.rewind(); + btreeIndex.setEndKey(buffer1.array()); + BlockletMinMaxIndex minMax = new BlockletMinMaxIndex(); + minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey }); + minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey }); + index.setBtreeIndex(btreeIndex); + index.setMinMaxIndex(minMax); + footer.setBlockletIndex(index); + return footer; + } + + private DataFileFooter getFileFooterWithOnlyDictionaryKey(byte[] startKey, byte[] endKey, + byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) { + DataFileFooter footer = new DataFileFooter(); + BlockletIndex index = new BlockletIndex(); + BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex(); + ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + 0); + buffer.putInt(startKey.length); + buffer.putInt(0); + buffer.put(startKey); + buffer.rewind(); + btreeIndex.setStartKey(buffer.array()); + ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length); + buffer1.putInt(endKey.length); + buffer1.putInt(0); + buffer1.put(endKey); + buffer1.rewind(); + btreeIndex.setEndKey(buffer1.array()); + BlockletMinMaxIndex minMax = new BlockletMinMaxIndex(); + minMax.setMaxValues(new byte[][] { endKey }); + minMax.setMinValues(new byte[][] { startKey }); + index.setBtreeIndex(btreeIndex); + index.setMinMaxIndex(minMax); + footer.setBlockletIndex(index); + return footer; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java index 022ec6a..5f77b4d 100644 --- a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java @@ -19,8 +19,7 @@ package org.apache.carbondata.core.keygenerator.directdictionary; -import org.apache.carbondata.core.carbon.metadata.datatype.DataType; -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator; +import org.apache.carbondata.core.metadata.datatype.DataType; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java index 0dbac8f..0f9fbce 100644 --- a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java @@ -22,6 +22,8 @@ package org.apache.carbondata.core.load; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; + import org.junit.Before; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java new file mode 100644 index 0000000..680e519 --- /dev/null +++ b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.metadata; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; +import org.apache.carbondata.core.metadata.schema.table.TableInfo; +import org.apache.carbondata.core.metadata.schema.table.TableSchema; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.constants.CarbonCommonConstants; + +import mockit.Mock; +import mockit.MockUp; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class CarbonMetadataTest { + + private static CarbonMetadata carbonMetadata; + + private static String tableUniqueName; + + @BeforeClass public static void setUp() { + carbonMetadata = CarbonMetadata.getInstance(); + carbonMetadata.loadTableMetadata(getTableInfo(10000)); + tableUniqueName = "carbonTestDatabase_carbonTestTable"; + } + + @AfterClass public static void tearDown() { + carbonMetadata.removeTable(tableUniqueName); + carbonMetadata = null; + } + + @Test public void testOnlyOneInstanceIsCreatedofCarbonMetadata() { + CarbonMetadata carbonMetadata1 = CarbonMetadata.getInstance(); + assertEquals(carbonMetadata, carbonMetadata1); + } + + @Test public void testNumberOfTablesPresentInTheMetadata() { + int expectedResult = 1; + assertEquals(expectedResult, carbonMetadata.getNumberOfTables()); + } + + @Test public void testGetCarbonTableReturingProperCarbonTable() { + assertNotNull(carbonMetadata.getCarbonTable(tableUniqueName)); + } + + @Test public void testGetCarbonTableReturingNullWhenInvalidNameIsPassed() { + assertNull(carbonMetadata.getCarbonTable("notpresent")); + } + + @Test public void testGetCarbonTableReturingProperTableWithProperDimensionCount() { + int expectedResult = 1; + assertEquals(expectedResult, + carbonMetadata.getCarbonTable(tableUniqueName).getNumberOfDimensions("carbonTestTable")); + } + + @Test public void testGetCarbonTableReturingProperTableWithProperMeasureCount() { + int expectedResult = 1; + assertEquals(expectedResult, + carbonMetadata.getCarbonTable(tableUniqueName).getNumberOfMeasures("carbonTestTable")); + } + + @Test public void testGetCarbonTableReturingProperTableWithProperDatabaseName() { + String expectedResult = "carbonTestDatabase"; + assertEquals(expectedResult, carbonMetadata.getCarbonTable(tableUniqueName).getDatabaseName()); + } + + @Test public void testGetCarbonTableReturingProperTableWithProperFactTableName() { + String expectedResult = "carbonTestTable"; + assertEquals(expectedResult, carbonMetadata.getCarbonTable(tableUniqueName).getFactTableName()); + } + + @Test public void testGetCarbonTableReturingProperTableWithProperTableUniqueName() { + String expectedResult = "carbonTestDatabase_carbonTestTable"; + assertEquals(expectedResult, + carbonMetadata.getCarbonTable(tableUniqueName).getTableUniqueName()); + } + + @Test public void testloadMetadataTableWhenTableIsAlreadyExistsAndTimeStampIsChanged() { + long expectedLastUpdatedTime = 10000L; + assertEquals(expectedLastUpdatedTime, + carbonMetadata.getCarbonTable(tableUniqueName).getTableLastUpdatedTime()); + CarbonMetadata carbonMetadata1 = CarbonMetadata.getInstance(); + carbonMetadata1.loadTableMetadata(getTableInfo(100001)); + long expectedTableLastUpdatedTime = 100001L; + assertEquals(expectedTableLastUpdatedTime, + carbonMetadata.getCarbonTable(tableUniqueName).getTableLastUpdatedTime()); + long expectedNumberOfTables = 1; + assertEquals(expectedNumberOfTables, carbonMetadata.getNumberOfTables()); + } + + private static ColumnSchema getColumnarDimensionColumn() { + ColumnSchema dimColumn = new ColumnSchema(); + dimColumn.setColumnar(true); + dimColumn.setColumnName("IMEI"); + dimColumn.setColumnUniqueId(UUID.randomUUID().toString()); + dimColumn.setDataType(DataType.STRING); + dimColumn.setDimensionColumn(true); + List<Encoding> encodeList = + new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + encodeList.add(Encoding.DICTIONARY); + dimColumn.setEncodingList(encodeList); + dimColumn.setNumberOfChild(0); + return dimColumn; + } + + private static ColumnSchema getColumnarMeasureColumn() { + ColumnSchema dimColumn = new ColumnSchema(); + dimColumn.setColumnName("IMEI_COUNT"); + dimColumn.setColumnUniqueId(UUID.randomUUID().toString()); + dimColumn.setDataType(DataType.STRING); + return dimColumn; + } + + private static TableSchema getTableSchema() { + TableSchema tableSchema = new TableSchema(); + List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); + columnSchemaList.add(getColumnarDimensionColumn()); + columnSchemaList.add(getColumnarMeasureColumn()); + tableSchema.setListOfColumns(columnSchemaList); + tableSchema.setTableId(UUID.randomUUID().toString()); + tableSchema.setTableName("carbonTestTable"); + return tableSchema; + } + + private static TableInfo getTableInfo(long timeStamp) { + TableInfo info = new TableInfo(); + info.setDatabaseName("carbonTestDatabase"); + info.setLastUpdatedTime(timeStamp); + info.setTableUniqueName("carbonTestDatabase_carbonTestTable"); + info.setFactTable(getTableSchema()); + info.setStorePath("/test/store"); + return info; + } + + @Test public void testGetCarbonDimensionBasedOnColIdentifier() { + CarbonTable carbonTable = new CarbonTable(); + String columnIdentifier = "1"; + final List<CarbonDimension> carbonDimensions = new ArrayList(); + ColumnSchema colSchema1 = new ColumnSchema(); + ColumnSchema colSchema2 = new ColumnSchema(); + colSchema1.setColumnUniqueId("1"); + colSchema2.setColumnUniqueId("2"); + carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 2, 1)); + carbonDimensions.add(new CarbonDimension(colSchema2, 2, 2, 2, 2)); + new MockUp<CarbonTable>() { + @Mock public String getFactTableName() { + return "carbonTestTable"; + } + + @Mock public List<CarbonDimension> getDimensionByTableName(String tableName) { + return carbonDimensions; + } + }; + CarbonDimension expectedResult = carbonDimensions.get(0); + CarbonDimension actualResult = + carbonMetadata.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnIdentifier); + assertEquals(expectedResult, actualResult); + } + + @Test + public void testGetCarbonDimensionBasedOnColIdentifierWhenChildDimensionColumnEqualsColumnIdentifier() { + CarbonTable carbonTable = new CarbonTable(); + String columnIdentifier = "9"; + final List<CarbonDimension> carbonDimensions = new ArrayList(); + ColumnSchema colSchema1 = new ColumnSchema(); + ColumnSchema colSchema2 = new ColumnSchema(); + colSchema1.setColumnUniqueId("1"); + carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 2, 1)); + final List<CarbonDimension> carbonChildDimensions = new ArrayList(); + ColumnSchema colSchema3 = new ColumnSchema(); + colSchema3.setColumnUniqueId("9"); + colSchema2.setColumnUniqueId("2"); + carbonChildDimensions.add(new CarbonDimension(colSchema3, 1, 1, 2, 1)); + new MockUp<CarbonTable>() { + @Mock public String getFactTableName() { + return "carbonTestTable"; + } + + @Mock public List<CarbonDimension> getDimensionByTableName(String tableName) { + return carbonDimensions; + } + }; + + new MockUp<CarbonDimension>() { + @Mock public int numberOfChild() { + return 1; + } + + @Mock public List<CarbonDimension> getListOfChildDimensions() { + return carbonChildDimensions; + } + + }; + + CarbonDimension expectedResult = carbonChildDimensions.get(0); + CarbonDimension actualResult = + carbonMetadata.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnIdentifier); + assertEquals(expectedResult, actualResult); + } + + @Test public void testGetCarbonDimensionBasedOnColIdentifierNullCase() { + CarbonTable carbonTable = new CarbonTable(); + String columnIdentifier = "3"; + final List<CarbonDimension> carbonDimensions = new ArrayList(); + ColumnSchema colSchema1 = new ColumnSchema(); + colSchema1.setColumnUniqueId("1"); + colSchema1.setNumberOfChild(1); + CarbonDimension carbonDimension = new CarbonDimension(colSchema1, 1, 1, 2, 1); + carbonDimensions.add(carbonDimension); + final List<CarbonDimension> carbonChildDimensions = new ArrayList(); + ColumnSchema colSchema2 = new ColumnSchema(); + colSchema2.setColumnUniqueId("9"); + colSchema2.setNumberOfChild(0); + carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 2, 1)); + + new MockUp<CarbonTable>() { + @Mock public String getFactTableName() { + return "carbonTestTable"; + } + + @Mock public List<CarbonDimension> getDimensionByTableName(String tableName) { + return carbonDimensions; + } + }; + + new MockUp<CarbonDimension>() { + @Mock public List<CarbonDimension> getListOfChildDimensions() { + return carbonChildDimensions; + } + }; + + CarbonDimension result = + carbonMetadata.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnIdentifier); + assertNull(result); + } + +} \ No newline at end of file
