ColumnPaginationFilter doesn't return the requested number of columns ---------------------------------------------------------------------
Key: HBASE-5247 URL: https://issues.apache.org/jira/browse/HBASE-5247 Project: HBase Issue Type: Bug Components: client Affects Versions: 0.90.4 Environment: Hbase 0.90.4-cdh3u2 & standard apache download 0.90.4 Hadoop 0.20.2-cdh3u2 & standard apache download 0.20.205 Reporter: Jamal Priority: Minor The ColumnPaginationFilter doesn't return the correct number of columns when the limit is less than the total number of columns. I don't know much about the internals of Hbase, but it appears that the problem is more re-producible when a table is split accross at least 2 regions. I have created a unit test which can re-produce the issue which is below. Sorry, I didn't see a place to upload an attachment. To run it, just create a standard maven project with the following 2 files: pom.xml: <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>org.sandbox</groupId> <artifactId>pagination</artifactId> <version>0.0.1-SNAPSHOT</version> <properties> <spring.version>3.0.6.RELEASE</spring.version> </properties> <dependencies> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-context</artifactId> <version>${spring.version}</version> <exclusions> <exclusion> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase</artifactId> <version>0.90.4</version> </dependency> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-test</artifactId> <version>${spring.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.8.2</version> <scope>test</scope> </dependency> </dependencies> <repositories> <repository> <id>com.springsource.repository.maven.release</id> <url>http://maven.springframework.org/release/</url> <snapshots> <enabled>false</enabled> </snapshots> </repository> </repositories> </project> PaginationTest.java: import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.ColumnPaginationFilter; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @RunWith(SpringJUnit4ClassRunner.class) @TestExecutionListeners({}) public class PaginationTest { private byte[] TABLE_NAME = Bytes.toBytes("pagination-test-table"); private Configuration conf = HBaseConfiguration.create(); private HTable table; private HBaseAdmin admin; private byte[] COLUMN_FAMILY = Bytes.toBytes("stuff"); byte[] rowKey = Bytes.toBytes("row5"); private Logger logger = LoggerFactory.getLogger(getClass()); private int rowCount = 5000, columnCount = 12; @Before public void setup() throws IOException, InterruptedException { admin = new HBaseAdmin(conf); if (!admin.isTableAvailable(TABLE_NAME)) { HTableDescriptor desc = new HTableDescriptor(TABLE_NAME); HColumnDescriptor columnDesc = new HColumnDescriptor(COLUMN_FAMILY); desc.addFamily(columnDesc); admin.createTable(desc); } table = new HTable(conf, TABLE_NAME); for (int z = 0; z < 3; z++) { List<Put> puts = new ArrayList<Put>(columnCount); for (int j = 0; j < rowCount; j++) { byte[] rowKey = Bytes.toBytes("row" + j); Put put = new Put(rowKey); for (int i = 0; i < columnCount; i++) { put.add(COLUMN_FAMILY, Bytes.toBytes(DigestUtils.md5Hex("something:" + i)), Bytes.toBytes(i)); } puts.add(put); } table.put(puts); table.flushCommits(); admin.split(TABLE_NAME); // Wait for split to "finish" Thread.sleep(30000); // Wait for flush to "finish" admin.flush(TABLE_NAME); } } @After public void cleanup() throws IOException { table.close(); admin.disableTable(TABLE_NAME); admin.deleteTable(TABLE_NAME); } @Test public void runConditions() throws IOException { showAllColumns(); scanFirst5Columns(); scanNextFileColumns(); getFirstFiveColumns(); } public void showAllColumns() throws IOException { logger.info("Current contents of row:" + Bytes.toString(rowKey)); Get get = new Get(rowKey); get.addFamily(COLUMN_FAMILY); Result result = table.get(get); for (KeyValue kv : result.list()) { // logger.info("kv:" + kv); logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:" + Bytes.toInt(kv.getValue())); } // Confirm the # of columns is as expected Assert.assertTrue(result.list().size() == columnCount); } public void scanFirst5Columns() throws IOException { // Try Scanning for 5 columns logger.info("Scan the first 5 columns of row:" + Bytes.toString(rowKey)); Scan scan = new Scan(rowKey, rowKey); Filter filter = new ColumnPaginationFilter(5, 0); scan.setFilter(filter); ResultScanner rs = table.getScanner(scan); try { for (Result r = rs.next(); r != null; r = rs.next()) { for (KeyValue kv : r.list()) { // logger.info("kv:" + kv); logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:" + Bytes.toInt(kv.getValue())); } // Confirm the # of columns is as expected Assert.assertTrue(r.list().size() == 5); } } finally { rs.close(); } } public void scanNextFileColumns() throws IOException { // Try Scanning for next 5 columns logger.info("Scan the next 5 columns of row:" + Bytes.toString(rowKey)); Scan scan = new Scan(rowKey, rowKey); Filter filter = new ColumnPaginationFilter(5, 5); scan.setFilter(filter); ResultScanner rs = table.getScanner(scan); try { for (Result r = rs.next(); r != null; r = rs.next()) { for (KeyValue kv : r.list()) { // logger.info("kv:" + kv); logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:" + Bytes.toInt(kv.getValue())); } // Confirm the # of columns is as expected Assert.assertTrue(r.list().size() == 5); } } finally { rs.close(); } } public void getFirstFiveColumns() throws IOException { // Try Getting 5 columns logger.info("Get the first 5 columns of row:" + Bytes.toString(rowKey)); Get get = new Get(rowKey); get.addFamily(COLUMN_FAMILY); Filter filter = new ColumnPaginationFilter(5, 0); get.setFilter(filter); Result firstFiveResult = table.get(get); for (KeyValue kv : firstFiveResult.list()) { // logger.info("kv:" + kv); logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:" + Bytes.toInt(kv.getValue())); } // Confirm the # of columns is as expected Assert.assertTrue(firstFiveResult.list().size() == 5); } } -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa For more information on JIRA, see: http://www.atlassian.com/software/jira