ColumnPaginationFilter doesn't return the requested number of columns
---------------------------------------------------------------------
Key: HBASE-5247
URL: https://issues.apache.org/jira/browse/HBASE-5247
Project: HBase
Issue Type: Bug
Components: client
Affects Versions: 0.90.4
Environment: Hbase 0.90.4-cdh3u2 & standard apache download 0.90.4
Hadoop 0.20.2-cdh3u2 & standard apache download 0.20.205
Reporter: Jamal
Priority: Minor
The ColumnPaginationFilter doesn't return the correct number of columns when
the limit is less than the total number of columns. I don't know much about
the internals of Hbase, but it appears that the problem is more re-producible
when a table is split accross at least 2 regions. I have created a unit test
which can re-produce the issue which is below. Sorry, I didn't see a place to
upload an attachment. To run it, just create a standard maven project with the
following 2 files:
pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<modelVersion>4.0.0</modelVersion>
<groupId>org.sandbox</groupId>
<artifactId>pagination</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<spring.version>3.0.6.RELEASE</spring.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>0.90.4</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>com.springsource.repository.maven.release</id>
<url>http://maven.springframework.org/release/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
</project>
PaginationTest.java:
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@TestExecutionListeners({})
public class PaginationTest {
private byte[] TABLE_NAME = Bytes.toBytes("pagination-test-table");
private Configuration conf = HBaseConfiguration.create();
private HTable table;
private HBaseAdmin admin;
private byte[] COLUMN_FAMILY = Bytes.toBytes("stuff");
byte[] rowKey = Bytes.toBytes("row5");
private Logger logger = LoggerFactory.getLogger(getClass());
private int rowCount = 5000, columnCount = 12;
@Before
public void setup() throws IOException, InterruptedException {
admin = new HBaseAdmin(conf);
if (!admin.isTableAvailable(TABLE_NAME)) {
HTableDescriptor desc = new
HTableDescriptor(TABLE_NAME);
HColumnDescriptor columnDesc = new
HColumnDescriptor(COLUMN_FAMILY);
desc.addFamily(columnDesc);
admin.createTable(desc);
}
table = new HTable(conf, TABLE_NAME);
for (int z = 0; z < 3; z++) {
List<Put> puts = new ArrayList<Put>(columnCount);
for (int j = 0; j < rowCount; j++) {
byte[] rowKey = Bytes.toBytes("row" + j);
Put put = new Put(rowKey);
for (int i = 0; i < columnCount; i++) {
put.add(COLUMN_FAMILY,
Bytes.toBytes(DigestUtils.md5Hex("something:" + i)),
Bytes.toBytes(i));
}
puts.add(put);
}
table.put(puts);
table.flushCommits();
admin.split(TABLE_NAME);
// Wait for split to "finish"
Thread.sleep(30000);
// Wait for flush to "finish"
admin.flush(TABLE_NAME);
}
}
@After
public void cleanup() throws IOException {
table.close();
admin.disableTable(TABLE_NAME);
admin.deleteTable(TABLE_NAME);
}
@Test
public void runConditions() throws IOException {
showAllColumns();
scanFirst5Columns();
scanNextFileColumns();
getFirstFiveColumns();
}
public void showAllColumns() throws IOException {
logger.info("Current contents of row:" +
Bytes.toString(rowKey));
Get get = new Get(rowKey);
get.addFamily(COLUMN_FAMILY);
Result result = table.get(get);
for (KeyValue kv : result.list()) {
// logger.info("kv:" + kv);
logger.info("key:" + Bytes.toString(kv.getQualifier())
+ " value:"
+ Bytes.toInt(kv.getValue()));
}
// Confirm the # of columns is as expected
Assert.assertTrue(result.list().size() == columnCount);
}
public void scanFirst5Columns() throws IOException {
// Try Scanning for 5 columns
logger.info("Scan the first 5 columns of row:" +
Bytes.toString(rowKey));
Scan scan = new Scan(rowKey, rowKey);
Filter filter = new ColumnPaginationFilter(5, 0);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
for (KeyValue kv : r.list()) {
// logger.info("kv:" + kv);
logger.info("key:" +
Bytes.toString(kv.getQualifier())
+ " value:" +
Bytes.toInt(kv.getValue()));
}
// Confirm the # of columns is as expected
Assert.assertTrue(r.list().size() == 5);
}
} finally {
rs.close();
}
}
public void scanNextFileColumns() throws IOException {
// Try Scanning for next 5 columns
logger.info("Scan the next 5 columns of row:" +
Bytes.toString(rowKey));
Scan scan = new Scan(rowKey, rowKey);
Filter filter = new ColumnPaginationFilter(5, 5);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
for (KeyValue kv : r.list()) {
// logger.info("kv:" + kv);
logger.info("key:" +
Bytes.toString(kv.getQualifier())
+ " value:" +
Bytes.toInt(kv.getValue()));
}
// Confirm the # of columns is as expected
Assert.assertTrue(r.list().size() == 5);
}
} finally {
rs.close();
}
}
public void getFirstFiveColumns() throws IOException {
// Try Getting 5 columns
logger.info("Get the first 5 columns of row:" +
Bytes.toString(rowKey));
Get get = new Get(rowKey);
get.addFamily(COLUMN_FAMILY);
Filter filter = new ColumnPaginationFilter(5, 0);
get.setFilter(filter);
Result firstFiveResult = table.get(get);
for (KeyValue kv : firstFiveResult.list()) {
// logger.info("kv:" + kv);
logger.info("key:" + Bytes.toString(kv.getQualifier())
+ " value:"
+ Bytes.toInt(kv.getValue()));
}
// Confirm the # of columns is as expected
Assert.assertTrue(firstFiveResult.list().size() == 5);
}
}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators:
https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
