-Remove empty lines. -Implement tests within DataStoreTestBase and DataStoreTestUtil. -Fix size() implementation for MemStore, Accumulo, Solr, JCache and OrientDB.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/8d99c055 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/8d99c055 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/8d99c055 Branch: refs/heads/master Commit: 8d99c0554f4dcd620489eefd7963bce4d0a89dab Parents: eb49374 Author: dev <carlosr...@gmail.com> Authored: Tue May 1 00:49:33 2018 -0500 Committer: dev <carlosr...@gmail.com> Committed: Tue May 1 00:49:33 2018 -0500 ---------------------------------------------------------------------- .../gora/accumulo/query/AccumuloResult.java | 3 +- .../gora/accumulo/store/AccumuloStoreTest.java | 27 +++++++ .../gora/avro/query/DataFileAvroResult.java | 1 - .../org/apache/gora/memory/store/MemStore.java | 4 +- .../main/java/org/apache/gora/query/Result.java | 2 +- .../apache/gora/store/DataStoreTestBase.java | 48 +++++++++++ .../apache/gora/store/DataStoreTestUtil.java | 85 +++++++++++++++++++- .../gora/dynamodb/query/DynamoDBResult.java | 1 - .../gora/hbase/query/HBaseScannerResult.java | 1 - .../apache/gora/hbase/store/TestHBaseStore.java | 24 +++++- .../apache/gora/jcache/query/JCacheResult.java | 5 +- .../gora/mongodb/store/TestMongoStore.java | 14 +++- .../gora/orientdb/query/OrientDBResult.java | 5 +- .../org/apache/gora/solr/query/SolrResult.java | 4 +- 14 files changed, 209 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-accumulo/src/main/java/org/apache/gora/accumulo/query/AccumuloResult.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/query/AccumuloResult.java b/gora-accumulo/src/main/java/org/apache/gora/accumulo/query/AccumuloResult.java index 7c208f8..217a0c3 100644 --- a/gora-accumulo/src/main/java/org/apache/gora/accumulo/query/AccumuloResult.java +++ b/gora-accumulo/src/main/java/org/apache/gora/accumulo/query/AccumuloResult.java @@ -53,7 +53,7 @@ public class AccumuloResult<K,T extends PersistentBase> extends ResultBase<K,T> public AccumuloResult(DataStore<K,T> dataStore, Query<K,T> query, Scanner scanner) { super(dataStore, query); - if (this.limit > -1) { + if (this.limit > 0) { scanner.setBatchSize((int) this.limit); } iterator = new RowIterator(scanner.iterator()); @@ -98,5 +98,4 @@ public class AccumuloResult<K,T extends PersistentBase> extends ResultBase<K,T> public int size() { return (int) this.limit; } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AccumuloStoreTest.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AccumuloStoreTest.java b/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AccumuloStoreTest.java index f8b7526..54f5519 100644 --- a/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AccumuloStoreTest.java +++ b/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AccumuloStoreTest.java @@ -24,9 +24,13 @@ import org.apache.gora.examples.generated.WebPage; import org.apache.gora.store.DataStore; import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.DataStoreTestBase; +import static org.apache.gora.store.DataStoreTestBase.log; +import org.apache.gora.store.DataStoreTestUtil; +import static org.apache.gora.store.DataStoreTestUtil.testResultSize; import org.apache.hadoop.conf.Configuration; import org.junit.Before; import org.junit.Ignore; +import org.junit.Test; /** * Tests extending {@link org.apache.gora.store.DataStoreTestBase} @@ -57,4 +61,27 @@ public class AccumuloStoreTest extends DataStoreTestBase { @Override public void testDeleteByQueryFields() throws IOException { } + + @Test + @Ignore("Accumulo does not support Result#size() without limit set") + @Override + public void testResultSize() throws Exception { + } + + @Test + @Ignore("Accumulo does not support Result#size() without limit set") + @Override + public void testResultSizeStartKey() throws Exception { + } + + @Ignore("Accumulo does not support Result#size() without limit set") + @Override + public void testResultSizeEndKey() throws Exception { + } + + @Test + @Ignore("Accumulo does not support Result#size() without limit set") + @Override + public void testResultSizeKeyRange() throws Exception { + } } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-core/src/main/java/org/apache/gora/avro/query/DataFileAvroResult.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/avro/query/DataFileAvroResult.java b/gora-core/src/main/java/org/apache/gora/avro/query/DataFileAvroResult.java index 089f692..0adeb90 100644 --- a/gora-core/src/main/java/org/apache/gora/avro/query/DataFileAvroResult.java +++ b/gora-core/src/main/java/org/apache/gora/avro/query/DataFileAvroResult.java @@ -86,5 +86,4 @@ public class DataFileAvroResult<K, T extends PersistentBase> extends ResultBase< public int size() { return (int) (end - start); } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java b/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java index e1fe7f2..6f1b3e2 100644 --- a/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java +++ b/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java @@ -92,7 +92,9 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { @Override public int size() { - return map.navigableKeySet().size(); + int totalSize = map.navigableKeySet().size(); + int intLimit = (int)this.limit; + return intLimit > 0 && totalSize>intLimit ? intLimit : totalSize; } } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-core/src/main/java/org/apache/gora/query/Result.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/query/Result.java b/gora-core/src/main/java/org/apache/gora/query/Result.java index d1ead9a..6854d89 100644 --- a/gora-core/src/main/java/org/apache/gora/query/Result.java +++ b/gora-core/src/main/java/org/apache/gora/query/Result.java @@ -96,7 +96,7 @@ public interface Result<K, T extends Persistent> { void close() throws IOException; /** - * Returns the number items the result has. + * Returns the number of items the result has. * * @return the number of items */ http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java ---------------------------------------------------------------------- diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java index 2de6a41..625df39 100644 --- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java +++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java @@ -358,4 +358,52 @@ public abstract class DataStoreTestBase { log.info("test method: testGetPartitions"); DataStoreTestUtil.testGetPartitions(webPageStore); } + + @Test + public void testResultSize() throws Exception { + log.info("test method: testResultSize"); + DataStoreTestUtil.testResultSizeWebPages(webPageStore); + } + + @Test + public void testResultSizeStartKey() throws Exception { + log.info("test method: testResultSizeStartKey"); + DataStoreTestUtil.testResultSizeWebPagesStartKey(webPageStore); + } + + @Test + public void testResultSizeEndKey() throws Exception { + log.info("test method: testResultSizeEndKey"); + DataStoreTestUtil.testResultSizeWebPagesEndKey(webPageStore); + } + + @Test + public void testResultSizeKeyRange() throws Exception { + log.info("test method: testResultSizeKeyRange"); + DataStoreTestUtil.testResultSizeWebPagesKeyRange(webPageStore); + } + + @Test + public void testResultSizeWithLimit() throws Exception { + log.info("test method: testResultSizeWithLimit"); + DataStoreTestUtil.testResultSizeWebPagesWithLimit(webPageStore); + } + + @Test + public void testResultSizeStartKeyWithLimit() throws Exception { + log.info("test method: testResultSizeStartKeyWithLimit"); + DataStoreTestUtil.testResultSizeWebPagesStartKeyWithLimit(webPageStore); + } + + @Test + public void testResultSizeEndKeyWithLimit() throws Exception { + log.info("test method: testResultSizeEndKeyWithLimit"); + DataStoreTestUtil.testResultSizeWebPagesEndKeyWithLimit(webPageStore); + } + + @Test + public void testResultSizeKeyRangeWithLimit() throws Exception { + log.info("test method: testResultSizeKeyRangeWithLimit"); + DataStoreTestUtil.testResultSizeWebPagesKeyRangeWithLimit(webPageStore); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java ---------------------------------------------------------------------- diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java index b6e3d7c..5a38df4 100644 --- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java +++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java @@ -814,7 +814,7 @@ public class DataStoreTestUtil { if(setEndKeys) query.setEndKey(sortedUrls.get(j)); Result<String, WebPage> result = query.execute(); - + int r=0; while(result.next()) { WebPage page = result.get(); @@ -1209,4 +1209,87 @@ public class DataStoreTestUtil { return fieldNames; } + public static void testResultSize(DataStore<String, WebPage> store, + boolean setStartKeys, boolean setEndKeys, boolean setLimit) + throws Exception { + createWebPageData(store); + + //create sorted set of urls + List<String> sortedUrls = new ArrayList<>(); + Collections.addAll(sortedUrls, URLS); + Collections.sort(sortedUrls); + + //try all ranges + for(int i=0; i<sortedUrls.size(); i++) { + for(int j=i; j<sortedUrls.size(); j++) { + Query<String, WebPage> query = store.newQuery(); + if(setStartKeys) + query.setStartKey(sortedUrls.get(i)); + if(setEndKeys) + query.setEndKey(sortedUrls.get(j)); + int expectedLength = (setEndKeys ? j+1: sortedUrls.size()) - + (setStartKeys ? i: 0); + if (setLimit){ + //limit half of the expected results + int limit = expectedLength/2; + if (limit > 0){ + query.setLimit(limit); + } else { + continue; + } + } + Result<String, WebPage> result = query.execute(); + int size = result.size(); + int r=0; + while(result.next()) { + r++; + } + assertEquals(r, size); + if(!setEndKeys) + break; + } + if(!setStartKeys) + break; + } + } + + public static void testResultSizeWebPages(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, false, false, false); + } + + public static void testResultSizeWebPagesStartKey(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, true, false, false); + } + + public static void testResultSizeWebPagesEndKey(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, false, false, false); + } + + public static void testResultSizeWebPagesKeyRange(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, true, true, false); + } + + public static void testResultSizeWebPagesWithLimit(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, false, false, true); + } + + public static void testResultSizeWebPagesStartKeyWithLimit(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, true, false, true); + } + + public static void testResultSizeWebPagesEndKeyWithLimit(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, false, false, true); + } + + public static void testResultSizeWebPagesKeyRangeWithLimit(DataStore<String, WebPage> store) + throws Exception { + testResultSize(store, true, true, true); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/query/DynamoDBResult.java ---------------------------------------------------------------------- diff --git a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/query/DynamoDBResult.java b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/query/DynamoDBResult.java index c191880..61810e4 100644 --- a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/query/DynamoDBResult.java +++ b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/query/DynamoDBResult.java @@ -88,5 +88,4 @@ public class DynamoDBResult<K, T extends Persistent> extends ResultWSBase<K, T> public int size() { return dynamoDBResultSet.size(); } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-hbase/src/main/java/org/apache/gora/hbase/query/HBaseScannerResult.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/main/java/org/apache/gora/hbase/query/HBaseScannerResult.java b/gora-hbase/src/main/java/org/apache/gora/hbase/query/HBaseScannerResult.java index ce4ea51..72a64fa 100644 --- a/gora-hbase/src/main/java/org/apache/gora/hbase/query/HBaseScannerResult.java +++ b/gora-hbase/src/main/java/org/apache/gora/hbase/query/HBaseScannerResult.java @@ -74,5 +74,4 @@ public class HBaseScannerResult<K, T extends PersistentBase> extends HBaseResult public int size() { return (int) this.limit; } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java index c79588a..b4440be 100644 --- a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java +++ b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java @@ -245,5 +245,27 @@ public class TestHBaseStore extends DataStoreTestBase { exceptionalConf.set("gora.hbase.mapping.file","gora-hbase-mapping-mismatch.xml"); DataStoreFactory.createDataStore(HBaseStore.class, String.class, WebPage.class, exceptionalConf); } - + + @Test + @Ignore("HBase does not support Result#size() without limit set") + @Override + public void testResultSize() throws Exception { + } + + @Test + @Ignore("HBase does not support Result#size() without limit set") + @Override + public void testResultSizeStartKey() throws Exception { + } + + @Ignore("HBase does not support Result#size() without limit set") + @Override + public void testResultSizeEndKey() throws Exception { + } + + @Test + @Ignore("HBase does not support Result#size() without limit set") + @Override + public void testResultSizeKeyRange() throws Exception { + } } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-jcache/src/main/java/org/apache/gora/jcache/query/JCacheResult.java ---------------------------------------------------------------------- diff --git a/gora-jcache/src/main/java/org/apache/gora/jcache/query/JCacheResult.java b/gora-jcache/src/main/java/org/apache/gora/jcache/query/JCacheResult.java index 0eba8f1..6048e12 100644 --- a/gora-jcache/src/main/java/org/apache/gora/jcache/query/JCacheResult.java +++ b/gora-jcache/src/main/java/org/apache/gora/jcache/query/JCacheResult.java @@ -84,7 +84,8 @@ public class JCacheResult<K, T extends PersistentBase> extends ResultBase<K, T> @Override public int size() { - return cacheKeySet.size(); + int totalSize = cacheKeySet.size(); + int intLimit = (int)this.limit; + return intLimit > 0 && totalSize>intLimit ? intLimit : totalSize; } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java index 9deb906..824cc9a 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java @@ -65,7 +65,19 @@ public abstract class TestMongoStore extends DataStoreTestBase { public void testQueryKeyRange() throws Exception { // Skip until GORA-66 is fixed: need better semantic for end/start keys } - + + @Ignore("Skip until GORA-66 is fixed: need better semantic for end/start keys") + @Override + public void testResultSizeKeyRange() throws Exception { + // Skip until GORA-66 is fixed: need better semantic for end/start keys + } + + @Ignore("Skip until GORA-66 is fixed: need better semantic for end/start keys") + @Override + public void testResultSizeKeyRangeWithLimit() throws Exception { + // Skip until GORA-66 is fixed: need better semantic for end/start keys + } + @Ignore("MongoStore doesn't support 3 types union field yet") @Override public void testGet3UnionField() throws Exception { http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-orientdb/src/main/java/org/apache/gora/orientdb/query/OrientDBResult.java ---------------------------------------------------------------------- diff --git a/gora-orientdb/src/main/java/org/apache/gora/orientdb/query/OrientDBResult.java b/gora-orientdb/src/main/java/org/apache/gora/orientdb/query/OrientDBResult.java index c0bf7e5..3d6a3e1 100644 --- a/gora-orientdb/src/main/java/org/apache/gora/orientdb/query/OrientDBResult.java +++ b/gora-orientdb/src/main/java/org/apache/gora/orientdb/query/OrientDBResult.java @@ -102,7 +102,8 @@ public class OrientDBResult<K, T extends PersistentBase> extends ResultBase<K, T @Override public int size() { - return size; + int totalSize = size; + int intLimit = (int)this.limit; + return intLimit > 0 && totalSize>intLimit ? intLimit : totalSize; } - } http://git-wip-us.apache.org/repos/asf/gora/blob/8d99c055/gora-solr/src/main/java/org/apache/gora/solr/query/SolrResult.java ---------------------------------------------------------------------- diff --git a/gora-solr/src/main/java/org/apache/gora/solr/query/SolrResult.java b/gora-solr/src/main/java/org/apache/gora/solr/query/SolrResult.java index e4f3c69..8524c87 100644 --- a/gora-solr/src/main/java/org/apache/gora/solr/query/SolrResult.java +++ b/gora-solr/src/main/java/org/apache/gora/solr/query/SolrResult.java @@ -127,6 +127,8 @@ public class SolrResult<K, T extends PersistentBase> extends ResultBase<K, T> { @Override public int size() { - return list.size(); + int totalSize = list.size(); + int intLimit = (int)this.limit; + return intLimit > 0 && totalSize>intLimit ? intLimit : totalSize; } }