Merge branch 'ACCUMULO-4636' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/116da380 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/116da380 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/116da380 Branch: refs/heads/IGNITE Commit: 116da380a36e5088c5bf9be70f9b88850089ac23 Parents: 54c82dc 2ff2678 Author: Mike Miller <mmil...@apache.org> Authored: Thu May 18 14:29:13 2017 -0400 Committer: Mike Miller <mmil...@apache.org> Committed: Thu May 18 14:29:13 2017 -0400 ---------------------------------------------------------------------- .../core/client/mock/MockScannerBase.java | 4 +- .../apache/accumulo/core/iterators/Filter.java | 5 +- .../core/iterators/FirstEntryInRowIterator.java | 11 ++-- .../accumulo/core/iterators/IteratorUtil.java | 2 +- .../system/ColumnFamilySkippingIterator.java | 17 ++--- .../iterators/system/ColumnQualifierFilter.java | 68 +++++++++++--------- .../core/iterators/system/DeletingIterator.java | 28 ++++---- .../core/iterators/system/VisibilityFilter.java | 51 ++++++++++----- .../core/iterators/user/VersioningIterator.java | 5 +- .../core/iterators/user/VisibilityFilter.java | 44 +++++++++---- .../core/iterators/system/ColumnFilterTest.java | 58 ++++++++++++----- .../iterators/system/VisibilityFilterTest.java | 20 +++++- .../core/iterators/user/FilterTest.java | 12 ++-- .../user/TransformingIteratorTest.java | 2 +- .../problems/ProblemReportingIterator.java | 4 +- .../apache/accumulo/tserver/InMemoryMap.java | 7 -- .../performance/scan/CollectTabletStats.java | 4 +- 17 files changed, 218 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java index 981404c,aee9771..a830da3 --- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java @@@ -391,12 -372,4 +391,12 @@@ public class IteratorUtil } return toIteratorSettings(ic); } + + public static SortedKeyValueIterator<Key,Value> setupSystemScanIterators(SortedKeyValueIterator<Key,Value> source, Set<Column> cols, Authorizations auths, + byte[] defaultVisibility) throws IOException { + DeletingIterator delIter = new DeletingIterator(source, false); + ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter); + ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, cols); - return new VisibilityFilter(colFilter, auths, defaultVisibility); ++ return VisibilityFilter.wrap(colFilter, auths, defaultVisibility); + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java index d02b7f2,ca10c65..7ff08ec --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java @@@ -56,53 -60,41 +56,53 @@@ import org.junit.Assert import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + public class TransformingIteratorTest { - private static final String TABLE_NAME = "test_table"; + private static Authorizations authorizations = new Authorizations("vis0", "vis1", "vis2", "vis3", "vis4"); - private Connector connector; - private Scanner scanner; + private static final Map<String,String> EMPTY_OPTS = ImmutableMap.of(); + private TransformingIterator titer; - @Before - public void setUpMockAccumulo() throws Exception { - MockInstance instance = new MockInstance("test"); - connector = instance.getConnector("user", new PasswordToken("password")); - connector.securityOperations().changeUserAuthorizations("user", authorizations); + private TreeMap<Key,Value> data = new TreeMap<>(); - if (connector.tableOperations().exists(TABLE_NAME)) - connector.tableOperations().delete(TABLE_NAME); - connector.tableOperations().create(TABLE_NAME); - BatchWriterConfig bwCfg = new BatchWriterConfig(); - bwCfg.setMaxWriteThreads(1); + @Before + public void createData() throws Exception { + data.clear(); + generateRow(data, "row1"); + generateRow(data, "row2"); + generateRow(data, "row3"); + } - BatchWriter bw = connector.createBatchWriter(TABLE_NAME, bwCfg); - bw.addMutation(createDefaultMutation("row1")); - bw.addMutation(createDefaultMutation("row2")); - bw.addMutation(createDefaultMutation("row3")); + private void setUpTransformIterator(Class<? extends TransformingIterator> clazz) throws IOException { + setUpTransformIterator(clazz, true); + } - bw.flush(); - bw.close(); + private void setUpTransformIterator(Class<? extends TransformingIterator> clazz, boolean setupAuths) throws IOException { + SortedMapIterator source = new SortedMapIterator(data); + ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(source); - VisibilityFilter visFilter = new VisibilityFilter(cfsi, authorizations, new byte[0]); ++ SortedKeyValueIterator<Key,Value> visFilter = VisibilityFilter.wrap(cfsi, authorizations, new byte[0]); + ReuseIterator reuserIter = new ReuseIterator(); + reuserIter.init(visFilter, EMPTY_OPTS, null); + try { + titer = clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } - scanner = connector.createScanner(TABLE_NAME, authorizations); - scanner.addScanIterator(new IteratorSetting(20, ReuseIterator.class)); - } + IteratorEnvironment iterEnv = EasyMock.createMock(IteratorEnvironment.class); + EasyMock.expect(iterEnv.getIteratorScope()).andReturn(IteratorScope.scan).anyTimes(); + EasyMock.replay(iterEnv); - private void setUpTransformIterator(Class<? extends TransformingIterator> clazz) { - IteratorSetting cfg = new IteratorSetting(21, clazz); - cfg.setName("keyTransformIter"); - TransformingIterator.setAuthorizations(cfg, new Authorizations("vis0", "vis1", "vis2", "vis3")); - scanner.addScanIterator(cfg); + Map<String,String> opts; + if (setupAuths) { + IteratorSetting cfg = new IteratorSetting(21, clazz); + TransformingIterator.setAuthorizations(cfg, new Authorizations("vis0", "vis1", "vis2", "vis3")); + opts = cfg.getOptions(); + } else { + opts = ImmutableMap.of(); + } + titer.init(reuserIter, opts, iterEnv); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java index 83b4615,e419780..7e1676a --- a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java @@@ -31,17 -31,17 +31,17 @@@ import org.apache.accumulo.core.iterato import org.apache.accumulo.server.AccumuloServerContext; public class ProblemReportingIterator implements InterruptibleIterator { - private SortedKeyValueIterator<Key,Value> source; + private final SortedKeyValueIterator<Key,Value> source; private boolean sawError = false; - private boolean continueOnError; + private final boolean continueOnError; private String resource; - private String table; + private String tableId; private final AccumuloServerContext context; - public ProblemReportingIterator(AccumuloServerContext context, String table, String resource, boolean continueOnError, + public ProblemReportingIterator(AccumuloServerContext context, String tableId, String resource, boolean continueOnError, SortedKeyValueIterator<Key,Value> source) { this.context = context; - this.table = table; + this.tableId = tableId; this.resource = resource; this.continueOnError = continueOnError; this.source = source; http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java ----------------------------------------------------------------------
