This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 64995a6241 Modified VisibilityFilter to handle multiple Authorizations
objects (#6402)
64995a6241 is described below
commit 64995a62416aaefbfa27a7f25fc3db7b3fc2be85
Author: Dave Marion <[email protected]>
AuthorDate: Mon Jul 20 10:16:10 2026 -0400
Modified VisibilityFilter to handle multiple Authorizations objects (#6402)
Accumulo Access provides an AccessEvaluator that accepts
a Collection of Sets of Strings (effectively
Collection<Authorizations>). The existing user
VisibilityFilter only accepts a single Authorizations
object. This modifies the user VisibilityFilter to accept
one or more Authorizations.
Co-authored-by: Christopher Tubbs <[email protected]>
---
.../core/clientImpl/access/BytesAccess.java | 15 ++++++
.../core/iterators/user/VisibilityFilter.java | 53 ++++++++++++++++++----
.../core/iterators/user/VisibilityFilterTest.java | 35 +++++++++++++-
.../accumulo/test/functional/VisibilityIT.java | 45 ++++++++++++++++++
4 files changed, 137 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java
index 4c65621b0f..21ce1698e0 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java
@@ -20,6 +20,8 @@ package org.apache.accumulo.core.clientImpl.access;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -93,6 +95,19 @@ public class BytesAccess {
}
}
+ public static BytesEvaluator newEvaluator(Collection<Authorizations>
authsSet) {
+ Collection<Set<String>> convertedAuths = new ArrayList<>();
+ for (Authorizations auths : authsSet) {
+ List<byte[]> bytesAuths = auths.getAuthorizations();
+ Set<String> stringAuths = new HashSet<>(bytesAuths.size());
+ for (var auth : bytesAuths) {
+ stringAuths.add(new String(auth, ISO_8859_1));
+ }
+ convertedAuths.add(stringAuths);
+ }
+ return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths));
+ }
+
public static BytesEvaluator newEvaluator(Authorizations auths) {
List<byte[]> bytesAuths = auths.getAuthorizations();
Set<String> stringAuths = new HashSet<>(bytesAuths.size());
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
index 92e5079ab6..9de259d646 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
@@ -21,6 +21,8 @@ package org.apache.accumulo.core.iterators.user;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Map;
import org.apache.accumulo.access.InvalidAccessExpressionException;
@@ -44,12 +46,14 @@ import org.slf4j.LoggerFactory;
*/
public class VisibilityFilter extends Filter implements OptionDescriber {
+ private static final Logger log =
LoggerFactory.getLogger(VisibilityFilter.class);
+ private static final BytesAccess.BytesEvaluator EMPTY_EVALUATOR =
+ BytesAccess.newEvaluator(Authorizations.EMPTY);
+
private BytesAccess.BytesEvaluator accessEvaluator;
protected Map<ByteSequence,Boolean> cache;
private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]);
- private static final Logger log =
LoggerFactory.getLogger(VisibilityFilter.class);
-
private static final String AUTHS = "auths";
private static final String FILTER_INVALID_ONLY = "filterInvalid";
@@ -64,10 +68,31 @@ public class VisibilityFilter extends Filter implements
OptionDescriber {
if (!filterInvalid) {
String auths = options.get(AUTHS);
- Authorizations authObj = auths == null || auths.isEmpty() ? new
Authorizations()
- : new Authorizations(auths.getBytes(UTF_8));
-
- this.accessEvaluator = BytesAccess.newEvaluator(authObj);
+ if (auths == null || auths.isEmpty()) {
+ this.accessEvaluator = EMPTY_EVALUATOR;
+ } else if (!auths.startsWith(Authorizations.HEADER)) {
+ // the old serialization format does not support multiple auth sets,
so treat the whole
+ // thing as one set
+ this.accessEvaluator = BytesAccess.newEvaluator(new
Authorizations(auths.getBytes(UTF_8)));
+ } else {
+ String[] authParts = auths.split(Authorizations.HEADER);
+ if (authParts.length == 0) {
+ this.accessEvaluator = EMPTY_EVALUATOR;
+ } else {
+ Collection<Authorizations> authSet = new ArrayList<>();
+ for (int i = 0; i < authParts.length; i++) {
+ String part = authParts[i];
+ if (part.isEmpty()) {
+ continue;
+ } else {
+ // split removes the HEADER, need to add it back
+ String serializedAuthString = Authorizations.HEADER +
authParts[i];
+ authSet.add(new
Authorizations(serializedAuthString.getBytes(UTF_8)));
+ }
+ }
+ this.accessEvaluator = BytesAccess.newEvaluator(authSet);
+ }
+ }
}
this.cache = new LRUMap<>(1000);
}
@@ -132,13 +157,15 @@ public class VisibilityFilter extends Filter implements
OptionDescriber {
IteratorOptions io = super.describeOptions();
io.setName("visibilityFilter");
io.setDescription("The VisibilityFilter allows you to filter for key/value"
- + " pairs by a set of authorizations or filter invalid labels from
corrupt files.");
+ + " pairs by a set(s) of authorizations or filter invalid labels from
corrupt files.");
io.addNamedOption(FILTER_INVALID_ONLY,
"if 'true', the iterator is instructed to ignore the authorizations
and"
+ " only filter invalid visibility labels (default: false)");
- io.addNamedOption(AUTHS,
- "the serialized set of authorizations to filter against (default:
empty"
- + " string, accepts only entries visible by all)");
+ io.addNamedOption(AUTHS, "The concatenated serialized set(s) of
Authorizations used to "
+ + "filter out Key-Value pairs. Default value is an empty string which
allows all "
+ + "pairs to pass through the filter. The value can be set
programmatically using "
+ + "the setAuthorizations method or manually using the output of the
Authorizations.serialize() "
+ + "method (concatenated with no separator if multiple
Authorizations).");
return io;
}
@@ -146,6 +173,12 @@ public class VisibilityFilter extends Filter implements
OptionDescriber {
setting.addOption(AUTHS, auths.serialize());
}
+ public static void setAuthorizations(IteratorSetting setting,
Collection<Authorizations> auths) {
+ StringBuilder builder = new StringBuilder();
+ auths.forEach(a -> builder.append(a.serialize()));
+ setting.addOption(AUTHS, builder.toString());
+ }
+
public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean
featureEnabled) {
setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled));
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java
b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java
index 075e2bef33..db42441117 100644
---
a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
@@ -155,6 +156,22 @@ public class VisibilityFilterTest {
verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
}
+ @Test
+ public void testMulitAllowAuthorizedLabelsOnly() throws IOException {
+ IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
+ VisibilityFilter.setAuthorizations(is,
+ List.of(new Authorizations("abc"), new Authorizations("def")));
+
+ TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
+ verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1);
+
+ source = createSourceWithHiddenData(30, 500);
+ verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30);
+
+ source = createSourceWithHiddenData(1000, 500);
+ verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
+ }
+
@Test
public void testAllowUnauthorizedLabelsOnly() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
@@ -171,6 +188,23 @@ public class VisibilityFilterTest {
verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
}
+ @Test
+ public void testMultiAllowUnauthorizedLabelsOnly() throws IOException {
+ IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
+ VisibilityFilter.setNegate(is, true);
+ VisibilityFilter.setAuthorizations(is,
+ List.of(new Authorizations("abc"), new Authorizations("def")));
+
+ TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
+ verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2);
+
+ source = createSourceWithHiddenData(30, 500);
+ verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
+
+ source = createSourceWithHiddenData(1000, 500);
+ verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
+ }
+
@Test
public void testNoLabels() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
@@ -255,5 +289,4 @@ public class VisibilityFilterTest {
Key k = new Key("row", "cf", "cq", "abc");
assertTrue(copyFilter.accept(k, new Value()));
}
-
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java
index 40fe8b4c87..c6fc0c57e9 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@ -37,12 +37,14 @@ import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VisibilityFilter;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.ByteArraySet;
@@ -92,6 +94,7 @@ public class VisibilityIT extends AccumuloClusterHarness {
insertData(c, table);
queryData(c, table);
+ queryDataMultiAuth(c, table);
deleteData(c, table);
insertDefaultData(c, table2);
@@ -222,6 +225,48 @@ public class VisibilityIT extends AccumuloClusterHarness {
queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(),
expected);
}
+ /**
+ * Configures Scanners with the users default authorizations, then it adds a
+ * MultiAuthVisibilityFilter with different sets of Authorizations
+ */
+ private void queryDataMultiAuth(AccumuloClient c, String tableName) throws
Exception {
+
+ c.securityOperations().changeUserAuthorizations(getAdminPrincipal(),
+ new Authorizations("A", "B", "FOO", "L", "M", "Z"));
+
+ Authorizations userAuths =
c.securityOperations().getUserAuthorizations(c.whoami());
+
+ Set<String> expectedUserAuths =
+ Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
"v11", "v12", "v13");
+ try (Scanner scanner = c.createScanner(tableName, userAuths);
+ BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {
+ verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {}));
+
+ bs.setRanges(Collections.singleton(new Range()));
+ verify(bs.iterator(), expectedUserAuths.toArray(new String[] {}));
+ }
+
+ Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M");
+ Authorizations entity2 = new Authorizations("B", "FOO", "Z");
+ // should only see entries with no column visibility, B and/or FOO
+ Set<String> expectedAuths = Set.of("v1", "v3", "v11");
+
+ IteratorSetting is = new IteratorSetting(100, "userAuths",
VisibilityFilter.class);
+ VisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2));
+
+ try (Scanner scanner = c.createScanner(tableName, userAuths);
+ BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {
+
+ scanner.addScanIterator(is);
+ verify(scanner.iterator(), expectedAuths.toArray(new String[] {}));
+
+ bs.setRanges(Collections.singleton(new Range()));
+ bs.addScanIterator(is);
+ verify(bs.iterator(), expectedAuths.toArray(new String[] {}));
+ }
+
+ }
+
private void queryData(AccumuloClient c, String tableName, Set<String>
allAuths,
Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws
Exception {
