ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.
Signed-off-by: Josh Elser <els...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae1682d4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae1682d4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae1682d4 Branch: refs/heads/1.6.1-SNAPSHOT Commit: ae1682d4ae7a3031598748f676c90829d0b2a978 Parents: 73e0f9c Author: dallaybatta <vicky...@gmail.com> Authored: Mon Jun 9 11:29:54 2014 +0530 Committer: Josh Elser <els...@apache.org> Committed: Mon Jun 9 10:27:51 2014 -0400 ---------------------------------------------------------------------- .../org/apache/accumulo/core/Constants.java | 2 ++ .../client/mapred/AccumuloFileOutputFormat.java | 3 +- .../mapreduce/lib/impl/ConfiguratorBase.java | 37 +++++++++++++++++++- .../mapred/AccumuloFileOutputFormatTest.java | 3 ++ .../lib/impl/ConfiguratorBaseTest.java | 8 +++++ 5 files changed, 51 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/Constants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index e0e88eb..c9a51a9 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -115,4 +115,6 @@ public class Constants { public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"}; public static final String HDFS_TABLES_DIR = "/tables"; + + public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java index 8a1d6df..cfaaa58 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java @@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mapred; import java.io.IOException; import java.util.Arrays; +import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase; import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -145,7 +146,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> { final String extension = acuConf.get(Property.TABLE_FILE_TYPE); final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension); - final LRUMap validVisibilities = new LRUMap(1000); + final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf)); return new RecordWriter<Key,Value>() { FileSKVWriter out = null; http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java index e87d43b..cf1f0ea 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java @@ -82,7 +82,8 @@ public class ConfiguratorBase { * @since 1.6.0 */ public static enum GeneralOpts { - LOG_LEVEL + LOG_LEVEL, + VISIBILITY_CACHE_SIZE } /** @@ -100,6 +101,17 @@ public class ConfiguratorBase { } /** + * Provides a configuration key for a given feature enum. + * + * @param e + * the enum used to provide the unique part of the configuration key + * @return the configuration key + */ + protected static String enumToConfKey(Enum<?> e) { + return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase()); + } + + /** * Sets the connector information needed to communicate with Accumulo in this job. * * <p> @@ -369,4 +381,27 @@ public class ConfiguratorBase { return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt())); } + /** + * Sets the valid visibility count for this job. + * + * @param conf + * the Hadoop configuration object to configure + * @param visibilityCacheSize + * the LRU cache size + */ + public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) { + conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize); + } + + /** + * Gets the valid visibility count for this job. + * + * @param conf + * the Hadoop configuration object to configure + * @return the valid visibility count + */ + public static int getVisibilityCacheSize(Configuration conf) { + return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE); + } + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java index aad544b..3a4d641 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase; import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; @@ -53,6 +54,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; public class AccumuloFileOutputFormatTest { + private static final int JOB_VISIBILITY_CACHE_SIZE = 3000; private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName(); private static final String INSTANCE_NAME = PREFIX + "_mapred_instance"; private static final String BAD_TABLE = PREFIX + "_mapred_bad_table"; @@ -145,6 +147,7 @@ public class AccumuloFileOutputFormatTest { JobConf job = new JobConf(getConf()); job.setJarByClass(this.getClass()); + ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE); job.setInputFormat(AccumuloInputFormat.class); http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java index d5ebb22..7c1f98b 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; @@ -126,4 +127,11 @@ public class ConfiguratorBaseTest { assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf)); } + @Test + public void testSetVisibiltyCacheSize(){ + Configuration conf = new Configuration(); + assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf)); + ConfiguratorBase.setVisibilityCacheSize(conf, 2000); + assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf)); + } }