SAMZA-589; sanitize sensitive data beore displaying configs to UI
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/cad2c828 Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/cad2c828 Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/cad2c828 Branch: refs/heads/samza-sql Commit: cad2c828692244d110e9e3a19c39b257c3040b4c Parents: 8170149 Author: Tommy Becker <[email protected]> Authored: Mon Mar 16 14:53:32 2015 -0700 Committer: Chris Riccomini <[email protected]> Committed: Mon Mar 16 14:53:32 2015 -0700 ---------------------------------------------------------------------- build.gradle | 1 + .../versioned/jobs/configuration.md | 5 ++++- .../java/org/apache/samza/config/Config.java | 5 +++++ .../java/org/apache/samza/config/MapConfig.java | 21 +++++++++++++++++--- .../org/apache/samza/config/TestConfig.java | 21 +++++++++++++++++++- .../webapp/ApplicationMasterRestServlet.scala | 2 +- .../webapp/ApplicationMasterWebServlet.scala | 2 +- 7 files changed, 50 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 113da2a..97de3a2 100644 --- a/build.gradle +++ b/build.gradle @@ -56,6 +56,7 @@ rat { '**/.git/**', '**/.gitignore', '**/.gradle/**', + '**/.idea/**', '**/.project', '**/.settings/**', '**/.DS_Store/**', http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/docs/learn/documentation/versioned/jobs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/learn/documentation/versioned/jobs/configuration.md b/docs/learn/documentation/versioned/jobs/configuration.md index e094f60..c0ebb69 100644 --- a/docs/learn/documentation/versioned/jobs/configuration.md +++ b/docs/learn/documentation/versioned/jobs/configuration.md @@ -58,6 +58,9 @@ Configuration keys that absolutely must be defined for a Samza job are: ### Configuration Keys -A complete list of configuration keys can be found on the [Configuration Table](configuration-table.html) page. +A complete list of configuration keys can be found on the [Configuration Table](configuration-table.html) page. Note +that configuration keys prefixed with "sensitive." are treated specially, in that the values associated with such keys +will be masked in logs and Samza's YARN ApplicationMaster UI. This is to prevent accidental disclosure only; no +encryption is done. ## [Packaging »](packaging.html) http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/main/java/org/apache/samza/config/Config.java ---------------------------------------------------------------------- diff --git a/samza-api/src/main/java/org/apache/samza/config/Config.java b/samza-api/src/main/java/org/apache/samza/config/Config.java index 9f7ade0..2b99050 100644 --- a/samza-api/src/main/java/org/apache/samza/config/Config.java +++ b/samza-api/src/main/java/org/apache/samza/config/Config.java @@ -33,6 +33,9 @@ import java.util.regex.Pattern; * Store and retrieve named, typed values as configuration for classes implementing this interface. */ public abstract class Config implements Map<String, String> { + public static final String SENSITIVE_PREFIX = "sensitive."; + public static final String SENSITIVE_MASK = "********"; + public Config subset(String prefix) { return subset(prefix, true); } @@ -221,6 +224,8 @@ public abstract class Config implements Map<String, String> { } } + public abstract Config sanitize(); + public void clear() { throw new ConfigException("Config is immutable."); } http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/main/java/org/apache/samza/config/MapConfig.java ---------------------------------------------------------------------- diff --git a/samza-api/src/main/java/org/apache/samza/config/MapConfig.java b/samza-api/src/main/java/org/apache/samza/config/MapConfig.java index 1a83923..38d7424 100644 --- a/samza-api/src/main/java/org/apache/samza/config/MapConfig.java +++ b/samza-api/src/main/java/org/apache/samza/config/MapConfig.java @@ -31,11 +31,11 @@ import java.util.Set; */ public class MapConfig extends Config { private final Map<String, String> map; - + public MapConfig() { this.map = Collections.emptyMap(); } - + public MapConfig(Map<String, String> map) { this(Collections.singletonList(map)); } @@ -79,6 +79,21 @@ public class MapConfig extends Config { } @Override + public Config sanitize() { + return new MapConfig(sanitizeMap()); + } + + private Map<String, String> sanitizeMap() { + Map<String, String> sanitized = new HashMap<String, String>(map); + for (Entry<String, String> entry : sanitized.entrySet()) { + if (entry.getKey().startsWith(SENSITIVE_PREFIX)) { + entry.setValue(SENSITIVE_MASK); + } + } + return sanitized; + } + + @Override public int hashCode() { final int prime = 31; int result = 1; @@ -105,6 +120,6 @@ public class MapConfig extends Config { @Override public String toString() { - return map.toString(); + return sanitizeMap().toString(); } } http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/test/java/org/apache/samza/config/TestConfig.java ---------------------------------------------------------------------- diff --git a/samza-api/src/test/java/org/apache/samza/config/TestConfig.java b/samza-api/src/test/java/org/apache/samza/config/TestConfig.java index b4100c2..d9f378d 100644 --- a/samza-api/src/test/java/org/apache/samza/config/TestConfig.java +++ b/samza-api/src/test/java/org/apache/samza/config/TestConfig.java @@ -19,7 +19,7 @@ package org.apache.samza.config; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import java.util.Map; import java.util.HashMap; @@ -61,4 +61,23 @@ public class TestConfig { Class c4 = getClass(mc.getLong("testkey", defaultLong)); assertEquals(Long.class, c4); } + + @Test + public void testSanitize() { + Map<String, String> m = new HashMap<String, String>() {{ + put("key1", "value1"); + put("key2", "value2"); + put("sensitive.key3", "secret1"); + put("sensitive.key4", "secret2"); + }}; + + Config config = new MapConfig(m); + assertFalse(config.toString().contains("secret")); + + Config sanitized = config.sanitize(); + assertEquals("value1", sanitized.get("key1")); + assertEquals("value2", sanitized.get("key2")); + assertEquals(Config.SENSITIVE_MASK, sanitized.get("sensitive.key3")); + assertEquals(Config.SENSITIVE_MASK, sanitized.get("sensitive.key4")); + } } http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala index 0afee64..4c855bf 100644 --- a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala +++ b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala @@ -103,6 +103,6 @@ class ApplicationMasterRestServlet(config: Config, state: SamzaAppMasterState, r } get("/config") { - jsonMapper.writeValueAsString(new HashMap[String, Object](config.toMap)) + jsonMapper.writeValueAsString(new HashMap[String, Object](config.sanitize.toMap)) } } http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala index 734d9d2..7fd5122 100644 --- a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala +++ b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala @@ -37,7 +37,7 @@ class ApplicationMasterWebServlet(config: Config, state: SamzaAppMasterState) ex get("/") { layoutTemplate("/WEB-INF/views/index.scaml", - "config" -> TreeMap(config.toMap.toArray: _*), + "config" -> TreeMap(config.sanitize.toMap.toArray: _*), "state" -> state, "rmHttpAddress" -> WebAppUtils.getRMWebAppURLWithScheme(yarnConfig)) }
