Repository: incubator-falcon Updated Branches: refs/heads/master 90faa0eab -> fd5478547
FALCON-746 Add ACL validation and enforcement tests. Contributed by Raghav Kumar Gautam Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/fd547854 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/fd547854 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/fd547854 Branch: refs/heads/master Commit: fd547854726a941008de6c36138432ef81682cb0 Parents: 90faa0e Author: Ruslan Ostafiychuk <rostafiyc...@apache.org> Authored: Thu Oct 9 15:47:10 2014 +0300 Committer: Ruslan Ostafiychuk <rostafiyc...@apache.org> Committed: Thu Oct 9 15:49:30 2014 +0300 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + falcon-regression/README.md | 113 ++++++- .../regression/Entities/ClusterMerlin.java | 12 + .../falcon/regression/Entities/FeedMerlin.java | 13 + .../regression/Entities/ProcessMerlin.java | 13 + .../falcon/regression/core/bundle/Bundle.java | 29 ++ .../core/enumsAndConstants/MerlinConstants.java | 22 +- .../core/interfaces/IEntityManagerHelper.java | 6 + .../core/supportClasses/ExecResult.java | 26 ++ .../falcon/regression/core/util/AssertUtil.java | 25 +- .../regression/core/util/CleanupUtil.java | 15 +- .../falcon/regression/core/util/MathUtil.java | 16 + .../falcon/regression/core/util/Util.java | 1 + .../regression/security/AclValidationTest.java | 148 ++++++++++ .../regression/security/ClusterAclTest.java | 152 ++++++++++ .../falcon/regression/security/EntityOp.java | 291 +++++++++++++++++++ .../falcon/regression/security/FeedAclTest.java | 211 ++++++++++++++ .../regression/security/ProcessAclTest.java | 209 +++++++++++++ 18 files changed, 1282 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 8281765..d31ea85 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-746 Add ACL validation and enforcement tests (Raghav Kumar Gautam via Ruslan Ostafiychuk) + FALCON-743 Adding tests for cases related to usage of pipelines tag (Paul Isaychuk via Raghav Kumar Gautam) http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/README.md ---------------------------------------------------------------------- diff --git a/falcon-regression/README.md b/falcon-regression/README.md index 3a0a3cc..b8139b4 100644 --- a/falcon-regression/README.md +++ b/falcon-regression/README.md @@ -15,19 +15,114 @@ limitations under the License. -falcon-regression +Falcon Regression ================= This project had 2 modules : -1. merlin -2. merlin-core +1. merlin: it has all the system test for falcon +2. merlin-core: it has all the utils used by merlin -merlin had all the test for apache falcon project -merlin-core has al the utils used by merlin +Requirements +------------ +In addition to falcon server and prism, running full falcon regression requires three clusters. +Each of these clusters must have: -Build Command : ------------------- +- hadoop +- oozie +- hive +- hcat +For specific tests it may be possible to run it without all clusters and components. + +Prior to running tests Merlin.properties needs to be created and populated with cluster details. + +Configuring Merlin.properties +----------------------------- +Merlin.properties must be created before running falcon regression tests. +The file must be created at the location: + + falcon/falcon-regression/merlin/src/main/resources/Merlin.properties + +Populate it with prism related properties: + + #prism properties + prism.oozie_url = http://node-1.example.com:11000/oozie/ + prism.oozie_location = /usr/lib/oozie/bin + prism.qa_host = node-1.example.com + prism.service_user = falcon + prism.hadoop_url = node-1.example.com:8020 + prism.hadoop_location = /usr/lib/hadoop/bin/hadoop + prism.ivory_hostname = http://node-1.example.com:15443 + prism.storeLocation = hdfs://node-1.example.com:8020/apps/falcon + +Specify the clusters that you would be using for testing: + + servers = cluster1,cluster2,cluster3 + +For each cluster specify properties: + + #cluster1 properties + cluster1.oozie_url = http://node-1.example.com:11000/oozie/ + cluster1.oozie_location = /usr/lib/oozie/bin + cluster1.qa_host = node-1.example.com + cluster1.service_user = falcon + cluster1.password = rgautam + cluster1.hadoop_url = node-1.example.com:8020 + cluster1.hadoop_location = /usr/lib/hadoop/bin/hadoop + cluster1.ivory_hostname = http://node-1.example.com:15443 + cluster1.cluster_readonly = webhdfs://node-1.example.com:50070 + cluster1.cluster_execute = node-1.example.com:8032 + cluster1.cluster_write = hdfs://node-1.example.com:8020 + cluster1.activemq_url = tcp://node-1.example.com:61616?daemon=true + cluster1.storeLocation = hdfs://node-1.example.com:8020/apps/falcon + cluster1.colo = default + cluster1.namenode.kerberos.principal = nn/node-1.example.com@none + cluster1.hive.metastore.kerberos.principal = hive/node-1.example.com@none + cluster1.hcat_endpoint = thrift://node-1.example.com:9083 + cluster1.service_stop_cmd = /usr/lib/falcon/bin/falcon-stop + cluster1.service_start_cmd = /usr/lib/falcon/bin/falcon-start + +Running Tests +------------- +After creating Merlin.properties file. You can run the following commands to run the tests. + + cd falcon-regression + mvn clean test -Phadoop-2 -Fast Build : mvn clean install -DskipTests -DskipCheck=true -Phadoop-2 -Regression build : mvn clean install -Phadoop-2 Profiles Supported: hadoop-2 + +To run a specific test: + + mvn clean test -Phadoop-2 -Dtest=EmbeddedPigScriptTest + +If you want to use specific version of any component, they can be specified using -D, for eg: + + mvn clean test -Phadoop-2 -Doozie.version=4.1.0 -Dhadoop.version=2.6.0 + +Security Tests: +--------------- +ACL tests require multiple user account setup: + + other.user.name=root + admin.user.name=falcon + admin2.user.name=falcon2 + +ACL tests also require group name of the current user: + + current_user.group.name=users + +For testing with kerberos set keytabs properties for different users: + + current_user_keytab=/home/qa/hadoopqa/keytabs/qa.headless.keytab + admin.user.keytab=/home/qa/hadoopqa/keytabs/falcon.headless.keytab + admin2.user.keytab=/home/qa/hadoopqa/keytabs/falcon2.headless.keytab + other.user.keytab=/home/qa/hadoopqa/keytabs/root.headless.keytab + +Automatic capture of oozie logs +------------------------------- +For full falcon regression runs. It might be desirable to pull all oozie job +info and logs at the end of the test. This can be done by configuring Merlin.properties: + + log.capture.oozie = true + log.capture.oozie.skip_info = false + log.capture.oozie.skip_log = true + log.capture.location = ../ http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java index 33b67f7..b1a94be 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java @@ -21,6 +21,7 @@ package org.apache.falcon.regression.Entities; import org.apache.commons.beanutils.PropertyUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.cluster.ACL; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.regression.core.util.Util; import org.testng.Assert; @@ -81,4 +82,15 @@ public class ClusterMerlin extends Cluster { nameMap.put(oldName, newName); return nameMap; } + + /** + * Set ACL. + */ + public void setACL(String owner, String group, String permission) { + ACL acl = new ACL(); + acl.setOwner(owner); + acl.setGroup(group); + acl.setPermission(permission); + this.setACL(acl); + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java index 025fc15..02f572e 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.entity.v0.feed.ACL; import org.apache.falcon.entity.v0.feed.ActionType; import org.apache.falcon.entity.v0.feed.Cluster; import org.apache.falcon.entity.v0.feed.Clusters; @@ -173,4 +174,16 @@ public class FeedMerlin extends Feed { } } } + + /** + * Set ACL. + */ + public void setACL(String owner, String group, String permission) { + ACL acl = new ACL(); + acl.setOwner(owner); + acl.setGroup(group); + acl.setPermission(permission); + this.setACL(acl); + } + } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java index a10dfc1..d81f577 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.process.ACL; import org.apache.falcon.entity.v0.process.Cluster; import org.apache.falcon.entity.v0.process.Clusters; import org.apache.falcon.entity.v0.process.Input; @@ -252,6 +253,18 @@ public class ProcessMerlin extends Process { this.pipelines = null; } } + + /** + * Set ACL. + */ + public void setACL(String owner, String group, String permission) { + ACL acl = new ACL(); + acl.setOwner(owner); + acl.setGroup(group); + acl.setPermission(permission); + this.setACL(acl); + } + } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java index eac3cb4..8e55493 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java @@ -1024,4 +1024,33 @@ public class Bundle { process.setPipelineTag(pipelines); setProcessData(process.toString()); } + + /** + * Set ACL of bundle's cluster. + */ + public void setCLusterACL(String owner, String group, String permission) { + ClusterMerlin clusterMerlin = getClusterElement(); + clusterMerlin.setACL(owner, group, permission); + writeClusterElement(clusterMerlin); + + } + + /** + * Set ACL of bundle's input feed. + */ + public void setInputFeedACL(String owner, String group, String permission) { + String feedName = getInputFeedNameFromBundle(); + FeedMerlin feedMerlin = getFeedElement(feedName); + feedMerlin.setACL(owner, group, permission); + writeFeedElement(feedMerlin, feedName); + } + + /** + * Set ACL of bundle's process. + */ + public void setProcessACL(String owner, String group, String permission) { + ProcessMerlin processMerlin = getProcessObject(); + processMerlin.setACL(owner, group, permission); + setProcessData(processMerlin.toString()); + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java index 3a56530..e877803 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java @@ -27,7 +27,7 @@ import org.apache.log4j.Logger; import java.util.HashMap; /** - * Class for test contants. + * Class for test constants. */ public final class MerlinConstants { private MerlinConstants() { @@ -37,9 +37,22 @@ public final class MerlinConstants { public static final boolean IS_SECURE = "kerberos".equals(new Configuration().get("hadoop.security.authentication", "simple")); + /** the user that is going to run tests. */ public static final String CURRENT_USER_NAME = System.getProperty("user.name"); - private static final String CURRENT_USER_KEYTAB_STR = - "current_user_keytab"; + /** keytab of current user. */ + private static final String CURRENT_USER_KEYTAB_STR = "current_user_keytab"; + /** group of the current user. */ + public static final String CURRENT_USER_GROUP = + Config.getProperty("current_user.group.name", "users"); + + /** a user that does not belong to the group of current user. */ + public static final String DIFFERENT_USER = Config.getProperty("other.user.name", "root"); + + /** falcon admin user: the user that started falcon prism/server processes. */ + public static final String ADMIN_USER_NAME = Config.getProperty("admin.user.name", "falcon"); + + /** a user that belongs to falcon admin group but is not same as falcon admin user. */ + public static final String ADMIN2_USER_NAME = Config.getProperty("admin2.user.name", "falcon2"); private static final String USER_2_NAME_STR = "user2_name"; private static final String USER_2_KEYTAB_STR = "user2_keytab"; public static final String USER2_NAME; @@ -64,6 +77,9 @@ public final class MerlinConstants { keyTabMap = new HashMap<String, String>(); keyTabMap.put(CURRENT_USER_NAME, currentUserKeytab); keyTabMap.put(user2Name, user2Keytab); + keyTabMap.put(ADMIN_USER_NAME, Config.getProperty("admin.user.keytab")); + keyTabMap.put(ADMIN2_USER_NAME, Config.getProperty("admin2.user.keytab")); + keyTabMap.put(DIFFERENT_USER, Config.getProperty("other.user.keytab")); } public static String getKeytabForUser(String user) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java index 66a1b7f..52f1bc1 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java @@ -361,6 +361,12 @@ public abstract class IEntityManagerHelper { getEntityType(), getEntityName(data) + colo), "get", user); } + public ServiceResponse getEntityDependencies(String data, String user) + throws IOException, URISyntaxException, AuthenticationException { + return Util.sendRequest(createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), + getEntityType(), getEntityName(data) + colo), "get", user); + } + public InstancesResult getRunningInstance(String name) throws IOException, URISyntaxException, AuthenticationException { return getRunningInstance(name, null); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java new file mode 100644 index 0000000..73732d5 --- /dev/null +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java @@ -0,0 +1,26 @@ +package org.apache.falcon.regression.core.supportClasses; + +public final class ExecResult { + + private final int exitVal; + private final String output; + private final String error; + + public ExecResult(final int exitVal, final String output, final String error) { + this.exitVal = exitVal; + this.output = output; + this.error = error; + } + + public int getExitVal() { + return exitVal; + } + + public String getOutput() { + return output; + } + + public String getError() { + return error; + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java index d8ac903..c0c5755 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java @@ -157,11 +157,30 @@ public final class AssertUtil { * @throws JAXBException */ public static void assertSucceeded(ServiceResponse response) throws JAXBException { - Assert.assertEquals(Util.parseResponse(response).getStatus(), + final APIResult apiResult = Util.parseResponse(response); + Assert.assertEquals(apiResult.getStatus(), APIResult.Status.SUCCEEDED, "Status should be SUCCEEDED"); - Assert.assertEquals(Util.parseResponse(response).getStatusCode(), 200, + Assert.assertEquals(apiResult.getStatusCode(), 200, "Status code should be 200"); - Assert.assertNotNull(Util.parseResponse(response).getMessage(), "Status message is null"); + Assert.assertNotNull(apiResult.getMessage(), "Status message is null"); + } + + /** + * Checks that ServiceResponse status is SUCCEEDED. + * + * @param response ServiceResponse + * @return if the response was a success or not + */ + public static boolean checkSucceeded(ServiceResponse response) { + final APIResult apiResult; + try { + apiResult = Util.parseResponse(response); + } catch (JAXBException e) { + return false; + } + return apiResult.getStatus() == APIResult.Status.SUCCEEDED + && apiResult.getStatusCode() == 200 + && apiResult.getMessage() != null; } /** http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java index bf99291..6b07589 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java @@ -47,22 +47,23 @@ public final class CleanupUtil { public static List<String> getAllProcesses(ColoHelper prism) throws IOException, URISyntaxException, AuthenticationException, JAXBException { - return getAllEntitiesOfOneType(prism.getProcessHelper()); + return getAllEntitiesOfOneType(prism.getProcessHelper(), null); } public static List<String> getAllFeeds(ColoHelper prism) throws IOException, URISyntaxException, AuthenticationException, JAXBException { - return getAllEntitiesOfOneType(prism.getFeedHelper()); + return getAllEntitiesOfOneType(prism.getFeedHelper(), null); } public static List<String> getAllClusters(ColoHelper prism) throws IOException, URISyntaxException, AuthenticationException, JAXBException { - return getAllEntitiesOfOneType(prism.getClusterHelper()); + return getAllEntitiesOfOneType(prism.getClusterHelper(), null); } - private static List<String> getAllEntitiesOfOneType(IEntityManagerHelper iEntityManagerHelper) + public static List<String> getAllEntitiesOfOneType(IEntityManagerHelper iEntityManagerHelper, + String user) throws IOException, URISyntaxException, AuthenticationException, JAXBException { - final EntitiesResult entitiesResult = getEntitiesResultOfOneType(iEntityManagerHelper); + final EntitiesResult entitiesResult = getEntitiesResultOfOneType(iEntityManagerHelper, user); List<String> clusters = new ArrayList<String>(); for (EntityResult entity : entitiesResult.getEntities()) { clusters.add(entity.getName()); @@ -71,9 +72,9 @@ public final class CleanupUtil { } private static EntitiesResult getEntitiesResultOfOneType( - IEntityManagerHelper iEntityManagerHelper) + IEntityManagerHelper iEntityManagerHelper, String user) throws IOException, URISyntaxException, AuthenticationException, JAXBException { - final ServiceResponse clusterResponse = iEntityManagerHelper.listEntities(); + final ServiceResponse clusterResponse = iEntityManagerHelper.listEntities(user); JAXBContext jc = JAXBContext.newInstance(EntitiesResult.class); Unmarshaller u = jc.createUnmarshaller(); return (EntitiesResult) u.unmarshal( http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java index 1bbe8d9..7c6c73d 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java @@ -18,6 +18,11 @@ package org.apache.falcon.regression.core.util; +import org.apache.commons.lang.ArrayUtils; +import org.testng.Assert; + +import java.util.Arrays; + public class MathUtil { private MathUtil() { throw new AssertionError("Instantiating utility class..."); @@ -57,4 +62,15 @@ public class MathUtil { } return result; } + + public static Object[][] append(Object[][] arr1, Object[][] arr2) { + Assert.assertFalse(ArrayUtils.isEmpty(arr1), "arr1 can't be empty:" + + Arrays.deepToString(arr1)); + Assert.assertFalse(ArrayUtils.isEmpty(arr2), "arr2 can't be empty:" + + Arrays.deepToString(arr2)); + Assert.assertEquals(arr1[0].length, arr2[0].length, "Array rows are not compatible. " + + "row of first array: " + Arrays.deepToString(arr1[0]) + + "row of second array: " + Arrays.deepToString(arr2[0])); + return (Object[][]) ArrayUtils.addAll(arr1, arr2); + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java index 30eaf61..1d07fc5 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java @@ -595,6 +595,7 @@ public final class Util { LIST_URL("/api/entities/list"), SUBMIT_URL("/api/entities/submit"), GET_ENTITY_DEFINITION("/api/entities/definition"), + DEPENDENCIES("/api/entities/dependencies"), DELETE_URL("/api/entities/delete"), SCHEDULE_URL("/api/entities/schedule"), VALIDATE_URL("/api/entities/validate"), http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java new file mode 100644 index 0000000..73c3b30 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.commons.httpclient.HttpStatus; +import org.apache.falcon.regression.Entities.ClusterMerlin; +import org.apache.falcon.regression.Entities.FeedMerlin; +import org.apache.falcon.regression.Entities.ProcessMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.lang.reflect.Method; + +/** + * Tests if ACL info is consistent with user submitting the entity + */ +@Test(groups = "embedded") +public class AclValidationTest extends BaseTestClass { + private static final Logger LOGGER = Logger.getLogger(AclValidationTest.class); + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String baseTestDir = baseHDFSDir + "/AuthorizationTest"; + private String aggregateWorkflowDir = baseTestDir + "/aggregator"; + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + + ClusterMerlin clusterMerlin; + FeedMerlin feedMerlin; + ProcessMerlin processMerlin; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setup(Method method) throws Exception { + LOGGER.info("test name: " + method.getName()); + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + clusterMerlin = bundles[0].getClusterElement(); + feedMerlin = new FeedMerlin(bundles[0].getInputFeedFromBundle()); + processMerlin = bundles[0].getProcessObject(); + } + + /** + * Test a cluster's acl validations for different aclOwner and aclGroup + * @param aclOwner owner for the acl + * @param aclGroup group for the acl + * @throws Exception + */ + @Test(dataProvider = "generateUserAndGroup") + public void submitClusterBadAcl(String aclOwner, String aclGroup) throws Exception { + clusterMerlin.setACL(aclOwner,aclGroup, "*"); + final ServiceResponse serviceResponse = + prism.getClusterHelper().submitEntity(clusterMerlin.toString()); + AssertUtil.assertFailedWithStatus(serviceResponse, HttpStatus.SC_BAD_REQUEST, + "Cluster's ACL owner should be same as submitting user"); + } + + /** + * Test a feed's acl validations for different aclOwner and aclGroup + * @param aclOwner owner for the acl + * @param aclGroup group for the acl + * @throws Exception + */ + @Test(dataProvider = "generateUserAndGroup") + public void submitFeedBadAcl(String aclOwner, String aclGroup) throws Exception { + bundles[0].submitClusters(prism); + feedMerlin.setACL(aclOwner, aclGroup, "*"); + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + AssertUtil.assertFailedWithStatus(serviceResponse, HttpStatus.SC_BAD_REQUEST, + "Feed's ACL owner should be same as submitting user"); + } + + /** + * Test a process's acl validations for different aclOwner and aclGroup + * @param aclOwner owner for the acl + * @param aclGroup group for the acl + * @throws Exception + */ + @Test(dataProvider = "generateUserAndGroup") + public void submitProcessBadAcl(String aclOwner, String aclGroup) throws Exception { + bundles[0].submitAndScheduleAllFeeds(); + processMerlin.setACL(aclOwner, aclGroup, "*"); + final ServiceResponse serviceResponse = + prism.getProcessHelper().submitEntity(processMerlin.toString()); + AssertUtil.assertFailedWithStatus(serviceResponse, HttpStatus.SC_BAD_REQUEST, + "Process's ACL owner should be same as submitting user"); + } + + @DataProvider(name = "generateUserAndGroup") + public Object[][] generateUserAndGroup() { + return new Object[][] { + {MerlinConstants.USER2_NAME, MerlinConstants.CURRENT_USER_GROUP}, + {MerlinConstants.CURRENT_USER_NAME, MerlinConstants.DIFFERENT_USER}, + {MerlinConstants.CURRENT_USER_NAME, "nonexistinggroup"}, + {"nonexistinguser", MerlinConstants.CURRENT_USER_GROUP}, + }; + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeBundles(); + } + + @AfterClass(alwaysRun = true) + public void tearDownClass() throws IOException { + cleanTestDirs(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java new file mode 100644 index 0000000..bbcb43d --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.KerberosHelper; +import org.apache.falcon.regression.core.util.MathUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.lang.reflect.Method; + +@Test(groups = "embedded") +public class ClusterAclTest extends BaseTestClass { + private static final Logger LOGGER = Logger.getLogger(ClusterAclTest.class); + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String baseTestDir = baseHDFSDir + "/ClusterAclTest"; + private String aggregateWorkflowDir = baseTestDir + "/aggregator"; + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + private final IEntityManagerHelper clusterHelper = prism.getClusterHelper(); + private String clusterString; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setup(Method method) throws Exception { + LOGGER.info("test name: " + method.getName()); + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].setCLusterACL(MerlinConstants.CURRENT_USER_NAME, + MerlinConstants.CURRENT_USER_GROUP, "*"); + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + clusterString = bundles[0].getClusters().get(0); + } + + /** + * Test cluster read operations by different users + * @param user the user that would perform operation + * @param op the operation that user would perform + * @param isAllowed is the user allowed that operation + * @throws Exception + */ + @Test(dataProvider = "generateUserReadOpsPermissions") + public void othersReadCluster(final String user, EntityOp op, final boolean isAllowed) + throws Exception { + bundles[0].submitClusters(prism); + bundles[0].submitFeeds(prism); + final boolean executeRes = op.executeAs(user, clusterHelper, clusterString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + " " + + "performing: " + op); + } + + @DataProvider(name = "generateUserReadOpsPermissions") + public Object[][] generateUserReadOpsPermissions() { + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + new EntityOp[]{EntityOp.dependency, EntityOp.listing, EntityOp.definition}, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + new EntityOp[]{EntityOp.dependency, EntityOp.listing, EntityOp.definition}, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + /** + * Test cluster deletion by different users + * @param deleteUser the user that would attempt to delete + * @param deleteAllowed is delete expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserAndDeletePermission") + public void othersDeleteCluster(final String deleteUser, final boolean deleteAllowed) + throws Exception { + bundles[0].submitClusters(prism); + final ServiceResponse response = clusterHelper.delete(clusterString, deleteUser); + if(deleteAllowed) { + AssertUtil.assertSucceeded(response); + } else { + AssertUtil.assertFailed(response); + } + } + + @DataProvider(name = "generateUserAndDeletePermission") + public Object[][] generateUserAndDeletePermission() { + return new Object[][] { + //first element is username, second element indicates if deletion is allowed + {MerlinConstants.ADMIN_USER_NAME, true}, + {MerlinConstants.ADMIN2_USER_NAME, true}, + {MerlinConstants.USER2_NAME, true}, + {"root", false}, + }; + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + removeBundles(); + } + + @AfterClass(alwaysRun = true) + public void tearDownClass() throws IOException { + cleanTestDirs(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java new file mode 100644 index 0000000..4119a1d --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.CleanupUtil; +import org.apache.falcon.regression.core.util.KerberosHelper; +import org.apache.falcon.regression.core.util.Util; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.log4j.Logger; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.List; + +/** + * All the falcon operation are implemented as enum. The benefit of this is that these operations + * can now be passed as parameters. + */ +enum EntityOp { + status() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + final ServiceResponse response; + try { + response = helper.getStatus(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + dependency() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + final ServiceResponse response; + try { + response = helper.getEntityDependencies(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + listing() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(user); + final String entityName = Util.readEntityName(data); + final List<String> entities; + try { + entities = CleanupUtil.getAllEntitiesOfOneType(helper, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (JAXBException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + logger.info("Checking for presence of " + entityName + " in " + entities); + return entities.contains(entityName); + } + }, + definition() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.getEntityDefinition(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + delete() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.delete(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + update() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.update(data, data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + schedule() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.schedule(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + submit() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.submitEntity(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + submitAndSchedule() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.submitAndSchedule(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + suspend() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.suspend(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + resume() { + @Override + public boolean executeAs(String user, IEntityManagerHelper helper, String data) { + KerberosHelper.loginFromKeytab(MerlinConstants.getKeytabForUser(user)); + final ServiceResponse response; + try { + response = helper.resume(data, user); + } catch (IOException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (URISyntaxException e) { + logger.warn("Caught exception: " + e); + return false; + } catch (AuthenticationException e) { + logger.warn("Caught exception: " + e); + return false; + } finally { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + return AssertUtil.checkSucceeded(response); + } + }, + ; + + private static Logger logger = Logger.getLogger(EntityOp.class); + public abstract boolean executeAs(String user, IEntityManagerHelper helper, String data); +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java new file mode 100644 index 0000000..18f548c --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.falcon.regression.Entities.FeedMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.KerberosHelper; +import org.apache.falcon.regression.core.util.MathUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.lang.reflect.Method; + +@Test(groups = "embedded") +public class FeedAclTest extends BaseTestClass { + private static final Logger LOGGER = Logger.getLogger(FeedAclTest.class); + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String baseTestDir = baseHDFSDir + "/FeedAclTest"; + private String aggregateWorkflowDir = baseTestDir + "/aggregator"; + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + private final IEntityManagerHelper feedHelper = prism.getFeedHelper(); + private String feedString; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setup(Method method) throws Exception { + LOGGER.info("test name: " + method.getName()); + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].setInputFeedACL(MerlinConstants.CURRENT_USER_NAME, + MerlinConstants.CURRENT_USER_GROUP, "*"); + feedString = bundles[0].getInputFeedFromBundle(); + + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + + /** + * Test feed read operation by different types of user + * @param user the user that would attempt operation + * @param op the read operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserReadOpsPermissions") + public void othersReadFeed(final String user, final EntityOp op, final boolean isAllowed) + throws Exception { + bundles[0].submitClusters(prism); + bundles[0].submitFeeds(prism); + final boolean executeRes = op.executeAs(user, feedHelper, feedString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserReadOpsPermissions") + public Object[][] generateUserReadOpsPermissions() { + final EntityOp[] falconReadOps = {EntityOp.status, EntityOp.dependency, + EntityOp.listing, EntityOp.definition}; + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + falconReadOps, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + falconReadOps, + new Boolean[]{false} + ); + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + /** + * Test edit operation on submitted feed by different users + * @param user the user that would attempt operation + * @param op the edit operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserSubmittedEditOpsPermission") + public void othersEditSubmittedFeed(final String user, final EntityOp op, boolean isAllowed) + throws Exception { + bundles[0].submitClusters(prism); + bundles[0].submitFeeds(prism); + if (op == EntityOp.update) { + FeedMerlin feedMerlin = new FeedMerlin(feedString); + feedMerlin.addProperty("abc", "xyz"); + feedString = feedMerlin.toString(); + } + final boolean executeRes = op.executeAs(user, feedHelper, feedString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserSubmittedEditOpsPermission") + public Object[][] generateUserSubmittedEditOpsPermission() { + final EntityOp[] falconEditOps = {EntityOp.submit, EntityOp.delete, EntityOp.update, + EntityOp.schedule, EntityOp.submitAndSchedule}; + + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + falconEditOps, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + falconEditOps, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + /** + * Test edit operation on scheduled feed by different users + * @param user the user that would attempt operation + * @param op the edit operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserScheduledEditOpsPermission") + public void othersEditScheduledFeed(final String user, final EntityOp op, boolean isAllowed) + throws Exception { + bundles[0].submitClusters(prism); + bundles[0].submitAndScheduleAllFeeds(); + if(op == EntityOp.resume) { + feedHelper.suspend(feedString); + } else if (op == EntityOp.update) { + FeedMerlin feedMerlin = new FeedMerlin(feedString); + feedMerlin.addProperty("abc", "xyz"); + feedString = feedMerlin.toString(); + } + final boolean executeRes = op.executeAs(user, feedHelper, feedString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserScheduledEditOpsPermission") + public Object[][] generateUserScheduledEditOpsPermission() { + + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.suspend, EntityOp.resume}, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.schedule, + EntityOp.submitAndSchedule, EntityOp.suspend, EntityOp.resume}, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + removeBundles(); + } + + @AfterClass(alwaysRun = true) + public void tearDownClass() throws IOException { + cleanTestDirs(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fd547854/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java new file mode 100644 index 0000000..c9191af --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.falcon.regression.Entities.ProcessMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.KerberosHelper; +import org.apache.falcon.regression.core.util.MathUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.lang.reflect.Method; + +@Test(groups = "embedded") +public class ProcessAclTest extends BaseTestClass { + private static final Logger LOGGER = Logger.getLogger(ProcessAclTest.class); + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String baseTestDir = baseHDFSDir + "/ProcessAclTest"; + private String aggregateWorkflowDir = baseTestDir + "/aggregator"; + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + private final IEntityManagerHelper processHelper = prism.getProcessHelper(); + String processString; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setup(Method method) throws Exception { + LOGGER.info("test name: " + method.getName()); + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].setProcessACL(MerlinConstants.CURRENT_USER_NAME, + MerlinConstants.CURRENT_USER_GROUP, "*"); + processString = bundles[0].getProcessData(); + + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + } + + /** + * Test process read operation by different types of user + * @param user the user that would attempt operation + * @param op the read operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserReadOpsPermissions") + public void othersReadProcess(final String user, final EntityOp op, final boolean isAllowed) + throws Exception { + bundles[0].submitProcess(true); + final boolean executeRes = op.executeAs(user, processHelper, processString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserReadOpsPermissions") + public Object[][] generateUserReadOpsPermissions() { + final EntityOp[] falconReadOps = {EntityOp.status, EntityOp.dependency, + EntityOp.listing, EntityOp.definition}; + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + falconReadOps, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + falconReadOps, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + /** + * Test edit operation on submitted process by different users + * @param user the user that would attempt operation + * @param op the edit operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserSubmittedEditOpsPermission") + public void othersEditSubmittedProcess(final String user, final EntityOp op, boolean isAllowed) + throws Exception { + bundles[0].submitProcess(true); + if (op == EntityOp.update) { + ProcessMerlin processMerlin = new ProcessMerlin(processString); + processMerlin.setProperty("abc", "xyz"); + processString = processMerlin.toString(); + } + final boolean executeRes = op.executeAs(user, processHelper, processString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserSubmittedEditOpsPermission") + public Object[][] generateUserSubmittedEditOpsPermission() { + final EntityOp[] falconEditOps = {EntityOp.submit, EntityOp.delete, EntityOp.update, + EntityOp.schedule, EntityOp.submitAndSchedule}; + + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + falconEditOps, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + falconEditOps, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + /** + * Test edit operation on scheduled process by different users + * @param user the user that would attempt operation + * @param op the edit operation that would be performed + * @param isAllowed is operation expected to go through + * @throws Exception + */ + @Test(dataProvider = "generateUserScheduledEditOpsPermission") + public void othersEditScheduledProcess(final String user, final EntityOp op, boolean isAllowed) + throws Exception { + bundles[0].submitFeedsScheduleProcess(); + if(op == EntityOp.resume) { + processHelper.suspend(processString); + } else if (op == EntityOp.update) { + ProcessMerlin processMerlin = new ProcessMerlin(processString); + processMerlin.setProperty("abc", "xyz"); + processString = processMerlin.toString(); + } + final boolean executeRes = op.executeAs(user, processHelper, processString); + Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + + " performing: " + op); + } + + @DataProvider(name = "generateUserScheduledEditOpsPermission") + public Object[][] generateUserScheduledEditOpsPermission() { + + final Object[][] allowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.ADMIN_USER_NAME, MerlinConstants.ADMIN2_USER_NAME, + MerlinConstants.USER2_NAME}, + new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.suspend, EntityOp.resume}, + new Boolean[]{true} + ); + + final Object[][] notAllowedCombinations = MathUtil.crossProduct( + new String[]{MerlinConstants.DIFFERENT_USER}, + new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.schedule, + EntityOp.submitAndSchedule, EntityOp.suspend, EntityOp.resume}, + new Boolean[]{false} + ); + + return MathUtil.append(allowedCombinations, notAllowedCombinations); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + removeBundles(); + } + + @AfterClass(alwaysRun = true) + public void tearDownClass() throws IOException { + cleanTestDirs(); + } +}
