http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.2/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/default.json b/ambari-server/src/test/python/stacks/2.2/configs/default.json index 8188928..c5a6ae7 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/default.json @@ -185,18 +185,7 @@ "ignore_groupsusers_create": "false", "smokeuser": "ambari-qa", "kerberos_domain": "EXAMPLE.COM", - "user_group": "hadoop", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz" + "user_group": "hadoop" }, "ranger-knox-plugin-properties": { "POLICY_MGR_URL": "{{policymgr_mgr_url}}",
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.2/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/secured.json b/ambari-server/src/test/python/stacks/2.2/configs/secured.json index e224ebc..5bd8814 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/secured.json @@ -173,18 +173,7 @@ "user_group": "hadoop", "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab", "smokeuser_principal_name": "ambari...@example.com", - "kinit_path_local": "/usr/bin", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz" + "kinit_path_local": "/usr/bin" }, "webhcat-site": { "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar", http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py index 665119f..0d943c4 100644 --- a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py +++ b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py @@ -33,42 +33,35 @@ class TestMahoutClient(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) + self.assertResourceCalled('ExecuteHadoop', 'fs -rm -r -f /user/ambari-qa/mahoutsmokeoutput /user/ambari-qa/mahoutsmokeinput', + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/usr/hdp/current/hadoop-client/conf', + try_sleep = 5, + kinit_path_local = '/usr/bin/kinit', + tries = 3, + user = 'ambari-qa', + bin_dir = '/usr/hdp/current/hadoop-client/bin', + principal = UnknownConfigurationMock(), + ) + self.assertResourceCalled('ExecuteHadoop', 'fs -mkdir /user/ambari-qa/mahoutsmokeinput', + try_sleep = 5, + tries = 3, + bin_dir = '/usr/hdp/current/hadoop-client/bin', + user = 'ambari-qa', + conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) self.assertResourceCalled('File', '/tmp/sample-mahout-test.txt', - content = 'Test text which will be converted to sequence file.', - mode = 0755, - ) - self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput', - security_enabled = False, - hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', - keytab = UnknownConfigurationMock(), - kinit_path_local = '/usr/bin/kinit', - user = 'hdfs', - owner = 'ambari-qa', - hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', - type = 'directory', - action = ['create_on_execute'], - ) - self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput/sample-mahout-test.txt', - security_enabled = False, - hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', - keytab = UnknownConfigurationMock(), - kinit_path_local = '/usr/bin/kinit', - source = '/tmp/sample-mahout-test.txt', - user = 'hdfs', - owner = 'ambari-qa', - hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', - type = 'file', - action = ['create_on_execute'], - ) - self.assertResourceCalled('HdfsResource', None, - security_enabled = False, - hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', - keytab = UnknownConfigurationMock(), - kinit_path_local = '/usr/bin/kinit', - user = 'hdfs', - action = ['execute'], - hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', - ) + content = 'Test text which will be converted to sequence file.', + mode = 0755, + ) + self.assertResourceCalled('ExecuteHadoop', 'fs -put /tmp/sample-mahout-test.txt /user/ambari-qa/mahoutsmokeinput/', + try_sleep = 5, + tries = 3, + bin_dir = '/usr/hdp/current/hadoop-client/bin', + user = 'ambari-qa', + conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) self.assertResourceCalled('Execute', 'mahout seqdirectory --input /user/ambari-qa/mahoutsmokeinput/' 'sample-mahout-test.txt --output /user/ambari-qa/mahoutsmokeoutput/ ' '--charset utf-8', http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/dependency-reduced-pom.xml ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/dependency-reduced-pom.xml b/contrib/fast-hdfs-resource/dependency-reduced-pom.xml deleted file mode 100644 index c252f2e..0000000 --- a/contrib/fast-hdfs-resource/dependency-reduced-pom.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <groupId>org.apache.ambari</groupId> - <artifactId>fast-hdfs-resource</artifactId> - <name>fast-hdfs-resource</name> - <version>0.0.1-SNAPSHOT</version> - <url>http://maven.apache.org</url> - <build> - <plugins> - <plugin> - <artifactId>maven-shade-plugin</artifactId> - <version>2.3</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer> - <mainClass>org.apache.ambari.fast_hdfs_resource.Runner</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - <repositories> - <repository> - <id>hdp.internal</id> - <url>http://repo1.maven.org/maven2</url> - </repository> - </repositories> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> -</project> - http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/pom.xml b/contrib/fast-hdfs-resource/pom.xml deleted file mode 100644 index 6a92841..0000000 --- a/contrib/fast-hdfs-resource/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <groupId>org.apache.ambari</groupId> - <artifactId>fast-hdfs-resource</artifactId> - <version>0.0.1-SNAPSHOT</version> - <packaging>jar</packaging> - - <name>fast-hdfs-resource</name> - <url>http://maven.apache.org</url> - <repositories> - <repository> - <id>hdp.internal</id> - <url>http://repo1.maven.org/maven2</url> - </repository> - </repositories> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-tools</artifactId> - <version>1.2.1</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>1.2.1</version> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.2.2</version> - </dependency> - </dependencies> - - - <!-- Create executable jar with the application entry point --> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.3</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>org.apache.ambari.fast_hdfs_resource.Runner - </mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/resources/example.json ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/resources/example.json b/contrib/fast-hdfs-resource/resources/example.json deleted file mode 100644 index 605deb9..0000000 --- a/contrib/fast-hdfs-resource/resources/example.json +++ /dev/null @@ -1,57 +0,0 @@ -[ -{ - "target":"/tmp/some999", - "type":"directory", - "action":"delete" -}, -{ - "target":"/tmp/some999/more/dirs/for/recursive/tests", - "type":"directory", - "action":"create" -}, -{ - "target":"/tmp/some999/more/dirs/for/recursive/tests/file_empty.txt", - "type":"file", - "action":"create" -}, -{ - "target":"/tmp/some999", - "type":"directory", - "action":"create", - "owner":"oozie" -}, -{ - "target":"/tmp/some999", - "type":"directory", - "action":"create", - "group":"hive" -}, -{ - "target":"/tmp/some999", - "type":"directory", - "action":"create", - "mode":"777" -}, -{ - "target":"/tmp/some999/more/dirs", - "type":"directory", - "action":"create", - "owner":"yarn", - "group":"mapred", - "recursiveChown":true, - "mode":"757", - "recursiveChmod":true -}, -{ - "source":"/tmp/my.txt", - "target":"/tmp/some999/my_file.txt", - "type":"file", - "action":"create" -}, -{ - "source":"/tmp/a", - "target":"/tmp/some999/a_dir", - "type":"directory", - "action":"create" -} -] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/resources/test_perfomance.sh ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/resources/test_perfomance.sh b/contrib/fast-hdfs-resource/resources/test_perfomance.sh deleted file mode 100644 index 40339e3..0000000 --- a/contrib/fast-hdfs-resource/resources/test_perfomance.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Delete 2 dirs -sudo -u hdfs hadoop fs -rm -r /tmp/some999 -sudo -u hdfs hadoop fs -rm -r /tmp/some888 -# Create -sudo -u hdfs hadoop fs -mkdir -p /tmp/some999/more/dirs/for/recursive/tests -# Create + permissions + owner -sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/more/dirs/for/recursive/tests -sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/more/dirs/for/recursive/tests -sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/more/dirs/for/recursive/tests -# Empty dirs with permissions/owners to last dir" -sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/and_more/and_dirs/_andfor/recursive/tests -sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/_andfor/recursive/tests -sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/_andfor/recursive/tests -# Empty dirs with permissions/owners to last file -sudo -u hdfs hadoop fs -touchz /tmp/some888/file.txt -sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/file.txt -sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/file.txt -# Empty dirs with permissions/owners to last file -sudo -u hdfs hadoop fs -touchz /tmp/some888/and_more/and_dirs/file2.txt -sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/file2.txt -sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/file2.txt -# Recursive permissions -sudo -u hdfs hadoop fs -chmod -R 700 /tmp/some888 -sudo -u hdfs hadoop fs -chown -R hive:hive /tmp/some999 - - http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java deleted file mode 100644 index 21750e1..0000000 --- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.fast_hdfs_resource; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.File; -import java.lang.System; -import java.util.ArrayList; -import java.util.HashSet; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.FileSystem; - -/** - * Used to: 1) copy files/directories from localFS to hadoopFs 2) create empty - * files/directories in hadoopFs - */ -public class Resource { - private String source; - private String target; - private String type; - private String action; - private String owner; - private String group; - private String mode; - private boolean recursiveChown; - private boolean recursiveChmod; - private boolean changePermissionforParents; - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public String getTarget() { - return target; - } - - public void setTarget(String target) { - this.target = target; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getAction() { - return action; - } - - public void setAction(String action) { - this.action = action; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getGroup() { - return group; - } - - public void setGroup(String group) { - this.group = group; - } - - public String getMode() { - return mode; - } - - public void setMode(String mode) { - this.mode = mode; - } - - public boolean isRecursiveChown() { - return recursiveChown; - } - - public void setRecursiveChown(boolean recursiveChown) { - this.recursiveChown = recursiveChown; - } - - public boolean isRecursiveChmod() { - return recursiveChmod; - } - - public void setRecursiveChmod(boolean recursiveChmod) { - this.recursiveChmod = recursiveChmod; - } - - public boolean isChangePermissionOnParents() { - return changePermissionforParents; - } - - public void setChangePermissionOnParents(boolean changePermissionforParents) { - this.changePermissionforParents = changePermissionforParents; - } - - - - - @Override - public String toString() { - return "Resource [source=" + source + ", target=" + target + ", type=" - + type + ", action=" + action + ", owner=" + owner + ", group=" + group - + ", mode=" + mode + ", recursiveChown=" + recursiveChown - + ", recursiveChmod=" + recursiveChmod - + ", changePermissionforParents=" + changePermissionforParents + "]"; - } - - /* - * Check if parameters are correctly set - */ - public static void checkResourceParameters(Resource resource, - FileSystem dfs) throws IllegalArgumentException, IOException { - - ArrayList<String> actionsAvailable = new ArrayList<String>(); - actionsAvailable.add("create"); - actionsAvailable.add("delete"); - ArrayList<String> typesAvailable = new ArrayList<String>(); - typesAvailable.add("file"); - typesAvailable.add("directory"); - - if (resource.getTarget() == null) - throw new IllegalArgumentException( - "Path to resource in HadoopFs must be filled."); - - if (resource.getAction() == null - || !actionsAvailable.contains(resource.getAction())) - throw new IllegalArgumentException("Action is not supported."); - - if (resource.getType() == null - || !typesAvailable.contains(resource.getType())) - throw new IllegalArgumentException("Type is not supported."); - - // Check consistency for ("type":"file" == file in hadoop) - if (dfs.isFile(new Path(resource.getTarget())) - && !"file".equals(resource.getType())) - throw new IllegalArgumentException( - "Cannot create a directory " + resource.getTarget() + - " because file is present on the given path."); - // Check consistency for ("type":"directory" == directory in hadoop) - else if (dfs.isDirectory(new Path(resource.getTarget())) - && !"directory".equals(resource.getType())) - throw new IllegalArgumentException( - "Cannot create a file " + resource.getTarget() + - " because directory is present on the given path."); - - if(resource.getSource() != null) { - File source = new File(resource.getSource()); - if(source.isFile() - && !"file".equals(resource.getType())) - throw new IllegalArgumentException( - "Cannot create a directory " + resource.getTarget() + - " because source " + resource.getSource() + "is a file"); - else if(source.isDirectory() - && !"directory".equals(resource.getType())) - throw new IllegalArgumentException( - "Cannot create a file " + resource.getTarget() + - " because source " + resource.getSource() + "is a directory"); - } - } - - /* - * Create/copy resource - {type} - */ - public static void createResource(Resource resource, - FileSystem dfs, Path pathHadoop) throws IOException { - - boolean isCreate = (resource.getSource() == null) ? true : false; - - if (isCreate && resource.getType().equals("directory")) { - dfs.mkdirs(pathHadoop); // empty dir(s) - } else if (isCreate && resource.getType().equals("file")) { - dfs.createNewFile(pathHadoop); // empty file - } else { - dfs.copyFromLocalFile(new Path(resource.getSource()), pathHadoop);// copy - } - } - - /* - * Set permissions on resource - {mode} - */ - public static void setMode(Resource resource, - FileSystem dfs, Path pathHadoop) throws IOException { - - if (resource.getMode() != null) { - FsPermission permission = new FsPermission(resource.getMode()); - dfs.setPermission(pathHadoop, permission); - - // Recursive - - // Get the list of sub-directories and files - HashSet<String> resultSet = new HashSet<String>(); - - if (resource.isRecursiveChmod()) - resource.fillDirectoryList(dfs, resource.getTarget(), resultSet); - - if(resource.isChangePermissionOnParents()) - resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet); - - for (String path : resultSet) { - dfs.setPermission(new Path(path), permission); - } - - } - } - - /* - * Set owner on resource - {owner} - */ - public static void setOwner(Resource resource, FileSystem dfs, - Path pathHadoop) throws IOException { - - if (!(resource.getOwner() == null && resource.getGroup() == null)) { - dfs.setOwner(pathHadoop, resource.getOwner(), resource.getGroup()); - - // Get the list of sub-directories and files - HashSet<String> resultSet = new HashSet<String>(); - if (resource.isRecursiveChown()) - resource.fillDirectoryList(dfs, resource.getTarget(), resultSet); - if(resource.isChangePermissionOnParents()) - resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet); - - for (String path : resultSet) { - dfs.setOwner(new Path(path), resource.getOwner(), resource.getGroup()); - } - } - } - - public void fillInParentDirectories(FileSystem dfs, String path, HashSet<String> resultSet) throws IOException { - Path filePath = new Path(path); - - while(true) { - filePath = filePath.getParent(); - - // if(filePath.isRoot()) { - if(filePath.getParent() == null) { - break; - } - resultSet.add(filePath.toString()); - } - } - - /* - * List all files and sub-directories recursively - */ - public void fillDirectoryList(FileSystem dfs, String path, - HashSet<String> resultSet) throws IOException { - - FileStatus[] fileStatus = dfs.listStatus(new Path(path)); - if (fileStatus != null) { - // Go through all resources in directory - for (FileStatus fs : fileStatus) { - String pathToResource = path + "/" + fs.getPath().getName(); - - resultSet.add(pathToResource); - - if (fs.isDir()) { - // recursive - fillDirectoryList(dfs, pathToResource, resultSet); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java ---------------------------------------------------------------------- diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java deleted file mode 100644 index 291a2d9..0000000 --- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.fast_hdfs_resource; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; - -import com.google.gson.Gson; - -public class Runner { - public static void main(String[] args) - throws IOException, URISyntaxException { - // 1 - Check arguments - if (args.length != 1) { - System.err.println("Incorrect number of arguments. Please provide:\n" - + "1) Path to json file\n" - + "Exiting..."); - System.exit(1); - } - - // 2 - Check if json-file exists - final String jsonFilePath = args[0]; - File file = new File(jsonFilePath); - - if (!file.isFile()) { - System.err - .println("File " + jsonFilePath + " doesn't exist.\nExiting..."); - System.exit(1); - } - - Gson gson = new Gson(); - Resource[] resources = null; - FileSystem dfs = null; - - try { - Configuration conf = new Configuration(); - dfs = FileSystem.get(conf); - - // 3 - Load data from JSON - resources = (Resource[]) gson.fromJson(new FileReader(jsonFilePath), - Resource[].class); - - // 4 - Connect to HDFS - System.out.println("Using filesystem uri: " + FileSystem.getDefaultUri(conf).toString()); - dfs.initialize(FileSystem.getDefaultUri(conf), conf); - - for (Resource resource : resources) { - System.out.println("Creating: " + resource); - - Resource.checkResourceParameters(resource, dfs); - - Path pathHadoop = new Path(resource.getTarget()); - if (resource.getAction().equals("create")) { - // 5 - Create - Resource.createResource(resource, dfs, pathHadoop); - Resource.setMode(resource, dfs, pathHadoop); - Resource.setOwner(resource, dfs, pathHadoop); - } else if (resource.getAction().equals("delete")) { - // 6 - Delete - dfs.delete(pathHadoop, true); - } - } - - } finally { - dfs.close(); - } - - System.out.println("All resources created."); - } - -}
