Repository: oozie Updated Branches: refs/heads/master a5c839a3a -> 72bce837d
OOZIE-2769 Extend FS action to allow setrep on a file (Artem Ervits via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/72bce837 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/72bce837 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/72bce837 Branch: refs/heads/master Commit: 72bce837d038e0c61435c77c56fadd81dcd05cbb Parents: a5c839a Author: Gezapeti Cseh <gezap...@gmail.com> Authored: Tue Jun 20 09:25:01 2017 +0200 Committer: Gezapeti Cseh <gezap...@gmail.com> Committed: Tue Jun 20 09:25:01 2017 +0200 ---------------------------------------------------------------------- .../src/main/resources/oozie-workflow-0.5.xsd | 6 +++++ .../oozie/action/hadoop/FsActionExecutor.java | 24 ++++++++++++++++++++ .../action/hadoop/TestFsActionExecutor.java | 13 +++++++++++ core/src/test/resources/wf-schema-valid.xml | 5 +++- .../src/site/twiki/WorkflowFunctionalSpec.twiki | 8 ++++++- release-log.txt | 1 + 6 files changed, 55 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/client/src/main/resources/oozie-workflow-0.5.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/oozie-workflow-0.5.xsd b/client/src/main/resources/oozie-workflow-0.5.xsd index 8fe2b47..87f38a3 100644 --- a/client/src/main/resources/oozie-workflow-0.5.xsd +++ b/client/src/main/resources/oozie-workflow-0.5.xsd @@ -215,6 +215,7 @@ <xs:element name="chmod" type="workflow:CHMOD"/> <xs:element name="touchz" type="workflow:TOUCHZ"/> <xs:element name="chgrp" type="workflow:CHGRP"/> + <xs:element name="setrep" type="workflow:SETREP"/> </xs:choice> </xs:sequence> </xs:complexType> @@ -305,6 +306,11 @@ <xs:attribute name="dir-files" type="xs:string"/> </xs:complexType> + <xs:complexType name="SETREP"> + <xs:attribute name="path" type="xs:string" use="required"/> + <xs:attribute name="replication-factor" type="xs:short" use="required"/> + </xs:complexType> + <xs:complexType name="TOUCHZ"> <xs:attribute name="path" type="xs:string" use="required"/> </xs:complexType> http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index e1150f7..85d3efa 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -223,6 +223,16 @@ public class FsActionExecutor extends ActionExecutor { chgrp(context, fsConf, nameNodePath, path, context.getWorkflow().getUser(), group, dirFiles, recursive); } + else { + if (command.equals("setrep")) { + Path path = getPath(commandElement, "path"); + String replicationFactor = + commandElement.getAttributeValue("replication-factor"); + if (commandElement.getAttributeValue("replication-factor") != null) { + setrep(context, path, Short.parseShort(replicationFactor)); + } + } + } } } } @@ -654,6 +664,20 @@ public class FsActionExecutor extends ActionExecutor { } } + void setrep(Context context, Path path, short replicationFactor) + throws ActionExecutorException, HadoopAccessorException { + try { + path = resolveToFullPath(null, path, true); + FileSystem fs = getFileSystemFor(path, context, null); + + if (fs.isFile(path)) { + fs.setReplication(path, replicationFactor); + } + } catch (IOException ex) { + convertException(ex); + } + } + public boolean supportsConfigurationJobXML() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java index 99e4d91..6939b57 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java @@ -1214,4 +1214,17 @@ public void testChmodRecursive() throws Exception { assertEquals(ActionExecutorException.ErrorType.ERROR, ((ActionExecutorException) e).getErrorType()); } } + + public void testSetRep() throws Exception { + FsActionExecutor ae = new FsActionExecutor(); + FileSystem fs = getFileSystem(); + Path path = new Path(getFsTestCaseDir(), "dir1"); + Context context = createContext("<fs/>"); + fs.mkdirs(path); + Path file = new Path(path + "/file"); + fs.createNewFile(file); + final short replicationFactor = (short) 2; + ae.setrep(context, path, replicationFactor); + assertEquals(replicationFactor, fs.getFileStatus(file).getReplication()); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/test/resources/wf-schema-valid.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-schema-valid.xml b/core/src/test/resources/wf-schema-valid.xml index 71bbcc8..085634f 100644 --- a/core/src/test/resources/wf-schema-valid.xml +++ b/core/src/test/resources/wf-schema-valid.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -<workflow-app xmlns="uri:oozie:workflow:0.1" name="test-wf"> +<workflow-app xmlns="uri:oozie:workflow:0.5" name="test-wf"> <start to="a"/> <decision name="a"> <switch> @@ -100,6 +100,9 @@ <delete path='/tmp2'/> <mkdir path='/tmp3'/> <delete path='/tmp3'/> + <mkdir path='/tmp4'/> + <touchz path='/tmp4/file'/> + <setrep path='/tmp4/file' replication-factor='2'/> </fs> <ok to="z"/> <error to="b"/> http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/docs/src/site/twiki/WorkflowFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki index 6655696..5a75b99 100644 --- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki +++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki @@ -1139,7 +1139,7 @@ All the above elements can be parameterized (templatized) using EL expressions. ---++++ 3.2.4 Fs (HDFS) action The =fs= action allows to manipulate files and directories in HDFS from a workflow application. The supported commands -are =move=, =delete=, =mkdir=, =chmod=, =touchz= and =chgrp=. +are =move=, =delete=, =mkdir=, =chmod=, =touchz=, =setrep= and =chgrp=. The FS commands are executed synchronously from within the FS action, the workflow job will wait until the specified file commands are completed before continuing to the next action. @@ -1174,6 +1174,8 @@ executed. Thus there is less chance of an error occurring while the =fs= action <touchz path='[PATH]' /> ... <chgrp path='[PATH]' group='[GROUP]' dir-files='false' /> + ... + <setrep path='[PATH]' replication-factor='2'/> </fs> <ok to="[NODE-NAME]"/> <error to="[NODE-NAME]"/> @@ -1218,6 +1220,9 @@ within the directory. To apply the =chgrp= command to the directory, without aff the =dir-files= attribute must be set to =false=. To apply the =chgrp= command recursively to all levels within a directory, put a =recursive= element inside the <chgrp> element. +The =setrep= command changes replication factor of an hdfs file(s). Changing RF of directories or symlinks is not +supported; this action requires an argument for RF. + *Example:* <verbatim> @@ -1230,6 +1235,7 @@ To apply the =chgrp= command recursively to all levels within a directory, put a <move source='${jobInput}' target='archives/${wf:id()}/processed-input'/> <chmod path='${jobOutput}' permissions='-rwxrw-rw-' dir-files='true'><recursive/></chmod> <chgrp path='${jobOutput}' group='testgroup' dir-files='true'><recursive/></chgrp> + <setrep path='archives/${wf:id()/filename(s)}' replication-factor='2'/> </fs> <ok to="myotherjob"/> <error to="errorcleanup"/> http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 10c3d56..05ed49c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2769 Extend FS action to allow setrep on a file (Artem Ervits via gezapeti) OOZIE-2815 amend - Oozie not always display job log (andras.piros via gezapeti) OOZIE-2946 Include find-sec-bugs plugin (Jan Hentschel via rkanter) OOZIE-2950 TestOozieCLI.testSlaEvents is not testing SLAServlet. (gezapeti)
