OOZIE-2393 Allow table drop in hcat prepare (abhishekbafna)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5ed59675 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5ed59675 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5ed59675 Branch: refs/heads/oya Commit: 5ed596750d9e902f2c073c678f10714e30e243e0 Parents: c979e55 Author: abhisek bafna <aba...@hortonworks.com> Authored: Wed Mar 22 17:54:10 2017 +0530 Committer: abhisek bafna <aba...@hortonworks.com> Committed: Wed Mar 22 17:54:10 2017 +0530 ---------------------------------------------------------------------- .../apache/oozie/dependency/HCatURIHandler.java | 29 ++++++-- .../action/hadoop/TestFsActionExecutor.java | 14 ++++ .../hadoop/TestLauncherHCatURIHandler.java | 28 ++++++-- .../apache/oozie/coord/TestHCatELFunctions.java | 72 +++++++++++++++++++- .../oozie/dependency/TestHCatURIHandler.java | 21 ++++-- .../org/apache/oozie/test/MiniHCatServer.java | 23 +++++++ .../org/apache/oozie/test/XHCatTestCase.java | 8 +++ .../src/site/twiki/DG_HCatalogIntegration.twiki | 9 ++- .../src/site/twiki/WorkflowFunctionalSpec.twiki | 36 +++++++--- release-log.txt | 1 + .../action/hadoop/HCatLauncherURIHandler.java | 6 +- .../java/org/apache/oozie/util/HCatURI.java | 34 ++++----- .../java/org/apache/oozie/util/TestHCatURI.java | 14 ++++ 13 files changed, 248 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java index 67b37ec..4cc284a 100644 --- a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java +++ b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java @@ -176,7 +176,11 @@ public class HCatURIHandler implements URIHandler { HCatClient client = ((HCatContext) context).getHCatClient(); try { HCatURI hcatUri = new HCatURI(uri); - client.dropPartitions(hcatUri.getDb(), hcatUri.getTable(), hcatUri.getPartitionMap(), true); + if (!hcatUri.getPartitionMap().isEmpty()) { + client.dropPartitions(hcatUri.getDb(), hcatUri.getTable(), hcatUri.getPartitionMap(), true); + } else { + client.dropTable(hcatUri.getDb(), hcatUri.getTable(), true); + } } catch (URISyntaxException e) { throw new HCatAccessorException(ErrorCode.E1501, e); @@ -189,10 +193,16 @@ public class HCatURIHandler implements URIHandler { @Override public void delete(URI uri, Configuration conf, String user) throws URIHandlerException { HCatClientWithToken client = null; + HCatClient hCatClient = null; try { HCatURI hcatUri = new HCatURI(uri); client = getHCatClient(uri, conf, user); - client.getHCatClient().dropPartitions(hcatUri.getDb(), hcatUri.getTable(), hcatUri.getPartitionMap(), true); + hCatClient = client.getHCatClient(); + if (!hcatUri.getPartitionMap().isEmpty()) { + hCatClient.dropPartitions(hcatUri.getDb(), hcatUri.getTable(), hcatUri.getPartitionMap(), true); + } else { + hCatClient.dropTable(hcatUri.getDb(), hcatUri.getTable(), true); + } } catch (URISyntaxException e){ throw new HCatAccessorException(ErrorCode.E1501, e); @@ -201,7 +211,7 @@ public class HCatURIHandler implements URIHandler { throw new HCatAccessorException(ErrorCode.E1501, e); } finally { - closeQuietly(client.getHCatClient(), client.getDelegationToken(),true); + closeQuietly(hCatClient, client != null ? client.getDelegationToken() : null, true); } } @@ -325,10 +335,17 @@ public class HCatURIHandler implements URIHandler { private boolean exists(URI uri, HCatClient client, boolean closeClient) throws HCatAccessorException { try { + boolean flag; HCatURI hcatURI = new HCatURI(uri.toString()); - List<HCatPartition> partitions = client.getPartitions(hcatURI.getDb(), hcatURI.getTable(), - hcatURI.getPartitionMap()); - return (partitions != null && !partitions.isEmpty()); + if (!hcatURI.getPartitionMap().isEmpty()) { + List<HCatPartition> partitions = client.getPartitions(hcatURI.getDb(), hcatURI.getTable(), + hcatURI.getPartitionMap()); + flag = partitions != null && !partitions.isEmpty(); + } else { + List<String> tables = client.listTableNamesByPattern(hcatURI.getDb(), hcatURI.getTable()); + flag = tables != null && !tables.isEmpty(); + } + return (flag); } catch (ConnectionFailureException e) { throw new HCatAccessorException(ErrorCode.E1501, e); http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java index b27ede7..09d723a 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java @@ -320,6 +320,20 @@ public class TestFsActionExecutor extends ActionExecutorTestCase { assertFalse(handler.exists(hcatURI, conf, getTestUser())); } + public void testDeleteHcatTable() throws Exception { + createTestTable(); + URI hcatURI = getHCatURI(db, table); + URIHandler handler = uriService.getURIHandler(hcatURI); + FsActionExecutor ae = new FsActionExecutor(); + Path path = new Path(hcatURI); + Path nameNodePath = new Path(getNameNodeUri()); + Context context = createContext("<fs/>"); + XConfiguration conf = new XConfiguration(); + assertTrue(handler.exists(hcatURI, conf, getTestUser())); + ae.delete(context, conf, nameNodePath, path, true); + assertFalse(handler.exists(hcatURI, conf, getTestUser())); + } + public void testDeleteWithGlob() throws Exception { FsActionExecutor ae = new FsActionExecutor(); http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java index 66d4ecc..4288974 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java @@ -64,9 +64,9 @@ public class TestLauncherHCatURIHandler extends XHCatTestCase { createTable(db, table, "year,month,dt,country"); } - private void dropTestTable() throws Exception { - dropTable(db, table, false); - dropDatabase(db, false); + private void dropTestTable(boolean ifExists) throws Exception { + dropTable(db, table, ifExists); + dropDatabase(db, ifExists); } @Test @@ -128,7 +128,27 @@ public class TestLauncherHCatURIHandler extends XHCatTestCase { assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size()); assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size()); - dropTestTable(); + dropTestTable(false); + } + + public void testDeleteTable() throws Exception { + try { + createTestTable(); + createPartitionForTestDelete(true, true); + + URI hcatURI = getHCatURI(db, table, "year=2012;month=12;dt=02;country=us"); + URIHandler uriHandler = uriService.getURIHandler(hcatURI); + assertTrue(uriHandler.exists(hcatURI, conf, getTestUser())); + + hcatURI = getHCatURI(db, table); + assertTrue(uriHandler.exists(hcatURI, conf, getTestUser())); + LauncherURIHandlerFactory uriHandlerFactory = new LauncherURIHandlerFactory(uriService.getLauncherConfig()); + LauncherURIHandler handler = uriHandlerFactory.getURIHandler(hcatURI); + handler.delete(hcatURI, conf); + assertFalse(uriHandler.exists(hcatURI, conf, getTestUser())); + } finally { + dropTestTable(true); + } } private void createPartitionForTestDelete(boolean partition1, boolean partition2) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java index e1cf133..e41df87 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java +++ b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java @@ -65,7 +65,7 @@ public class TestHCatELFunctions extends XHCatTestCase { } @Test - public void testHCatExists() throws Exception { + public void testHCatPartitionExists() throws Exception { dropTable("db1", "table1", true); dropDatabase("db1", true); createDatabase("db1"); @@ -112,6 +112,53 @@ public class TestHCatELFunctions extends XHCatTestCase { dropDatabase("db1", true); } + @Test + public void testHCatTableExists() throws Exception { + dropTable("db1", "table1", true); + dropDatabase("db1", true); + createDatabase("db1"); + createTable("db1", "table1"); + + Configuration protoConf = new Configuration(); + protoConf.set(OozieClient.USER_NAME, getTestUser()); + protoConf.set("hadoop.job.ugi", getTestUser() + "," + "group"); + Configuration conf = new XConfiguration(); + conf.set(OozieClient.APP_PATH, "appPath"); + conf.set(OozieClient.USER_NAME, getTestUser()); + + conf.set("test.dir", getTestCaseDir()); + conf.set("table1", getHCatURI("db1", "table1").toString()); + conf.set("table2", getHCatURI("db1", "table2").toString()); + + LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef( + LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", + LiteWorkflowStoreService.LiteControlNodeHandler.class)); + LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId"); + + WorkflowJobBean wf = new WorkflowJobBean(); + wf.setId(job.getId()); + wf.setAppName("name"); + wf.setAppPath("appPath"); + wf.setUser(getTestUser()); + wf.setGroup("group"); + wf.setWorkflowInstance(job); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + protoConf.writeXml(baos); + wf.setProtoActionConf(baos.toString()); + + WorkflowActionBean action = new WorkflowActionBean(); + action.setId("actionId"); + action.setName("actionName"); + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow"); + DagELFunctions.configureEvaluator(eval, wf, action); + + assertEquals(true, (boolean) eval.evaluate("${hcat:exists(wf:conf('table1'))}", Boolean.class)); + assertEquals(false, (boolean) eval.evaluate("${hcat:exists(wf:conf('table2'))}", Boolean.class)); + + dropTable("db1", "table1", true); + dropDatabase("db1", true); + } + /** * Test HCat databaseIn and databaseOut EL functions (phase 1) which echo * back the EL function itself @@ -374,6 +421,17 @@ public class TestHCatELFunctions extends XHCatTestCase { eval.setVariable(".dataout.ABC.unresolved", Boolean.FALSE); expr = "${coord:databaseOut('ABC')}"; assertEquals("mydb", CoordELFunctions.evalAndWrap(eval, expr)); + + init("coord-action-start", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".datain.ABC", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".datain.ABC.unresolved", Boolean.FALSE); + expr = "${coord:databaseIn('ABC')}"; + assertEquals("mydb", CoordELFunctions.evalAndWrap(eval, expr)); + + eval.setVariable(".dataout.ABC", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".dataout.ABC.unresolved", Boolean.FALSE); + expr = "${coord:databaseOut('ABC')}"; + assertEquals("mydb", CoordELFunctions.evalAndWrap(eval, expr)); } /** @@ -395,6 +453,17 @@ public class TestHCatELFunctions extends XHCatTestCase { expr = "${coord:tableOut('ABC')}"; assertEquals("clicks", CoordELFunctions.evalAndWrap(eval, expr)); + init("coord-action-start", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".datain.ABC", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".datain.ABC.unresolved", Boolean.FALSE); + expr = "${coord:tableIn('ABC')}"; + assertEquals("clicks", CoordELFunctions.evalAndWrap(eval, expr)); + + eval.setVariable(".dataout.ABC", "hcat://hcat.server.com:5080/mydb/clicks"); + eval.setVariable(".dataout.ABC.unresolved", Boolean.FALSE); + expr = "${coord:tableOut('ABC')}"; + assertEquals("clicks", CoordELFunctions.evalAndWrap(eval, expr)); + init("coord-sla-create", "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us"); eval.setVariable(".dataout.ABC", "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us"); eval.setVariable(".dataout.ABC.unresolved", Boolean.FALSE); @@ -581,5 +650,4 @@ public class TestHCatELFunctions extends XHCatTestCase { appInst.setName("mycoordinator-app"); CoordELFunctions.configureEvaluator(eval, ds, appInst); } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java index 615f5e1..6fd1a1a 100644 --- a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java +++ b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java @@ -61,9 +61,9 @@ public class TestHCatURIHandler extends XHCatTestCase { createTable(db, table, "year,month,dt,country"); } - private void dropTestTable() throws Exception { - dropTable(db, table, false); - dropDatabase(db, false); + private void dropTestTable(boolean ifExists) throws Exception { + dropTable(db, table, ifExists); + dropDatabase(db, ifExists); } @Test @@ -105,7 +105,20 @@ public class TestHCatURIHandler extends XHCatTestCase { ((HCatURIHandler)handler).delete(hcatURI, conf, getTestUser()); assertFalse(handler.exists(hcatURI, conf, getTestUser())); - dropTestTable(); + dropTestTable(false); } + public void testDeleteTable() throws Exception { + try { + createTestTable(); + URI hcatURI = getHCatURI(db, table); + URIHandler handler = uriService.getURIHandler(hcatURI); + + assertTrue(handler.exists(hcatURI, conf, getTestUser())); + ((HCatURIHandler) handler).delete(hcatURI, conf, getTestUser()); + assertFalse(handler.exists(hcatURI, conf, getTestUser())); + } finally { + dropTestTable(true); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java index 9b7a004..b2f7282 100644 --- a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java +++ b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java @@ -206,6 +206,12 @@ public class MiniHCatServer { return new URI(uri.toString()); } + public URI getHCatURI(String db, String table) throws URISyntaxException { + StringBuilder uri = new StringBuilder(); + uri.append("hcat://").append(getMetastoreAuthority()).append("/").append(db).append("/").append(table); + return new URI(uri.toString()); + } + public void createDatabase(String db, String location) throws Exception { HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(db).ifNotExists(true).location(location).build(); hcatClient.createDatabase(dbDesc); @@ -233,6 +239,23 @@ public class MiniHCatServer { assertTrue(tables.contains(table)); } + public void createTable(String db, String table) throws Exception { + List<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); + cols.add(new HCatFieldSchema("userid", Type.INT, "userid")); + cols.add(new HCatFieldSchema("viewtime", Type.BIGINT, "view time")); + cols.add(new HCatFieldSchema("pageurl", Type.STRING, "page url visited")); + cols.add(new HCatFieldSchema("ip", Type.STRING, "IP Address of the User")); + + // Remove this once NotificationListener is fixed and available in HCat snapshot + Map<String, String> tblProps = new HashMap<String, String>(); + tblProps.put(HCatConstants.HCAT_MSGBUS_TOPIC_NAME, "hcat." + db + "." + table); + HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(db, table, cols).fileFormat("textfile") + .tblProps(tblProps ).build(); + hcatClient.createTable(tableDesc); + List<String> tables = hcatClient.listTableNamesByPattern(db, "*"); + assertTrue(tables.contains(table)); + } + public void dropDatabase(String db, boolean ifExists) throws Exception { hcatClient.dropDatabase(db, ifExists, DropDBMode.CASCADE); List<String> dbNames = hcatClient.listDatabaseNamesByPattern(db); http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java index 2adbee7..58f4bea 100644 --- a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java @@ -61,6 +61,10 @@ public abstract class XHCatTestCase extends XFsTestCase { return hcatServer.getHCatURI(db, table, partitions); } + protected URI getHCatURI(String db, String table) throws URISyntaxException { + return hcatServer.getHCatURI(db, table); + } + protected void createDatabase(String db) throws Exception { if (db.equals("default")) return; @@ -71,6 +75,10 @@ public abstract class XHCatTestCase extends XFsTestCase { hcatServer.createTable(db, table, partitionCols); } + protected void createTable(String db, String table) throws Exception { + hcatServer.createTable(db, table); + } + protected void dropDatabase(String db, boolean ifExists) throws Exception { if (db.equals("default")) return; http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/docs/src/site/twiki/DG_HCatalogIntegration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_HCatalogIntegration.twiki b/docs/src/site/twiki/DG_HCatalogIntegration.twiki index fd6bcb6..39df67c 100644 --- a/docs/src/site/twiki/DG_HCatalogIntegration.twiki +++ b/docs/src/site/twiki/DG_HCatalogIntegration.twiki @@ -32,7 +32,7 @@ template is specified as a dataset and input events are defined in Coordinator f availability checks by polling the HDFS directory URIs resolved based on the nominal time. When all the data dependencies are met, the Coordinator's workflow is triggered which then consumes the available HDFS data. -With addition of HCatalog support, Coordinators also support specifying a set of HCatalog table partitions as a dataset. +With addition of HCatalog support, Coordinators also support specifying a set of HCatalog tables or table partitions as a dataset. The workflow is triggered when the HCatalog table partitions are available and the workflow actions can then read the partition data. A mix of HDFS and HCatalog dependencies can be specified as input data dependencies. Similar to HDFS directories, HCatalog table partitions can also be specified as output dataset events. @@ -58,7 +58,10 @@ documentation for the Oozie server side configuration required to support HCatal Oozie supports specifying HCatalog partitions as a data dependency through a URI notation. The HCatalog partition URI is used to identify a set of table partitions: hcat://bar:8020/logsDB/logsTable/dt=20090415;region=US. -The format to specify a HCatalog table partition URI is +The format to specify a HCatalog table URI is: +hcat://[metastore server]:[port]/[database name]/[table name] + +The format to specify a HCatalog table partition URI is: hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value];... For example, @@ -94,7 +97,7 @@ Refer to [[CoordinatorFunctionalSpec][Coordinator Functional Specification]] for ---+++ Workflow Refer to [[WorkflowFunctionalSpec][Workflow Functional Specification]] for more information about - * how to drop HCatalog partitions in the prepare block of a action + * how to drop HCatalog table/partitions in the prepare block of a action * the HCatalog EL functions available to use in workflows Refer to [[DG_ActionAuthentication][Action Authentication]] for more information about http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/docs/src/site/twiki/WorkflowFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki index 032b988..460f6e5 100644 --- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki +++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki @@ -777,8 +777,9 @@ Pipe properties can be overridden by specifying them in the =job-xml= file or =c </verbatim> The =prepare= element, if present, indicates a list of paths to delete before starting the job. This should be used -exclusively for directory cleanup or dropping of hcatalog table partitions for the job to be executed. The delete operation -will be performed in the =fs.default.name= filesystem for hdfs URIs. The format to specify a hcatalog table partition URI is +exclusively for directory cleanup or dropping of hcatalog table or table partitions for the job to be executed. The delete operation +will be performed in the =fs.default.name= filesystem for hdfs URIs. The format for specifying hcatalog table URI is +hcat://[metastore server]:[port]/[database name]/[table name] and format to specify a hcatalog table partition URI is hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value]. In case of a hcatalog URI, the hive-site.xml needs to be shipped using =file= tag and the hcatalog and hive jars need to be placed in workflow lib directory or specified using =archive= tag. @@ -1035,8 +1036,9 @@ section [#FilesArchives][Adding Files and Archives for the Job]. </verbatim> The =prepare= element, if present, indicates a list of paths to delete before starting the job. This should be used -exclusively for directory cleanup or dropping of hcatalog table partitions for the job to be executed. -The format to specify a hcatalog table partition URI is +exclusively for directory cleanup or dropping of hcatalog table or table partitions for the job to be executed. The delete operation +will be performed in the =fs.default.name= filesystem for hdfs URIs. The format for specifying hcatalog table URI is +hcat://[metastore server]:[port]/[database name]/[table name] and format to specify a hcatalog table partition URI is hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value]. In case of a hcatalog URI, the hive-site.xml needs to be shipped using =file= tag and the hcatalog and hive jars need to be placed in workflow lib directory or specified using =archive= tag. @@ -1182,8 +1184,13 @@ executed. Thus there is less chance of an error occurring while the =fs= action The =delete= command deletes the specified path, if it is a directory it deletes recursively all its content and then deletes the directory. By default it does skip trash. It can be moved to trash by setting the value of skip-trash to -'false'. It can also be used to drop hcat partitions. This is the only FS command which supports HCatalog URIs as well. -For eg: <verbatim><delete path='hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value];...'/></verbatim> +'false'. It can also be used to drop hcat tables/partitions. This is the only FS command which supports HCatalog URIs as well. +For eg: +<verbatim> +<delete path='hcat://[metastore server]:[port]/[database name]/[table name]'/> +OR +<delete path='hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value];...'/> +</verbatim> The =mkdir= command creates the specified directory, it creates all missing directories in the path. If the directory already exist it does a no-op. @@ -1539,9 +1546,10 @@ application must consider the possibility of collisions with activity spawned by </verbatim> The =prepare= element, if present, indicates a list of paths to delete before starting the Java application. This should -be used exclusively for directory cleanup or dropping of hcatalog table partitions for the Java application to be executed. +be used exclusively for directory cleanup or dropping of hcatalog table or table partitions for the Java application to be executed. In case of =delete=, a glob pattern can be used to specify path. -The format to specify a hcatalog table partition URI is +The format for specifying hcatalog table URI is +hcat://[metastore server]:[port]/[database name]/[table name] and format to specify a hcatalog table partition URI is hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value]. In case of a hcatalog URI, the hive-site.xml needs to be shipped using =file= tag and the hcatalog and hive jars need to be placed in workflow lib directory or specified using =archive= tag. @@ -2244,9 +2252,15 @@ It returns the block size in bytes of specified file. If the path is not a file, ---++++ 4.2.8 HCatalog EL Functions -For all the functions in this section the URI must be a hcatalog URI identifying a set of partitions in a table. -The format to specify a hcatalog table partition URI is -hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value]. For example: <pre>hcat://foo:8020/mydb/mytable/region=us;dt=20121212</pre> +For all the functions in this section the URI must be a hcatalog URI identifying a table or set of partitions in a table. +The format for specifying hcatalog table URI is +hcat://[metastore server]:[port]/[database name]/[table name] and format to specify a hcatalog table partition URI is +hcat://[metastore server]:[port]/[database name]/[table name]/[partkey1]=[value];[partkey2]=[value]. For example: + +<pre> +hcat://foo:8020/mydb/mytable +hcat://foo:8020/mydb/mytable/region=us;dt=20121212 +</pre> *boolean hcat:exists(String uri)* http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index dc459b7..4a459cb 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2393 Allow table drop in hcat prepare (abhishekbafna) OOZIE-2835 TestIOUtils shall not be an XTestCase (asasvari via pbacsko) OOZIE-2817 Increase test case stability in pre-commit job (gezapeti) OOZIE-2820 Fix more spelling errors in exceptions and logging (lzeke via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/sharelib/hcatalog/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java ---------------------------------------------------------------------- diff --git a/sharelib/hcatalog/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java b/sharelib/hcatalog/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java index 6798768..7edb40d 100644 --- a/sharelib/hcatalog/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java +++ b/sharelib/hcatalog/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java @@ -51,7 +51,11 @@ public class HCatLauncherURIHandler implements LauncherURIHandler { HCatClient client = getHCatClient(uri, conf); try { HCatURI hcatURI = new HCatURI(uri.toString()); - client.dropPartitions(hcatURI.getDb(), hcatURI.getTable(), hcatURI.getPartitionMap(), true); + if (!hcatURI.getPartitionMap().isEmpty()) { + client.dropPartitions(hcatURI.getDb(), hcatURI.getTable(), hcatURI.getPartitionMap(), true); + } else { + client.dropTable(hcatURI.getDb(), hcatURI.getTable(), true); + } System.out.println("Dropped partitions for " + uri); return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java ---------------------------------------------------------------------- diff --git a/sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java b/sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java index 8ec3fae..faeff2a 100644 --- a/sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java +++ b/sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java @@ -37,7 +37,7 @@ public class HCatURI { private URI uri; private String db; private String table; - private Map<String, String> partitions; + private Map<String, String> partitions = new LinkedHashMap<String, String>(); /** * Constructor using default configuration @@ -63,13 +63,12 @@ public class HCatURI { String[] paths = uri.getPath().split(PATH_SEPARATOR); - if (paths.length != 4) { + if (paths.length < 3 || paths.length > 4) { throw new URISyntaxException(uri.toString(), "URI path is not in expected format"); } db = paths[1]; table = paths[2]; - String partRaw = paths[3]; if (db == null || db.length() == 0) { throw new URISyntaxException(uri.toString(), "DB name is missing"); @@ -77,22 +76,25 @@ public class HCatURI { if (table == null || table.length() == 0) { throw new URISyntaxException(uri.toString(), "Table name is missing"); } - if (partRaw == null || partRaw.length() == 0) { - throw new URISyntaxException(uri.toString(), "Partition details are missing"); - } - partitions = new LinkedHashMap<String, String>(); - String[] parts = partRaw.split(PARTITION_SEPARATOR); - for (String part : parts) { - if (part == null || part.length() == 0) { - continue; + if (paths.length == 4) { + String partRaw = paths[3]; + if (partRaw == null || partRaw.length() == 0) { + throw new URISyntaxException(uri.toString(), "Partition details are missing"); } - String[] keyVal = part.split(PARTITION_KEYVAL_SEPARATOR); - if (keyVal.length != 2) { - throw new URISyntaxException(uri.toString(), "Partition key value pair is not specified properly in (" - + part + ")"); + + String[] parts = partRaw.split(PARTITION_SEPARATOR); + for (String part : parts) { + if (part == null || part.length() == 0) { + continue; + } + String[] keyVal = part.split(PARTITION_KEYVAL_SEPARATOR); + if (keyVal.length != 2) { + throw new URISyntaxException(uri.toString(), "Partition key value pair is not specified properly in (" + + part + ")"); + } + partitions.put(keyVal[0], keyVal[1]); } - partitions.put(keyVal[0], keyVal[1]); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5ed59675/sharelib/hcatalog/src/test/java/org/apache/oozie/util/TestHCatURI.java ---------------------------------------------------------------------- diff --git a/sharelib/hcatalog/src/test/java/org/apache/oozie/util/TestHCatURI.java b/sharelib/hcatalog/src/test/java/org/apache/oozie/util/TestHCatURI.java index 1b1a8fa..ce690da 100644 --- a/sharelib/hcatalog/src/test/java/org/apache/oozie/util/TestHCatURI.java +++ b/sharelib/hcatalog/src/test/java/org/apache/oozie/util/TestHCatURI.java @@ -43,7 +43,21 @@ public class TestHCatURI { assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); assertEquals(uri.getPartitionValue("region"), "us"); + } + @Test + public void testHCatTableURI() { + String input = "hcat://hcat.server.com:5080/mydb/clicks"; + HCatURI uri = null; + try { + uri = new HCatURI(input); + } + catch (Exception ex) { + System.err.print(ex.getMessage()); + } + assertEquals(uri.getServerEndPoint(), "hcat://hcat.server.com:5080"); + assertEquals(uri.getDb(), "mydb"); + assertEquals(uri.getTable(), "clicks"); } @Test(expected = URISyntaxException.class)
