[
https://issues.apache.org/jira/browse/HIVE-26012?focusedWorklogId=800977&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-800977
]
ASF GitHub Bot logged work on HIVE-26012:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 16/Aug/22 14:21
Start Date: 16/Aug/22 14:21
Worklog Time Spent: 10m
Work Description: dengzhhu653 commented on code in PR #3477:
URL: https://github.com/apache/hive/pull/3477#discussion_r946841781
##########
standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java:
##########
@@ -3717,4 +3683,135 @@ public void testDropDataConnectorIfNotExistsTrue()
throws Exception {
// No such data connector, ignore NoSuchObjectException
client.dropDataConnector("no_such_data_connector", true, false);
}
+
+ @Test
+ public void testIfFSWritesIsSkippedForDatabase() throws Throwable {
+ // create a database, check if the directory is created or not
+ // with skipFSWrites is true, the directory is not created
+ // with skipFSWrites is false, the directory is created
+ try {
+ // clear up any existing databases
+ silentDropDatabase(TEST_DB1_NAME);
+
+ String dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) +
"/testdb1.db";
+ String mgdLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/testdb1.db";
+
+ CreateDatabaseRequest req = new CreateDatabaseRequest();
+ req.setSkipFSWrites(true);
+ Database db = new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setLocation(dbLocation)
+ .setManagedLocation(mgdLocation)
+ .build(conf);
+ req.setDatabase(db);
+ req.setDatabaseName(TEST_DB1_NAME);
+ req.setLocationUri(dbLocation);
+ req.setManagedLocationUri(mgdLocation);
+ client.createDatabase(req);
+
+ String tblLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) +
"/testdb1.db/test_table";
+ String tblmgdLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) +
"/testdb1.db/test_table";
+
+ CreateTableRequest tblReq = new CreateTableRequest();
+ tblReq.setSkipFSWrites(true);
+ String tableName = "test_table";
+ Table table = new TableBuilder()
+ .setDbName(TEST_DB1_NAME)
+ .setTableName(tableName)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+ tblReq.setTable(table);
+ client.createTable(tblReq);
+
+ FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf);
+ assertFalse("Database's file system directory is skipped", fs.exists(new
Path(dbLocation)));
+ assertFalse("Table's file system directory is skipped", fs.exists(new
Path(tblLocation)));
+
+ fs = FileSystem.get(new Path(mgdLocation).toUri(), conf);
+ assertFalse("Database's managed location directory is skipped",
fs.exists(new Path(mgdLocation)));
+ assertFalse("Table's managed location directory is skipped",
fs.exists(new Path(tblmgdLocation)));
+
+ String dbLocationUri = "file:" + dbLocation.substring(7);
+ String mgdLocationUri = "file:" + mgdLocation.substring(7);
+ assertTrue("Database's dbLocation has been set",
db.getLocationUri().equals(dbLocationUri));
+ assertTrue("Database's managed location has been set",
db.getManagedLocationUri().equals(mgdLocationUri));
+ } catch (Throwable e) {
+ LOG.info(StringUtils.stringifyException(e));
+ e.printStackTrace();
+ LOG.info("testIfFSWritesIsSkippedForDatabase() failed.");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIfFSWritesIsSkippedForTable() throws Throwable {
+ // create a database, check if the directory is created or not
+ // with true, the directory is not created
+ // with false, the directory is created
+ try {
+ // clear up any existing databases
+ silentDropDatabase(TEST_DB1_NAME);
+ String tableName1 = "test_table1";
+ String tableName2 = "test_table2";
+
+ String dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) +
"/testdb1.db";
+ String mgdLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/testdb1.db";
+
+ Database db = new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setLocation(dbLocation)
+ .setManagedLocation(mgdLocation)
+ .build(conf);
+ client.createDatabase(db);
+
+ Table tbl1 = new TableBuilder()
+ .setDbName(TEST_DB1_NAME)
+ .setTableName(tableName1)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ Table tbl2 = new TableBuilder()
+ .setDbName(TEST_DB1_NAME)
+ .setTableName(tableName2)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ CreateTableRequest tblReq1 = new CreateTableRequest();
+ tblReq1.setTable(tbl1);
+ tblReq1.setSkipFSWrites(true);
+ String tbl1Location =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) +
"/testdb1.db/test_table1";
+ String tbl1mgdLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) +
"/testdb1.db/test_table1";
+
+ CreateTableRequest tblReq2 = new CreateTableRequest();
+ tblReq2.setTable(tbl2);
+ tblReq2.setSkipFSWrites(false);
+ String tbl2Location =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) +
"/testdb1.db/test_table2";
+ String tbl2mgdLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) +
"/testdb1.db/test_table2";
+
+ Path dbPath = new Path(db.getLocationUri());
+ FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf);
+ assertFalse("Table1's file system directory is skipped", fs.exists(new
Path(tbl1Location)));
+ assertFalse("Table2's file system directory is not skipped",
fs.exists(new Path(tbl2Location)));
Review Comment:
assertFalse -> assertTrue?
Issue Time Tracking
-------------------
Worklog Id: (was: 800977)
Time Spent: 4.5h (was: 4h 20m)
> HMS APIs to be enhanced for metadata replication
> ------------------------------------------------
>
> Key: HIVE-26012
> URL: https://issues.apache.org/jira/browse/HIVE-26012
> Project: Hive
> Issue Type: Improvement
> Components: Metastore
> Affects Versions: 3.1.0
> Reporter: Naveen Gangam
> Assignee: Hongdan Zhu
> Priority: Major
> Labels: pull-request-available
> Attachments: HMS APIs to be enhanced for metadata replication.docx
>
> Time Spent: 4.5h
> Remaining Estimate: 0h
>
> HMS currently has APIs like these that automatically create/delete the
> directories on the associated DFS.
> [create/drop]_database
> [create/drop]_table*
> [add/append/drop]_partition*
> This is expected and should be this way when query processors use this APIs.
> However, when tools that replicate hive metadata use this APIs on the target
> cluster, creating these dirs on target side which cause the replication of
> DFS-snapshots to fail.
> So we if provide an option to bypass this creation of dirs, dfs replications
> will be smoother. In the future we will need to restrict users that can use
> these APIs. So we will have some sort of an authorization policy.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
