[
https://issues.apache.org/jira/browse/HIVE-21764?focusedWorklogId=264512&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-264512
]
ASF GitHub Bot logged work on HIVE-21764:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 21/Jun/19 10:44
Start Date: 21/Jun/19 10:44
Worklog Time Spent: 10m
Work Description: maheshk114 commented on pull request #679: HIVE-21764 :
REPL DUMP should detect and bootstrap any rename table events where old table
was excluded but renamed table is included.
URL: https://github.com/apache/hive/pull/679#discussion_r296166521
##########
File path:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
##########
@@ -585,4 +585,143 @@ public void
testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws
.run("show tables")
.verifyResults(incrementalReplicatedTables);
}
+
+ @Test
+ public void testRenameTableScenariosBasic() throws Throwable {
+ String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ String lastReplId = replicateAndVerify(replPolicy, null, null, null,
+ null, new String[]{}, new String[]{});
+
+ String[] originalNonAcidTables = new String[]{"in1", "in2", "out3",
"out4", "out5", "out6"};
+ createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+
+ // Replicate and verify if only 2 tables are replicated to target.
+ String[] replicatedTables = new String[]{"in1", "in2"};
+ String[] bootstrapTables = new String[]{};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+ null, bootstrapTables, replicatedTables);
+
+ // Rename tables to make them satisfy the filter.
+ primary.run("use " + primaryDbName)
+ .run("alter table out3 rename to in3")
+ .run("alter table out4 rename to in4")
+ .run("alter table out5 rename to in5");
+
+ replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+ bootstrapTables = new String[]{"in3", "in4", "in5"};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+ null, bootstrapTables, replicatedTables);
+
+ primary.run("use " + primaryDbName)
+ .run("alter table in3 rename to in7")
+ .run("alter table in7 rename to in8") // Double rename, both
satisfying the filter, so no bootstrap.
+ .run("alter table in4 rename to out9") // out9 does not match the
filter so in4 should be dropped.
+ .run("alter table in5 rename to out10") // Rename from satisfying
name to not satisfying name.
+ .run("alter table out10 rename to in11");// from non satisfying to
satisfying, should be bootstrapped
+
+ replicatedTables = new String[]{"in1", "in2", "in8", "in11"};
+ bootstrapTables = new String[]{"in11"};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+ null, bootstrapTables, replicatedTables);
+
+ primary.run("use " + primaryDbName)
+ .run("alter table in8 rename to in12") // table is renamed from
satisfying to satisfying, no bootstrap
+ .run("alter table out9 rename to in13") // out9 does not match the
filter so in13 should be bootstrapped.
+ .run("alter table in13 rename to in14") // table is renamed from
satisfying to satisfying
+ .run("drop table in14"); // table in14 is dropped, so no
bootstrap should happen.
+
+ replicatedTables = new String[]{"in1", "in2", "in12", "in12"};
+ bootstrapTables = new String[]{};
+ replicateAndVerify(replPolicy, null, lastReplId, null,
+ null, bootstrapTables, replicatedTables);
+ }
+
+ @Test
+ public void testRenameTableScenariosAcidTable() throws Throwable {
+ String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ List<String> dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'",
+ "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
+ String lastReplId = replicateAndVerify(replPolicy, null, null,
dumpWithClause,
+ null, new String[]{}, new String[]{});
+
+ String[] originalNonAcidTables = new String[]{"in1", "out4"};
+ String[] originalFullAcidTables = new String[]{"in2", "out5"};
+ String[] originalMMAcidTables = new String[]{"out3", "out6"};
+ createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+ createTables(originalFullAcidTables, CreateTableType.FULL_ACID);
+ createTables(originalMMAcidTables, CreateTableType.MM_ACID);
+
+ // Replicate and verify if only 1 tables are replicated to target. Acid
tables are not dumped.
+ String[] replicatedTables = new String[]{"in1"};
+ String[] bootstrapTables = new String[]{};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId,
dumpWithClause,
+ null, bootstrapTables, replicatedTables);
+
+ // Rename tables to make them satisfy the filter and enable acid tables.
+ primary.run("use " + primaryDbName)
+ .run("alter table out3 rename to in3")
+ .run("alter table out4 rename to in4")
+ .run("alter table out5 rename to in5");
+
+ dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='true'",
+ "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+ replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+ bootstrapTables = new String[]{"in2", "in3", "in4", "in5"};
+ replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+ null, bootstrapTables, replicatedTables);
+ }
+
+ @Test
+ public void testRenameTableScenariosExternalTable() throws Throwable {
+ String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ List<String> loadWithClause =
ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE,
replica);
+ List<String> dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'",
+ "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname +
"'='false'",
+ "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname +
"'='false'",
+ "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
+ String lastReplId = replicateAndVerify(replPolicy, null, null,
dumpWithClause,
+ loadWithClause, new String[]{}, new String[]{});
+
+ String[] originalNonAcidTables = new String[]{"in1", "out4"};
+ String[] originalExternalTables = new String[]{"in2", "out5"};
+ String[] originalMMAcidTables = new String[]{"in3", "out6"};
+ createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+ createTables(originalExternalTables, CreateTableType.EXTERNAL);
+ createTables(originalMMAcidTables, CreateTableType.MM_ACID);
+
+ // Replicate and verify if only 1 tables are replicated to target. Acid
and external tables are not dumped.
+ String[] replicatedTables = new String[]{"in1"};
+ String[] bootstrapTables = new String[]{};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId,
dumpWithClause,
+ loadWithClause, bootstrapTables, replicatedTables);
+
+ // Rename tables to make them satisfy the filter and enable acid and
external tables.
+ primary.run("use " + primaryDbName)
+ .run("alter table out4 rename to in4")
+ .run("alter table out5 rename to in5");
+
+ dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
+ "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname +
"'='true'",
+ "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname +
"'='true'",
+ "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+ replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+ bootstrapTables = new String[]{"in2", "in3", "in4", "in5"};
+ lastReplId = replicateAndVerify(replPolicy, null, lastReplId,
dumpWithClause,
+ loadWithClause, null, replicatedTables);
+
+ dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
+ "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+
+ primary.run("use " + primaryDbName)
+ .run("alter table out6 rename to in6") // external table
bootstrap.
+ .run("alter table in5 rename to out7") // in5 should be deleted.
+ .run("alter table out7 rename to in7") // MM table bootstrap.
+ .run("alter table in1 rename to out10") // in1 should be deleted.
+ .run("alter table out10 rename to in11");// normal table
bootstrapped
+
+ replicatedTables = new String[]{"in2", "in3", "in4", "in11", "in6", "in7"};
+ bootstrapTables = new String[]{"in11", "in6", "in7"};
+ replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+ loadWithClause, bootstrapTables, replicatedTables);
+ }
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 264512)
Time Spent: 2h 50m (was: 2h 40m)
> REPL DUMP should detect and bootstrap any rename table events where old table
> was excluded but renamed table is included.
> -------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-21764
> URL: https://issues.apache.org/jira/browse/HIVE-21764
> Project: Hive
> Issue Type: Sub-task
> Components: repl
> Reporter: Sankar Hariappan
> Assignee: mahesh kumar behera
> Priority: Major
> Labels: DR, Replication, pull-request-available
> Attachments: HIVE-21764.01.patch
>
> Time Spent: 2h 50m
> Remaining Estimate: 0h
>
> REPL DUMP fetches the events from NOTIFICATION_LOG table based on regular
> expression + inclusion/exclusion list. So, in case of rename table event, the
> event will be ignored if old table doesn't match the pattern but the new
> table should be bootstrapped. REPL DUMP should have a mechanism to detect
> such tables and automatically bootstrap with incremental replication.Also, if
> renamed table is excluded from replication policy, then need to drop the old
> table at target as well.
> There are 4 scenarios that needs to be handled.
> # Both new name and old name satisfies the table name pattern filter.
> ## No need to do anything. The incremental event for rename should take care
> of the replication.
> # Both the names does not satisfy the table name pattern filter.
> ## Both the names are not in the scope of the policy and thus nothing needs
> to be done.
> # New name satisfies the pattern but the old name does not.
> ## The table will not be present at the target.
> ## Rename event handler for dump should detect this case and add the new
> table name to the list of table for bootstrap.
> ## All the events related to the table (new name) should be ignored.
> ## If there is a drop event for the table (with new name), then remove the
> table from the list of tables to be bootstrapped.
> ## In case of rename (double rename)
> ### If the new name satisfies the table pattern, then add the new name to
> the list of tables to be bootstrapped and remove the old name from the list
> of tables to be bootstrapped.
> ### If the new name does not satisfies then just removed the table name from
> the list of tables to be bootstrapped.
> # New name does not satisfies the pattern but the old name satisfies.
> ## Change the rename event to a drop event.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
