This is an automated email from the ASF dual-hosted git repository.
tdsilva pushed a commit to branch 4.14-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit f741006e7b5e32307cb2636a1a0bcb593116202a
Author: Kadir <kozde...@salesforce.com>
AuthorDate: Wed Dec 12 17:53:38 2018 -0800
PHOENIX-5025 Tool to clean up orphan views (addendum)
---
.../apache/phoenix/end2end/OrphanViewToolIT.java | 25 +++---
.../apache/phoenix/mapreduce/OrphanViewTool.java | 89 +++++++++++++---------
2 files changed, 71 insertions(+), 43 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java
index f9a1785..38d4afc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java
@@ -19,7 +19,6 @@ package org.apache.phoenix.end2end;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
-import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
@@ -27,9 +26,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
+import java.io.LineNumberReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -54,6 +53,7 @@ import org.slf4j.LoggerFactory;
@RunWith(Parameterized.class)
public class OrphanViewToolIT extends ParallelStatsDisabledIT {
+ private static final String SYSTEM_CHILD_LINK_NAME = SYSTEM_CATALOG_NAME;
private static final Logger LOG =
LoggerFactory.getLogger(OrphanViewToolIT.class);
private final boolean isMultiTenant;
@@ -206,9 +206,13 @@ public class OrphanViewToolIT extends
ParallelStatsDisabledIT {
}
private void verifyLineCount(String fileName, long lineCount) throws
IOException {
- if (Files.lines(Paths.get(fileName)).count() != lineCount)
- LOG.debug(Files.lines(Paths.get(fileName)).count() + " != " +
lineCount);
- assertTrue(Files.lines(Paths.get(fileName)).count() == lineCount);
+ LineNumberReader reader = new LineNumberReader(new
FileReader(fileName));
+ while (reader.readLine() != null) {
+ }
+ int count = reader.getLineNumber();
+ if (count != lineCount)
+ LOG.debug(count + " != " + lineCount);
+ assertTrue(count == lineCount);
}
private void verifyCountQuery(Connection connection, String query, String
schemaName, long count)
@@ -238,7 +242,6 @@ public class OrphanViewToolIT extends
ParallelStatsDisabledIT {
}
}
-
private void verifyNoChildLink(Connection connection, String
viewSchemaName) throws Exception {
// Verify that there there is no link in the system child link table
verifyCountQuery(connection, countChildLinksQuery, viewSchemaName, 0);
@@ -264,6 +267,7 @@ public class OrphanViewToolIT extends
ParallelStatsDisabledIT {
schemaName == null ? "IS NULL" : "= '" + schemaName + "'"));
connection.commit();
}
+
@Test
public void testDeleteBaseTableRows() throws Exception {
String baseTableName = generateUniqueName();
@@ -438,7 +442,8 @@ public class OrphanViewToolIT extends
ParallelStatsDisabledIT {
}
}
- public static String[] getArgValues(boolean clean, boolean identify,
boolean outputPath, boolean inputPath) {
+ public static String[] getArgValues(boolean clean, boolean identify,
boolean outputPath, boolean inputPath)
+ throws InterruptedException{
final List<String> args = Lists.newArrayList();
if (outputPath) {
args.add("-op");
@@ -454,8 +459,10 @@ public class OrphanViewToolIT extends
ParallelStatsDisabledIT {
if (identify) {
args.add("-i");
}
+ final long ageMs = 2000;
+ Thread.sleep(ageMs);
args.add("-a");
- args.add("0");
+ args.add(Long.toString(ageMs));
return args.toArray(new String[0]);
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java
index a8a30b6..2e0dd0d 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java
@@ -21,7 +21,6 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
-import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
@@ -71,6 +70,7 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.SchemaUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory;
*
*/
public class OrphanViewTool extends Configured implements Tool {
+ private static final String SYSTEM_CHILD_LINK_NAME = SYSTEM_CATALOG_NAME;
private static final Logger LOG =
LoggerFactory.getLogger(OrphanViewTool.class);
// Query all the views that are not "MAPPED" views
private static final String viewQuery = "SELECT " +
@@ -394,26 +395,48 @@ public class OrphanViewTool extends Configured implements
Tool {
}
private void gracefullyDropView(PhoenixConnection phoenixConnection,
Configuration configuration,
- Key key) throws Exception {
- PhoenixConnection tenantConnection;
- if (key.getTenantId() != null) {
- Properties tenantProps = new Properties();
- tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB,
key.getTenantId());
- tenantConnection =
ConnectionUtil.getInputConnection(configuration, tenantProps).
- unwrap(PhoenixConnection.class);
- } else {
- tenantConnection = phoenixConnection;
- }
-
- MetaDataClient client = new MetaDataClient(tenantConnection);
- org.apache.phoenix.parse.TableName pTableName =
org.apache.phoenix.parse.TableName
- .create(key.getSchemaName(), key.getTableName());
+ Key key) throws Exception {
+ PhoenixConnection tenantConnection = null;
+ boolean newConn = false;
try {
- client.dropTable(
- new DropTableStatement(pTableName, PTableType.VIEW, false,
true, true));
+ if (key.getTenantId() != null) {
+ Properties tenantProps = new Properties();
+ tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB,
key.getTenantId());
+ tenantConnection =
ConnectionUtil.getInputConnection(configuration, tenantProps).
+ unwrap(PhoenixConnection.class);
+ newConn = true;
+ } else {
+ tenantConnection = phoenixConnection;
+ }
+ String fullViewName = SchemaUtil.getTableName(key.getSchemaName(),
key.getTableName());
+ String dropTable = String.format("DROP VIEW IF EXISTS %s CASCADE",
fullViewName);
+ try {
+ tenantConnection.createStatement().execute(dropTable);
+ tenantConnection.commit();
+ }
+ catch (TableNotFoundException e) {
+ LOG.info("Ignoring view " + fullViewName + " as it has already
been dropped");
+ }
+ } finally {
+ if (newConn) {
+ tryClosingConnection(tenantConnection);
+ }
}
- catch (TableNotFoundException e) {
- LOG.info("Ignoring view " + pTableName + " as it has already been
dropped");
+ }
+
+ /**
+ * Try closing a connection if it is not null
+ * @param connection connection object
+ * @throws RuntimeException if closing the connection fails
+ */
+ private void tryClosingConnection(Connection connection) {
+ try {
+ if (connection != null) {
+ connection.close();
+ }
+ } catch (SQLException sqlE) {
+ LOG.error("Failed to close connection: ", sqlE);
+ throw new RuntimeException("Failed to close connection with
exception: ", sqlE);
}
}
@@ -812,17 +835,6 @@ public class OrphanViewTool extends Configured implements
Tool {
} catch (IllegalStateException e) {
printHelpAndExit(e.getMessage(), getOptions());
}
-
- Properties props = new Properties();
- long scn = System.currentTimeMillis() - ageMs;
- props.setProperty("CurrentSCN", Long.toString(scn));
- connection = ConnectionUtil.getInputConnection(configuration);
- PhoenixConnection phoenixConnection =
connection.unwrap(PhoenixConnection.class);
-
- if (clean) {
- // Take a snapshot of system tables to be modified
- createSnapshot(phoenixConnection, scn);
- }
if (outputPath != null) {
// Create files to log orphan views and links
for (int i = VIEW; i < ORPHAN_TYPE_COUNT; i++) {
@@ -834,7 +846,20 @@ public class OrphanViewTool extends Configured implements
Tool {
writer[i] = new BufferedWriter(new FileWriter(file));
}
}
+ Properties props = new Properties();
+ long scn = System.currentTimeMillis() - ageMs;
+ props.setProperty("CurrentSCN", Long.toString(scn));
+ connection = ConnectionUtil.getInputConnection(configuration,
props);
+ PhoenixConnection phoenixConnection =
connection.unwrap(PhoenixConnection.class);
identifyOrphanViews(phoenixConnection);
+ if (clean) {
+ // Close the connection with SCN
+ phoenixConnection.close();
+ connection = ConnectionUtil.getInputConnection(configuration);
+ phoenixConnection = connection.unwrap(PhoenixConnection.class);
+ // Take a snapshot of system tables to be modified
+ createSnapshot(phoenixConnection, scn);
+ }
for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) {
try {
dropOrLogOrphanViews(phoenixConnection, configuration,
entry.getKey());
@@ -843,10 +868,6 @@ public class OrphanViewTool extends Configured implements
Tool {
}
};
if (clean) {
- // Wait for the view drop tasks in the SYSTEM.TASK table to be
processed
- long timeInterval =
configuration.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
-
QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS);
- Thread.sleep(maxViewLevel * timeInterval);
// Clean up any remaining orphan view records from system
tables
for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) {
try {
