This is an automated email from the ASF dual-hosted git repository.
chinmayskulkarni pushed a commit to branch 4.x-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.3 by this push:
new b7d626e PHOENIX-5544: Dropping a base table with cascade with an
older client does not clear all child view metadata
b7d626e is described below
commit b7d626ec9956cbe2872a1f63a32212e92800887c
Author: Chinmay Kulkarni <chinmayskulka...@gmail.com>
AuthorDate: Wed Nov 6 00:59:19 2019 -0800
PHOENIX-5544: Dropping a base table with cascade with an older client does
not clear all child view metadata
---
.../phoenix/coprocessor/MetaDataEndpointImpl.java | 81 +++++++++++++++-------
.../phoenix/coprocessor/MetaDataProtocol.java | 1 +
.../query/DelegateConnectionQueryServices.java | 2 +-
.../java/org/apache/phoenix/util/ViewUtil.java | 40 +++++++++++
4 files changed, 98 insertions(+), 26 deletions(-)
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 3651927..5e6ad9f 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.coprocessor;
import static org.apache.hadoop.hbase.KeyValueUtil.createFirstOnRow;
+import static
org.apache.phoenix.coprocessor.generated.MetaDataProtos.MutationCode.UNABLE_TO_CREATE_CHILD_LINK;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ_BYTES;
@@ -57,7 +58,6 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS_BYTES
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORAGE_SCHEME_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS_BYTES;
-import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES;
@@ -76,6 +76,7 @@ import static
org.apache.phoenix.query.QueryConstants.VIEW_MODIFIED_PROPERTY_TAG
import static org.apache.phoenix.schema.PTableType.INDEX;
import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
import static org.apache.phoenix.util.SchemaUtil.getVarChars;
+import static org.apache.phoenix.util.ViewUtil.getSystemTableForChildLinks;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -1715,15 +1716,14 @@ public class MetaDataEndpointImpl extends
MetaDataProtocol implements Coprocesso
}
}
- // check if the table was dropped, but had child views that were
have not yet been cleaned up
+ // check if the table was previously dropped, but had child views
that have not
+ // yet been cleaned up.
+ // Note that for old clients connecting to a 4.15 server whose
metadata hasn't been
+ // upgraded, we disallow dropping a base table that has child
views, so in that case
+ // this is a no-op (See PHOENIX-5544)
if
(!Bytes.toString(schemaName).equals(QueryConstants.SYSTEM_SCHEMA_NAME)) {
- byte[] sysCatOrSysChildLink = SchemaUtil.getPhysicalTableName(
- clientVersion >= MIN_SPLITTABLE_SYSTEM_CATALOG ?
- SYSTEM_CHILD_LINK_NAME_BYTES :
SYSTEM_CATALOG_NAME_BYTES,
- env.getConfiguration()).getName();
- // TODO: PHOENIX-5544 In the case of old clients, this
actually does not do anything since the
- // parent->child links were already removed when dropping the
base table
- ViewUtil.dropChildViews(env, tenantIdBytes, schemaName,
tableName, sysCatOrSysChildLink);
+ ViewUtil.dropChildViews(env, tenantIdBytes, schemaName,
tableName,
+ getSystemTableForChildLinks(clientVersion,
env.getConfiguration()).getName());
}
byte[] parentTableKey = null;
@@ -1973,6 +1973,26 @@ public class MetaDataEndpointImpl extends
MetaDataProtocol implements Coprocesso
// 3. Finally write the mutations to create the table
if (tableType == PTableType.VIEW) {
+ // If we are connecting with an old client to a server
that has new metadata
+ // i.e. it was previously connected to by a 4.15 client,
then the client will
+ // also send the parent->child link metadata to
SYSTEM.CATALOG rather than using
+ // the new ChildLinkMetaDataEndpoint coprocessor. In this
case, we must continue
+ // doing the server-server RPC to send these mutations to
SYSTEM.CHILD_LINK.
+ if (clientVersion < MIN_SPLITTABLE_SYSTEM_CATALOG &&
+ getSystemTableForChildLinks(clientVersion,
env.getConfiguration()).equals(
+
SchemaUtil.getPhysicalTableName(SYSTEM_CHILD_LINK_NAME_BYTES,
+ env.getConfiguration()))) {
+ List<Mutation> childLinkMutations =
+
MetaDataUtil.removeChildLinkMutations(tableMetadata);
+ MetaDataResponse response =
+ processRemoteRegionMutations(
+
PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+ childLinkMutations,
UNABLE_TO_CREATE_CHILD_LINK);
+ if (response != null) {
+ done.run(response);
+ return;
+ }
+ }
// Pass in the parent's PTable so that we only tag cells
corresponding to the
// view's property in case they are different from the
parent
ViewUtil.addTagsToPutsForViewAlteredProperties(tableMetadata, parentTable);
@@ -2242,10 +2262,10 @@ public class MetaDataEndpointImpl extends
MetaDataProtocol implements Coprocesso
}
// after the view metadata is dropped, drop parent->child link
- MetaDataResponse response =
- processRemoteRegionMutations(
- SYSTEM_CHILD_LINK_NAME_BYTES,
- childLinkMutations,
MetaDataProtos.MutationCode.UNABLE_TO_DELETE_CHILD_LINK);
+ MetaDataResponse response = processRemoteRegionMutations(
+ getSystemTableForChildLinks(request.getClientVersion(),
+ env.getConfiguration()).getName(),
childLinkMutations,
+
MetaDataProtos.MutationCode.UNABLE_TO_DELETE_CHILD_LINK);
if (response != null) {
done.run(response);
return;
@@ -2348,32 +2368,44 @@ public class MetaDataEndpointImpl extends
MetaDataProtocol implements Coprocesso
if (tableType == PTableType.TABLE || tableType == PTableType.VIEW
|| tableType == PTableType.SYSTEM) {
// check to see if the table has any child views
- try (Table hTable =
env.getTable(SchemaUtil.getPhysicalTableName(
- clientVersion >= MIN_SPLITTABLE_SYSTEM_CATALOG ?
- SYSTEM_CHILD_LINK_NAME_BYTES :
SYSTEM_CATALOG_NAME_BYTES,
+ try (Table hTable =
env.getTable(getSystemTableForChildLinks(clientVersion,
env.getConfiguration()))) {
boolean hasChildViews =
ViewUtil.hasChildViews(hTable, tenantId,
schemaName, tableName,
clientTimeStamp);
if (hasChildViews) {
if (!isCascade) {
+ LOGGER.error("DROP without CASCADE on tables with
child views "
+ + "is not permitted");
// DROP without CASCADE on tables with child views
is not permitted
return new
MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
EnvironmentEdgeManager.currentTimeMillis(), null);
}
- try {
- if (clientVersion >=
MIN_SPLITTABLE_SYSTEM_CATALOG) {
+ // For 4.15+ clients and older clients connecting to
an upgraded server,
+ // add a task to drop child views of the base table.
+ if (clientVersion >= MIN_SPLITTABLE_SYSTEM_CATALOG ||
+
SchemaUtil.getPhysicalTableName(SYSTEM_CHILD_LINK_NAME_BYTES,
+
env.getConfiguration()).equals(hTable.getName())) {
+ try {
PhoenixConnection conn =
QueryUtil.getConnectionOnServer(env.getConfiguration())
.unwrap(PhoenixConnection.class);
Task.addTask(conn,
PTable.TaskType.DROP_CHILD_VIEWS,
Bytes.toString(tenantId),
Bytes.toString(schemaName),
Bytes.toString(tableName),
this.accessCheckEnabled);
+ } catch (Throwable t) {
+ LOGGER.error("Adding a task to drop child
views failed!", t);
}
- // else: the client version is old, so we cannot
add a task to cleanup
- // child view metadata since SYSTEM.TASK may not
exist
- } catch (Throwable t) {
- LOGGER.error("Adding a task to drop child views
failed!", t);
+ } else {
+ // (See PHOENIX-5544) For an old client connecting
to a non-upgraded
+ // server, we disallow dropping a base table that
has child views.
+ LOGGER.error("Dropping a table that has child
views is not permitted "
+ + "for old clients connecting to a new
server with old metadata."
+ + " Please upgrade the client to " +
+ MIN_SPLITTABLE_SYSTEM_CATALOG_VERSION);
+ return new MetaDataMutationResult(
+ MutationCode.UNALLOWED_TABLE_MUTATION,
+
EnvironmentEdgeManager.currentTimeMillis(), null);
}
}
}
@@ -2474,9 +2506,8 @@ public class MetaDataEndpointImpl extends
MetaDataProtocol implements Coprocesso
// Also if
QueryServices.ALLOW_SPLITTABLE_SYSTEM_CATALOG_ROLLBACK is true, we block adding
// a column to a parent table so that we can rollback
the upgrade if required.
if (clientVersion < MIN_SPLITTABLE_SYSTEM_CATALOG) {
- LOGGER.error(
- "Unable to add or drop a column as the
client is older than "
- + MIN_SPLITTABLE_SYSTEM_CATALOG);
+ LOGGER.error("Unable to add or drop a column as
the client is older "
+ + "than " +
MIN_SPLITTABLE_SYSTEM_CATALOG_VERSION);
return new
MetaDataProtocol.MetaDataMutationResult(MetaDataProtocol.MutationCode.UNALLOWED_TABLE_MUTATION,
EnvironmentEdgeManager.currentTimeMillis(), null);
} else if (allowSplittableSystemCatalogRollback) {
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index e217a3b..a83b04b 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -112,6 +112,7 @@ public abstract class MetaDataProtocol extends
MetaDataService {
public static final int CLIENT_KEY_VALUE_BUILDER_THRESHOLD =
VersionUtil.encodeVersion("0", "94", "14");
// Version at which we allow SYSTEM.CATALOG to split
public static final int MIN_SPLITTABLE_SYSTEM_CATALOG =
VersionUtil.encodeVersion("4", "15", "0");
+ public static final String MIN_SPLITTABLE_SYSTEM_CATALOG_VERSION =
"4.15.0";
// Version at and after which we will no longer expect client to serialize
thresholdBytes for
// spooling into the scan
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 761b758..0e0554e 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -363,7 +363,7 @@ public class DelegateConnectionQueryServices extends
DelegateQueryServices imple
@Override
public void clearUpgradeRequired() {
- getDelegate().isUpgradeRequired();
+ getDelegate().clearUpgradeRequired();
}
@Override
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ViewUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/ViewUtil.java
index 100c9ab..8c81234 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ViewUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ViewUtil.java
@@ -19,7 +19,10 @@ import com.google.common.base.Objects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -68,8 +71,11 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import static
org.apache.phoenix.coprocessor.MetaDataProtocol.MIN_SPLITTABLE_SYSTEM_CATALOG;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARENT_TENANT_ID_BYTES;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
@@ -219,6 +225,40 @@ public class ViewUtil {
}
}
+
+ /**
+ * Determines whether we should use SYSTEM.CATALOG or SYSTEM.CHILD_LINK to
find parent->child
+ * links i.e. {@link LinkType#CHILD_TABLE}.
+ * If the client is older than 4.15.0 and the SYSTEM.CHILD_LINK table does
not exist, we use
+ * the SYSTEM.CATALOG table. In all other cases, we use the
SYSTEM.CHILD_LINK table.
+ * This is required for backwards compatibility.
+ * @param clientVersion client version
+ * @param conf server-side configuration
+ * @return name of the system table to be used
+ * @throws SQLException
+ */
+ public static TableName getSystemTableForChildLinks(int clientVersion,
+ Configuration conf) throws SQLException, IOException {
+ byte[] fullTableName = SYSTEM_CHILD_LINK_NAME_BYTES;
+ if (clientVersion < MIN_SPLITTABLE_SYSTEM_CATALOG) {
+ try (PhoenixConnection connection =
QueryUtil.getConnectionOnServer(
+ conf).unwrap(PhoenixConnection.class);
+ HBaseAdmin admin =
connection.getQueryServices().getAdmin()) {
+
+ // If this is an old client and the CHILD_LINK table doesn't
exist i.e. metadata
+ // hasn't been updated since there was never a connection from
a 4.15 client
+ if (!admin.tableExists(SchemaUtil.getPhysicalTableName(
+ SYSTEM_CHILD_LINK_NAME_BYTES, conf))) {
+ fullTableName = SYSTEM_CATALOG_NAME_BYTES;
+ }
+ } catch (ClassNotFoundException e) {
+ logger.error("Error getting a connection on the server : " +
e);
+ throw new SQLException(e);
+ }
+ }
+ return SchemaUtil.getPhysicalTableName(fullTableName, conf);
+ }
+
public static boolean isDivergedView(PTable view) {
return view.getBaseColumnCount() ==
QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
}
