bbeaudreault commented on code in PR #5697:
URL: https://github.com/apache/hbase/pull/5697#discussion_r1513395609
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java:
##########
@@ -66,61 +66,112 @@ public class TableNamespaceManager {
private final MasterServices masterServices;
+ private volatile boolean migrationDone;
+
TableNamespaceManager(MasterServices masterServices) {
this.masterServices = masterServices;
}
- private void migrateNamespaceTable() throws IOException {
- try (Table nsTable =
masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
- ResultScanner scanner = nsTable.getScanner(
- new
Scan().addFamily(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES).readAllVersions());
- BufferedMutator mutator =
-
masterServices.getConnection().getBufferedMutator(TableName.META_TABLE_NAME)) {
+ private void tryMigrateNamespaceTable() throws IOException,
InterruptedException {
+ Optional<MigrateNamespaceTableProcedure> opt =
masterServices.getProcedures().stream()
+ .filter(p -> p instanceof MigrateNamespaceTableProcedure)
+ .map(p -> (MigrateNamespaceTableProcedure) p).findAny();
+ if (!opt.isPresent()) {
+ // the procedure is not present, check whether have the ns family in
meta table
+ TableDescriptor metaTableDesc =
+ masterServices.getTableDescriptors().get(TableName.META_TABLE_NAME);
+ if (metaTableDesc.hasColumnFamily(HConstants.NAMESPACE_FAMILY)) {
+ // normal case, upgrading is done or the cluster is created with 3.x
code
+ migrationDone = true;
+ } else {
+ // submit the migration procedure
+ MigrateNamespaceTableProcedure proc = new
MigrateNamespaceTableProcedure();
+ masterServices.getMasterProcedureExecutor().submitProcedure(proc);
+ }
+ } else {
+ if (opt.get().isFinished()) {
+ // the procedure is already done
+ migrationDone = true;
+ }
+ // we have already submitted the procedure, continue
+ }
+ }
+
+ private void addToCache(Result result, byte[] family, byte[] qualifier)
throws IOException {
+ Cell cell = result.getColumnLatestCell(family, qualifier);
+ NamespaceDescriptor ns =
+
ProtobufUtil.toNamespaceDescriptor(HBaseProtos.NamespaceDescriptor.parseFrom(CodedInputStream
+ .newInstance(cell.getValueArray(), cell.getValueOffset(),
cell.getValueLength())));
+ cache.put(ns.getName(), ns);
+ }
+
+ private void loadFromMeta() throws IOException {
+ try (Table table =
masterServices.getConnection().getTable(TableName.META_TABLE_NAME);
+ ResultScanner scanner = table.getScanner(HConstants.NAMESPACE_FAMILY)) {
for (Result result;;) {
result = scanner.next();
if (result == null) {
break;
}
- Put put = new Put(result.getRow());
- result
- .getColumnCells(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES,
- TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES)
- .forEach(c -> put.addColumn(HConstants.NAMESPACE_FAMILY,
- HConstants.NAMESPACE_COL_DESC_QUALIFIER, c.getTimestamp(),
CellUtil.cloneValue(c)));
- mutator.mutate(put);
+ addToCache(result, HConstants.NAMESPACE_FAMILY,
HConstants.NAMESPACE_COL_DESC_QUALIFIER);
}
}
- // schedule a disable procedure instead of block waiting here, as when
disabling a table we will
- // wait until master is initialized, but we are part of the
initialization...
- masterServices.getMasterProcedureExecutor().submitProcedure(
- new
DisableTableProcedure(masterServices.getMasterProcedureExecutor().getEnvironment(),
- TableName.NAMESPACE_TABLE_NAME, false));
}
- private void loadNamespaceIntoCache() throws IOException {
- try (Table table =
masterServices.getConnection().getTable(TableName.META_TABLE_NAME);
- ResultScanner scanner = table.getScanner(HConstants.NAMESPACE_FAMILY)) {
+ private void loadFromNamespace() throws IOException {
+ try (Table table =
masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
+ ResultScanner scanner =
+ table.getScanner(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES)) {
for (Result result;;) {
result = scanner.next();
if (result == null) {
break;
}
- Cell cell = result.getColumnLatestCell(HConstants.NAMESPACE_FAMILY,
- HConstants.NAMESPACE_COL_DESC_QUALIFIER);
- NamespaceDescriptor ns = ProtobufUtil
-
.toNamespaceDescriptor(HBaseProtos.NamespaceDescriptor.parseFrom(CodedInputStream
- .newInstance(cell.getValueArray(), cell.getValueOffset(),
cell.getValueLength())));
- cache.put(ns.getName(), ns);
+ addToCache(result, TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES,
+ TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES);
}
}
}
- public void start() throws IOException {
- TableState nsTableState =
MetaTableAccessor.getTableState(masterServices.getConnection(),
- TableName.NAMESPACE_TABLE_NAME);
- if (nsTableState != null && nsTableState.isEnabled()) {
- migrateNamespaceTable();
+ private boolean shouldLoadFromMeta() throws IOException {
+ if (migrationDone) {
+ return true;
}
+ // the implementation is bit tricky
+ // if there is already a disable namespace table procedure or the
namespace table is already
+ // disabled, we are safe to read from meta table as the migration is
already one. If not, since
Review Comment:
typo: already one vs already done
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MigrateNamespaceTableProcedure.java:
##########
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateNamespaceTableProcedureState;
+
+/**
+ * Migrate the namespace data to meta table's namespace family while upgrading
+ */
[email protected]
+public class MigrateNamespaceTableProcedure
+ extends StateMachineProcedure<MasterProcedureEnv,
MigrateNamespaceTableProcedureState>
+ implements GlobalProcedureInterface {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MigrateNamespaceTableProcedure.class);
+
+ private RetryCounter retryCounter;
+
+ @Override
+ public String getGlobalId() {
+ return getClass().getSimpleName();
+ }
+
+ private void migrate(MasterProcedureEnv env) throws IOException {
+ Connection conn = env.getMasterServices().getConnection();
+ try (Table nsTable = conn.getTable(TableName.NAMESPACE_TABLE_NAME);
+ ResultScanner scanner = nsTable.getScanner(
Review Comment:
put the ResultScanner in the try-with-resources?
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java:
##########
@@ -174,6 +233,10 @@ public void
validateTableAndRegionCount(NamespaceDescriptor desc) throws IOExcep
}
}
+ public void setMigrationDown() {
Review Comment:
typo: down vs done
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MigrateNamespaceTableProcedure.java:
##########
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateNamespaceTableProcedureState;
+
+/**
+ * Migrate the namespace data to meta table's namespace family while upgrading
+ */
[email protected]
+public class MigrateNamespaceTableProcedure
+ extends StateMachineProcedure<MasterProcedureEnv,
MigrateNamespaceTableProcedureState>
+ implements GlobalProcedureInterface {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MigrateNamespaceTableProcedure.class);
+
+ private RetryCounter retryCounter;
+
+ @Override
+ public String getGlobalId() {
+ return getClass().getSimpleName();
+ }
+
+ private void migrate(MasterProcedureEnv env) throws IOException {
+ Connection conn = env.getMasterServices().getConnection();
+ try (Table nsTable = conn.getTable(TableName.NAMESPACE_TABLE_NAME);
+ ResultScanner scanner = nsTable.getScanner(
+ new
Scan().addFamily(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES).readAllVersions());
+ BufferedMutator mutator =
conn.getBufferedMutator(TableName.META_TABLE_NAME)) {
+ for (Result result;;) {
+ result = scanner.next();
+ if (result == null) {
+ break;
+ }
Review Comment:
why do this instead of `for (Result result : scanner)`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
