Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2369
Change subject: [NO ISSUE] Metadata partition bootstrap, replication
......................................................................
[NO ISSUE] Metadata partition bootstrap, replication
- Send metadata partition id as part of metadata bootstrap
- Ignore requests to add replicas to myself
- Avoid hard checkpoint at shutdown
Change-Id: I08c91de570bbc90f5532329f72db21a424993fed
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
10 files changed, 53 insertions(+), 23 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/69/2369/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 1de6938..d42db39 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -416,10 +416,10 @@
}
@Override
- public void initializeMetadata(boolean newUniverse) throws Exception {
+ public void initializeMetadata(boolean newUniverse, int partitionId)
throws Exception {
LOGGER.info("Bootstrapping metadata");
MetadataNode.INSTANCE.initialize(this,
ncExtensionManager.getMetadataTupleTranslatorProvider(),
- ncExtensionManager.getMetadataExtensions());
+ ncExtensionManager.getMetadataExtensions(), partitionId);
//noinspection unchecked
ConcurrentHashMap<CcId, IAsterixStateProxy> proxyMap =
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 3717673..de3c691 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -668,8 +668,6 @@
@Override
public void stop(boolean dumpState, OutputStream os) throws IOException {
- // Shutdown checkpoint
- checkpointManager.doSharpCheckpoint();
}
@Override
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 8c733fb..30416a3 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.nc;
+import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -35,10 +36,15 @@
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.replication.api.PartitionReplica;
import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.hyracks.api.config.IApplicationConfig;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.storage.common.LocalResource;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class ReplicaManager implements IReplicaManager {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
/**
@@ -60,6 +66,10 @@
if (!partitions.contains(id.getPartition())) {
throw new IllegalStateException(
"This node is not the current master of partition(" +
id.getPartition() + ")");
+ }
+ if (isSelf(id)) {
+ LOGGER.info("ignoring request to add replica to ourselves");
+ return;
}
replicas.computeIfAbsent(id, k -> new PartitionReplica(k, appCtx));
replicas.get(id).sync();
@@ -88,6 +98,9 @@
@Override
public synchronized void promote(int partition) throws
HyracksDataException {
+ if (partitions.contains(partition)) {
+ return;
+ }
final PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository)
appCtx.getLocalResourceRepository();
localResourceRepository.cleanup(partition);
@@ -120,4 +133,13 @@
datasetLifecycleManager.close(resource.getPath());
}
}
+
+ private boolean isSelf(ReplicaIdentifier id) {
+ IApplicationConfig appConfig =
appCtx.getServiceContext().getAppConfig();
+ String host =
appConfig.getString(NCConfig.Option.REPLICATION_LISTEN_ADDRESS);
+ int port = appConfig.getInt(NCConfig.Option.REPLICATION_LISTEN_PORT);
+
+ final InetSocketAddress replicaAddress = new InetSocketAddress(host,
port);
+ return id.equals(ReplicaIdentifier.of(id.getPartition(),
replicaAddress));
+ }
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
index 001af23..c18b967 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
@@ -28,13 +28,19 @@
public class MetadataBootstrapTask implements INCLifecycleTask {
private static final long serialVersionUID = 1L;
+ private final int partitionId;
+
+ public MetadataBootstrapTask(int partitionId) {
+ this.partitionId = partitionId;
+ }
@Override
public void perform(CcId ccId, IControllerService cs) throws
HyracksDataException {
INcApplicationContext appContext = (INcApplicationContext)
cs.getApplicationContext();
try {
+ appContext.getReplicaManager().promote(partitionId);
SystemState state =
appContext.getTransactionSubsystem().getRecoveryManager().getSystemState();
- appContext.initializeMetadata(state ==
SystemState.PERMANENT_DATA_LOSS);
+ appContext.initializeMetadata(state ==
SystemState.PERMANENT_DATA_LOSS, partitionId);
} catch (Exception e) {
throw HyracksDataException.create(e);
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 8939059..25e768d 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -155,7 +155,7 @@
tasks.add(new StartReplicationServiceTask());
}
if (isMetadataNode) {
- tasks.add(new MetadataBootstrapTask());
+ tasks.add(new
MetadataBootstrapTask(clusterManager.getMetadataPartition().getPartitionId()));
}
tasks.add(new ExternalLibrarySetupTask(isMetadataNode));
tasks.add(new CheckpointTask());
@@ -184,7 +184,8 @@
}
// if current metadata node is active, we need to unbind its metadata
proxy objects
if (clusterManager.isMetadataNodeActive()) {
- MetadataNodeRequestMessage msg = new
MetadataNodeRequestMessage(false);
+ MetadataNodeRequestMessage msg =
+ new MetadataNodeRequestMessage(false,
clusterManager.getMetadataPartition().getPartitionId());
try {
messageBroker.sendApplicationMessageToNC(msg, metadataNodeId);
// when the current node responses, we will bind to the new one
@@ -207,7 +208,8 @@
}
private void requestMetadataNodeTakeover(String node) throws
HyracksDataException {
- MetadataNodeRequestMessage msg = new MetadataNodeRequestMessage(true);
+ MetadataNodeRequestMessage msg =
+ new MetadataNodeRequestMessage(true,
clusterManager.getMetadataPartition().getPartitionId());
try {
messageBroker.sendApplicationMessageToNC(msg, node);
} catch (Exception e) {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
index 4443825..817bbe6 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
@@ -34,9 +34,11 @@
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = LogManager.getLogger();
private final boolean export;
+ private final int partitionId;
- public MetadataNodeRequestMessage(boolean export) {
+ public MetadataNodeRequestMessage(boolean export, int partitionId) {
this.export = export;
+ this.partitionId = partitionId;
}
@Override
@@ -45,7 +47,7 @@
HyracksDataException hde = null;
try {
if (export) {
- appContext.initializeMetadata(false);
+ appContext.initializeMetadata(false, partitionId);
appContext.exportMetadataNodeStub();
appContext.bindMetadataNodeStub(getCcId());
} else {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
index a02bda5..82936b3 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
@@ -94,9 +94,10 @@
* Initializes the metadata node and bootstraps the metadata.
*
* @param newUniverse
+ * @param partitionId
* @throws Exception
*/
- void initializeMetadata(boolean newUniverse) throws Exception;
+ void initializeMetadata(boolean newUniverse, int partitionId) throws
Exception;
/**
* Unexports the metadata node from the RMI registry
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
index 948bdad..0b18f98 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
@@ -105,11 +105,6 @@
return accessor.getString(Option.METADATA_NODE);
}
- public ClusterPartition getMetadataPartition() {
- // metadata partition is always the first partition on the metadata
node
- return accessor.getNodePartitions().get(getMetadataNodeName())[0];
- }
-
public Map<String, String[]> getStores() {
return accessor.getStores();
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 72d5cf5..fd21941 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -145,11 +145,12 @@
}
public void initialize(INcApplicationContext runtimeContext,
- MetadataTupleTranslatorProvider tupleTranslatorProvider,
List<IMetadataExtension> metadataExtensions) {
+ MetadataTupleTranslatorProvider tupleTranslatorProvider,
List<IMetadataExtension> metadataExtensions,
+ int partitionId) {
this.tupleTranslatorProvider = tupleTranslatorProvider;
this.transactionSubsystem = runtimeContext.getTransactionSubsystem();
this.datasetLifecycleManager =
runtimeContext.getDatasetLifecycleManager();
- this.metadataStoragePartition =
runtimeContext.getMetadataProperties().getMetadataPartition().getPartitionId();
+ this.metadataStoragePartition = partitionId;
if (metadataExtensions != null) {
extensionDatasets = new HashMap<>();
for (IMetadataExtension metadataExtension : metadataExtensions) {
@@ -161,6 +162,10 @@
this.txnIdFactory = new BulkTxnIdFactory();
}
+ public int getMetadataStoragePartition() {
+ return metadataStoragePartition;
+ }
+
@Override
public void ensureMinimumTxnId(long maxId) throws ACIDException,
RemoteException {
txnIdFactory.ensureMinimumId(maxId);
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 9ebd21b..23806bd 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -25,7 +25,6 @@
import java.util.List;
import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.MetadataProperties;
@@ -44,6 +43,7 @@
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataNode;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.api.IMetadataIndex;
import org.apache.asterix.metadata.entities.BuiltinTypeMap;
@@ -311,11 +311,10 @@
if
(!appContext.getDatasetMemoryManager().reserve(index.getDatasetId().getId())) {
throw new IllegalStateException("Failed to reserve memory for
metadata dataset (" + datasetId + ")");
}
- ClusterPartition metadataPartition =
appContext.getMetadataProperties().getMetadataPartition();
- int metadataDeviceId = metadataPartition.getIODeviceNum();
- String metadataPartitionPath =
StoragePathUtil.prepareStoragePartitionPath(metadataPartition.getPartitionId());
+ String metadataPartitionPath =
+
StoragePathUtil.prepareStoragePartitionPath(MetadataNode.INSTANCE.getMetadataStoragePartition());
String resourceName = metadataPartitionPath + File.separator +
index.getFileNameRelativePath();
- FileReference file = ioManager.getFileReference(metadataDeviceId,
resourceName);
+ FileReference file = ioManager.resolve(resourceName);
index.setFile(file);
ITypeTraits[] typeTraits = index.getTypeTraits();
IBinaryComparatorFactory[] cmpFactories =
index.getKeyBinaryComparatorFactory();
--
To view, visit https://asterix-gerrit.ics.uci.edu/2369
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I08c91de570bbc90f5532329f72db21a424993fed
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <[email protected]>
