>From Wail Alkowaileet <[email protected]>:
Wail Alkowaileet has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17935 )
Change subject: [ASTERIXDB-3306][STO] Ensure no ongoing I/Os before cleaning a
partition
......................................................................
[ASTERIXDB-3306][STO] Ensure no ongoing I/Os before cleaning a partition
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
When cleaning a partition, we could end up deleting files that are being
produced by an ongoing operations – namely an asynchronous merge.
Change-Id: Ia9a84eedbcb33c29f03155a8605bb82af372f7f4
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/AllDatasetsReplicationStrategy.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
4 files changed, 33 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/35/17935/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index bf42541..aee5c01 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -196,9 +196,12 @@
lsmIOScheduler = createIoScheduler(storageProperties);
metadataMergePolicyFactory = new ConcurrentMergePolicyFactory();
indexCheckpointManagerProvider = new
IndexCheckpointManagerProvider(ioManager);
+ datasetLifecycleManager =
+ new DatasetLifecycleManager(storageProperties,
localResourceRepository, txnSubsystem.getLogManager(),
+ virtualBufferCache, indexCheckpointManagerProvider,
ioManager.getIODevices().size());
ILocalResourceRepositoryFactory
persistentLocalResourceRepositoryFactory =
new PersistentLocalResourceRepositoryFactory(ioManager,
indexCheckpointManagerProvider,
- persistedResourceRegistry);
+ persistedResourceRegistry, datasetLifecycleManager);
localResourceRepository =
(PersistentLocalResourceRepository)
persistentLocalResourceRepositoryFactory.createRepository();
configValidator = configValidatorFactory.create();
@@ -226,9 +229,6 @@
// Must start vbc now instead of by life cycle component manager
(lccm) because lccm happens after
// the metadata bootstrap task
((ILifeCycleComponent) virtualBufferCache).start();
- datasetLifecycleManager =
- new DatasetLifecycleManager(storageProperties,
localResourceRepository, txnSubsystem.getLogManager(),
- virtualBufferCache, indexCheckpointManagerProvider,
ioManager.getIODevices().size());
final String nodeId = getServiceContext().getNodeId();
final Set<Integer> nodePartitions =
metadataProperties.getNodePartitions(nodeId);
replicaManager = new ReplicaManager(this, nodePartitions);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/AllDatasetsReplicationStrategy.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/AllDatasetsReplicationStrategy.java
index c969777..84f0a39 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/AllDatasetsReplicationStrategy.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/AllDatasetsReplicationStrategy.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common.replication;
public class AllDatasetsReplicationStrategy implements IReplicationStrategy {
+ public static final IReplicationStrategy INSTANCE = new
AllDatasetsReplicationStrategy();
@Override
public boolean isMatch(int datasetId) {
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index f9bf175..9c08c7b 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -46,8 +46,10 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.dataflow.DatasetLocalResource;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.replication.AllDatasetsReplicationStrategy;
import org.apache.asterix.common.replication.IReplicationManager;
import org.apache.asterix.common.replication.IReplicationStrategy;
import org.apache.asterix.common.replication.ReplicationJob;
@@ -110,13 +112,15 @@
private final List<Path> storageRoots;
private final IIndexCheckpointManagerProvider
indexCheckpointManagerProvider;
private final IPersistedResourceRegistry persistedResourceRegistry;
+ private final IDatasetLifecycleManager datasetLifecycleManager;
public PersistentLocalResourceRepository(IIOManager ioManager,
IIndexCheckpointManagerProvider indexCheckpointManagerProvider,
- IPersistedResourceRegistry persistedResourceRegistry) {
+ IPersistedResourceRegistry persistedResourceRegistry,
IDatasetLifecycleManager datasetLifecycleManager) {
this.ioManager = ioManager;
this.indexCheckpointManagerProvider = indexCheckpointManagerProvider;
this.persistedResourceRegistry = persistedResourceRegistry;
+ this.datasetLifecycleManager = datasetLifecycleManager;
storageRoots = new ArrayList<>();
final List<IODeviceHandle> ioDevices = ioManager.getIODevices();
for (int i = 0; i < ioDevices.size(); i++) {
@@ -480,6 +484,7 @@
}
public synchronized void cleanup(int partition) throws
HyracksDataException {
+
datasetLifecycleManager.waitForIO(AllDatasetsReplicationStrategy.INSTANCE,
partition);
final Set<File> partitionIndexes = getPartitionIndexes(partition);
try {
for (File index : partitionIndexes) {
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
index d1de086..83c9f04 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.transaction.management.resource;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
@@ -28,18 +29,20 @@
private final IIOManager ioManager;
private final IIndexCheckpointManagerProvider
indexCheckpointManagerProvider;
private final IPersistedResourceRegistry persistedResourceRegistry;
+ private final IDatasetLifecycleManager datasetLifecycleManager;
public PersistentLocalResourceRepositoryFactory(IIOManager ioManager,
IIndexCheckpointManagerProvider indexCheckpointManagerProvider,
- IPersistedResourceRegistry persistedResourceRegistry) {
+ IPersistedResourceRegistry persistedResourceRegistry,
IDatasetLifecycleManager datasetLifecycleManager) {
this.ioManager = ioManager;
this.indexCheckpointManagerProvider = indexCheckpointManagerProvider;
this.persistedResourceRegistry = persistedResourceRegistry;
+ this.datasetLifecycleManager = datasetLifecycleManager;
}
@Override
public ILocalResourceRepository createRepository() {
return new PersistentLocalResourceRepository(ioManager,
indexCheckpointManagerProvider,
- persistedResourceRegistry);
+ persistedResourceRegistry, datasetLifecycleManager);
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17935
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: neo
Gerrit-Change-Id: Ia9a84eedbcb33c29f03155a8605bb82af372f7f4
Gerrit-Change-Number: 17935
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <[email protected]>
Gerrit-MessageType: newchange
