This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 30c4a78fcecf8d81a3cc070e348a38cae5600d1a
Author: Ali Alsuliman <ali.al.solai...@gmail.com>
AuthorDate: Thu May 6 01:39:41 2021 +0300
[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/app/function/DumpIndexDatasource.java | 14 +++++++++++---
.../apache/asterix/app/function/DumpIndexRewriter.java | 6 +++++-
.../apache/asterix/metadata/entities/NodeGroup.java | 18 +++++++++---------
.../org/apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../metadata/utils/SecondaryIndexOperationsHelper.java | 4 ++++
.../asterix/runtime/utils/ClusterStateManager.java | 2 --
6 files changed, 30 insertions(+), 16 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@ public class DumpIndexDatasource extends FunctionDataSource
{
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory
indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[]
comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[]
comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws
AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint
getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider
metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(),
indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory,
recDesc, comparatorFactories);
}
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import
org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@ public class DumpIndexRewriter extends FunctionRewriter {
IndexDataflowHelperFactory indexDataflowHelperFactory =
new
IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint)
secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(),
indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(),
secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(),
secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@ package org.apache.asterix.metadata.entities;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@ import org.apache.asterix.metadata.api.IMetadataEntity;
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String>
nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@ public class DatasetUtil {
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(),
nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new
NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx,
NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@ public abstract class SecondaryIndexOperationsHelper {
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@ public class ClusterStateManager implements
IClusterStateManager {
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list
similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new
AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new
String[] {}));
}
