>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483?usp=email )
Change subject: [NO ISSUE][*DB][STO] Increased log brevity in cloud storage ...................................................................... [NO ISSUE][*DB][STO] Increased log brevity in cloud storage Ext-ref: MB-68946 Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java 6 files changed, 47 insertions(+), 16 deletions(-) Approvals: Jenkins: Verified; Verified Ali Alsuliman: Looks good to me, approved Anon. E. Moose #1000171: Michael Blow: Looks good to me, but someone else must approve diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java index 526af2d..7712f08 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; @@ -50,6 +49,7 @@ import org.apache.asterix.common.config.CloudProperties; import org.apache.asterix.common.metadata.MetadataConstants; import org.apache.asterix.common.transactions.IRecoveryManager.SystemState; +import org.apache.asterix.common.utils.Partitions; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; @@ -82,12 +82,11 @@ protected final ICloudGuardian guardian; protected final IWriteBufferProvider writeBufferProvider; protected final String bucket; - protected final Set<Integer> partitions; + protected final Partitions partitions; protected final List<FileReference> partitionPaths; protected final IOManager localIoManager; protected final INamespacePathResolver nsPathResolver; private final List<FileStore> drivePaths; - private final String storageScheme; public AbstractCloudIOManager(IOManager ioManager, CloudProperties cloudProperties, INamespacePathResolver nsPathResolver, ICloudGuardian guardian) throws HyracksDataException { @@ -99,11 +98,10 @@ this.guardian = guardian; int numOfThreads = getIODevices().size() * getIOParallelism(); writeBufferProvider = new WriteBufferProvider(numOfThreads, cloudClient.getWriteBufferSize()); - partitions = new HashSet<>(); + partitions = new Partitions(); partitionPaths = new ArrayList<>(); this.localIoManager = ioManager; drivePaths = PhysicalDrive.getDrivePaths(ioDevices); - storageScheme = cloudProperties.getStorageScheme(); } /* @@ -179,7 +177,7 @@ } } } else { - LOGGER.info("Cleaning node partitions..."); + LOGGER.info("Cleaning node partitions {}...", partitions); for (FileReference partitionPath : partitionPaths) { CloudFileUtil.cleanDirectoryFiles(localIoManager, cloudFiles, partitionPath); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java index 8c3321a..5070d40 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java @@ -18,10 +18,9 @@ */ package org.apache.asterix.cloud.lazy.accessor; -import java.util.Collections; - import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.NoOpParallelCacher; +import org.apache.asterix.common.utils.Partitions; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.control.nc.io.IOManager; @@ -35,7 +34,7 @@ }; public InitialCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager) { - super(cloudClient, bucket, localIoManager, Collections.emptySet(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE); + super(cloudClient, bucket, localIoManager, Partitions.empty(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE); } @Override diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java index 91046a1..8dbf605 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java @@ -28,6 +28,7 @@ import org.apache.asterix.cloud.clients.CloudFile; import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.IParallelCacher; +import org.apache.asterix.common.utils.Partitions; import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -42,13 +43,13 @@ */ public class ReplaceableCloudAccessor extends AbstractLazyAccessor { private static final Logger LOGGER = LogManager.getLogger(); - private final Set<Integer> partitions; + private final Partitions partitions; private final ILazyAccessorReplacer replacer; protected final IParallelCacher cacher; private final IBulkOperationCallBack deleteCallBack; public ReplaceableCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager, - Set<Integer> partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) { + Partitions partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) { super(cloudClient, bucket, localIoManager); this.partitions = partitions; this.replacer = replacer; diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java index 56eaeeb..68c0556 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java @@ -21,12 +21,12 @@ import static org.apache.asterix.cloud.util.CloudFileUtil.DATA_FILTER; import java.util.Collection; -import java.util.Set; import org.apache.asterix.cloud.UncachedFileReference; import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.IParallelCacher; import org.apache.asterix.cloud.lazy.filesystem.IHolePuncher; +import org.apache.asterix.common.utils.Partitions; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IFileHandle; @@ -37,7 +37,7 @@ private final IHolePuncher puncher; public SelectiveCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager, - Set<Integer> partitions, IHolePuncher puncher, IParallelCacher cacher) { + Partitions partitions, IHolePuncher puncher, IParallelCacher cacher) { super(cloudClient, bucket, localIoManager, partitions, InitialCloudAccessor.NO_OP_REPLACER, cacher); this.puncher = puncher; } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java index 7befacf..fd37e61 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java @@ -49,7 +49,6 @@ // First get the set of local files Set<FileReference> localFiles = ioManager.list(partitionPath); Iterator<FileReference> localFilesIter = localFiles.iterator(); - LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath()); // Reconcile local files and cloud files while (localFilesIter.hasNext()) { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java index 45aaa49..cc93b91 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java @@ -20,9 +20,13 @@ import java.io.Serial; import java.io.Serializable; +import java.util.Set; + +import org.apache.commons.lang3.mutable.MutableBoolean; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntSortedSet; +import it.unimi.dsi.fastutil.ints.IntSortedSets; /** * A specialized bit set for storing partition IDs. @@ -36,15 +40,16 @@ private static final long serialVersionUID = 1L; private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1; private static final int MIN_PARTITION_ID = -1; + private static final Partitions EMPTY = new Partitions(IntSortedSets.EMPTY_SET); private static final short MINUS_ONE = (short) -1; private final IntSortedSet delegate; public Partitions() { - delegate = new IntSortedBitSet(); + this(new IntSortedBitSet()); } public Partitions(int initialMaxValue) { - delegate = new IntSortedBitSet(initialMaxValue + 1); + this(new IntSortedBitSet(initialMaxValue + 1)); } public Partitions(IntSortedSet delegate) { @@ -52,6 +57,13 @@ } /** + * Returns an unmodifiable empty Partitions instance. + */ + public static Partitions empty() { + return EMPTY; + } + + /** * Adds a partition ID to the set. Partition ID must be in the range [-1, 32766]. * @param k the partition ID to add * @return true if the partition ID was added, false if it was already present @@ -72,6 +84,27 @@ delegate.clear(); } + public boolean addAll(Set<Integer> activePartitions) { + MutableBoolean retval = new MutableBoolean(); + activePartitions.forEach(p -> { + if (add(p)) { + retval.setTrue(); + } + }); + return retval.booleanValue(); + } + + public int size() { + return delegate.size(); + } + + public boolean contains(int partitionNum) { + if (partitionNum > MAX_PARTITION_ID || partitionNum < MIN_PARTITION_ID) { + return false; + } + return delegate.contains((short) (partitionNum + 1)); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -107,4 +140,5 @@ builder.append(']'); return builder.toString(); } + } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6 Gerrit-Change-Number: 20483 Gerrit-PatchSet: 6 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
