frankgh commented on code in PR #3374:
URL: https://github.com/apache/cassandra/pull/3374#discussion_r1874174024
##########
src/java/org/apache/cassandra/service/CassandraDaemon.java:
##########
@@ -291,7 +300,7 @@ protected void setup()
{
SystemKeyspace.snapshotOnVersionChange();
}
- catch (IOException e)
+ catch (Throwable e)
Review Comment:
why the change here?
##########
src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.snapshot;
+
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.Clock;
+
+public class ClearSnapshotTask extends AbstractSnapshotTask<Void>
+{
+ private static final Logger logger =
LoggerFactory.getLogger(ClearSnapshotTask.class);
+
+ private final SnapshotManager manager;
+ private final Predicate<TableSnapshot> predicateForToBeCleanedSnapshots;
+ private final boolean deleteData;
+
+ public ClearSnapshotTask(SnapshotManager manager,
+ Predicate<TableSnapshot>
predicateForToBeCleanedSnapshots,
+ boolean deleteData)
+ {
+ super(null);
+ this.manager = manager;
+ this.predicateForToBeCleanedSnapshots =
predicateForToBeCleanedSnapshots;
+ this.deleteData = deleteData;
+ }
+
+ @Override
+ public SnapshotTaskType getTaskType()
+ {
+ return SnapshotTaskType.CLEAR;
+ }
+
+ @Override
+ public Void call()
+ {
+ Set<TableSnapshot> toRemove = new HashSet<>();
+
+ for (TableSnapshot snapshot : new GetSnapshotsTask(manager,
predicateForToBeCleanedSnapshots, false).call())
+ {
+ logger.debug("Removing snapshot {}{}", snapshot, deleteData ? ",
deleting data" : "");
+
+ toRemove.add(snapshot);
+
+ if (deleteData)
+ {
+ for (File snapshotDir : snapshot.getDirectories())
+ {
+ try
+ {
+ removeSnapshotDirectory(snapshotDir);
+ }
+ catch (Throwable ex)
+ {
+ logger.warn("Unable to remove snapshot directory {}",
snapshotDir, ex);
+ }
+ }
+ }
+ }
+
+ manager.getSnapshots().removeAll(toRemove);
+
+ return null;
+ }
+
+ /**
+ * Returns predicate which will pass the test when arguments match.
+ *
+ * @param tag name of snapshot
+ * @param options options for filtering
+ * @param keyspaceNames names of keyspaces a snapshot is supposed to be
from
+ * @return predicate which will pass the test when arguments match.
+ */
+ public static Predicate<TableSnapshot>
getPredicateForCleanedSnapshots(String tag, Map<String, Object> options,
String... keyspaceNames)
Review Comment:
do we need this as public? maybe make it package-private instead?
##########
src/java/org/apache/cassandra/repair/PreviewRepairTask.java:
##########
@@ -128,7 +129,7 @@ private void maybeSnapshotReplicas(TimeUUID parentSession,
String keyspace, List
for (String table : mismatchingTables)
{
// we can just check snapshot existence locally since the
repair coordinator is always a replica (unlike in the read case)
- if
(!Keyspace.open(keyspace).getColumnFamilyStore(table).snapshotExists(snapshotName))
+ if (SnapshotManager.instance.getSnapshot(keyspace, table,
snapshotName).isEmpty())
Review Comment:
I see a lot of calls checking for existence of a snapshot in the form of
getSnapshot -> isEmpty. I wonder if we should add a dedicated method to check
for snapshot existence?
##########
src/java/org/apache/cassandra/io/util/PathUtils.java:
##########
@@ -401,6 +401,18 @@ public static void deleteRecursive(Path path)
delete(path);
}
+ /**
+ * Empties everything in directory of "path" but keeps the directory
itself.
+ *
+ * @param path directory to be emptied
+ * @throws FSWriteError if any part of the tree cannot be deleted
+ */
+ public static void clearDirectory(Path path)
Review Comment:
it feels weird to have production code only for a test. I would move this
out of the prod code into test code. Especially concerning because of what this
method can do.
##########
src/java/org/apache/cassandra/service/StorageServiceMBean.java:
##########
@@ -308,7 +309,9 @@ public interface StorageServiceMBean extends
NotificationEmitter
* @param options map of options for cleanup operation, consult nodetool's
ClearSnapshot
* @param tag name of snapshot to clear, if null or empty string, all
snapshots of given keyspace will be cleared
* @param keyspaceNames name of keyspaces to clear snapshots for
+ * @deprecated See CASSANDRA-18111
*/
+ @Deprecated(since = "5.1")
Review Comment:
ah interesting, these move to its own MBean, something to keep in mind for
Cassandra ANalytics
##########
src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java:
##########
@@ -17,156 +17,541 @@
*/
package org.apache.cassandra.service.snapshot;
-
-import java.util.Collection;
-import java.util.PriorityQueue;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
+import javax.management.openmbean.TabularData;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Directories;
-
-import java.util.concurrent.TimeoutException;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-
-import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.TableDroppedNotification;
+import org.apache.cassandra.notifications.TablePreScrubNotification;
+import org.apache.cassandra.notifications.TruncationNotification;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
-import static java.util.Comparator.comparing;
-import static java.util.stream.Collectors.toList;
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static
org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.utils.FBUtilities.now;
+import static
org.apache.cassandra.service.snapshot.ClearSnapshotTask.getClearSnapshotPredicate;
+import static
org.apache.cassandra.service.snapshot.ClearSnapshotTask.getPredicateForCleanedSnapshots;
-public class SnapshotManager {
+public class SnapshotManager implements SnapshotManagerMBean,
INotificationConsumer, AutoCloseable
+{
+ private static final Logger logger =
LoggerFactory.getLogger(SnapshotManager.class);
- private static final ScheduledExecutorPlus executor =
executorFactory().scheduled(false, "SnapshotCleanup");
+ private ScheduledExecutorPlus snapshotCleanupExecutor;
- private static final Logger logger =
LoggerFactory.getLogger(SnapshotManager.class);
+ public static final SnapshotManager instance = new SnapshotManager();
private final long initialDelaySeconds;
private final long cleanupPeriodSeconds;
- private final SnapshotLoader snapshotLoader;
- @VisibleForTesting
- protected volatile ScheduledFuture<?> cleanupTaskFuture;
+ private volatile ScheduledFuture<?> cleanupTaskFuture;
+
+ private final String[] dataDirs;
+
+ private volatile boolean started = false;
/**
- * Expiring snapshots ordered by expiration date, to allow only iterating
over snapshots
- * that need to be removed on {@link this#clearExpiredSnapshots()}
+ * We read / list snapshots way more often than write / create them so COW
is ideal to use here.
+ * This enables us to not submit listing tasks or tasks computing snapshot
sizes to any executor's queue as they
+ * can be just run concurrently which gives way better throughput in case
+ * of excessive listing from clients (dashboards and similar) where
snapshot metrics are gathered.
*/
- private final PriorityQueue<TableSnapshot> expiringSnapshots = new
PriorityQueue<>(comparing(TableSnapshot::getExpiresAt));
+ private final List<TableSnapshot> snapshots = new CopyOnWriteArrayList<>();
- public SnapshotManager()
+ private SnapshotManager()
{
this(CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(),
-
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt());
+
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt(),
+ DatabaseDescriptor.getAllDataFileLocations());
}
@VisibleForTesting
- protected SnapshotManager(long initialDelaySeconds, long
cleanupPeriodSeconds)
+ SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds,
String[] dataDirs)
{
this.initialDelaySeconds = initialDelaySeconds;
this.cleanupPeriodSeconds = cleanupPeriodSeconds;
- snapshotLoader = new
SnapshotLoader(DatabaseDescriptor.getAllDataFileLocations());
+ this.dataDirs = dataDirs;
+ this.snapshotCleanupExecutor = createSnapshotCleanupExecutor();
}
- public Collection<TableSnapshot> getExpiringSnapshots()
+ public void registerMBean()
{
- return expiringSnapshots;
+ MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
}
- public synchronized void start()
+ public void unregisterMBean()
{
- addSnapshots(loadSnapshots());
- resumeSnapshotCleanup();
+ MBeanWrapper.instance.unregisterMBean(MBEAN_NAME);
}
- public synchronized void stop() throws InterruptedException,
TimeoutException
+ public synchronized SnapshotManager start(boolean
runPeriodicSnapshotCleaner)
+ {
+ if (started)
+ return this;
+
+ if (snapshotCleanupExecutor == null)
+ snapshotCleanupExecutor = createSnapshotCleanupExecutor();
+
+ executeTask(new ReloadSnapshotsTask(dataDirs));
+
+ if (runPeriodicSnapshotCleaner)
+ resumeSnapshotCleanup();
+
+ started = true;
+ return this;
+ }
+
+ @Override
+ public synchronized void close()
+ {
+ if (!started)
+ return;
+
+ pauseSnapshotCleanup();
+
+ shutdownAndWait(1, TimeUnit.MINUTES);
+ snapshots.clear();
+
+ started = false;
+ }
+
+ public synchronized void shutdownAndWait(long timeout, TimeUnit unit)
+ {
+ try
+ {
+ ExecutorUtils.shutdownNowAndWait(timeout, unit,
snapshotCleanupExecutor);
+ }
+ catch (InterruptedException | TimeoutException ex)
+ {
+ throw new RuntimeException(ex);
+ }
+ finally
+ {
+ snapshotCleanupExecutor = null;
+ }
+ }
+
+ public synchronized void restart(boolean runPeriodicSnapshotCleaner)
+ {
+ if (!started)
+ return;
+
+ logger.debug("Restarting SnapshotManager");
+ close();
+ start(runPeriodicSnapshotCleaner);
+ logger.debug("SnapshotManager restarted");
+ }
+
+
+ public synchronized void restart()
+ {
+ restart(true);
+ }
+
+ private static class ReloadSnapshotsTask extends
AbstractSnapshotTask<Set<TableSnapshot>>
+ {
+ private final String[] dataDirs;
+
+ public ReloadSnapshotsTask(String[] dataDirs)
+ {
+ super(null);
+ this.dataDirs = dataDirs;
+ }
+
+ @Override
+ public Set<TableSnapshot> call()
+ {
+ Set<TableSnapshot> tableSnapshots = new
SnapshotLoader(dataDirs).loadSnapshots();
+ new ClearSnapshotTask(SnapshotManager.instance, snapshot -> true,
false).call();
+ for (TableSnapshot snapshot : tableSnapshots)
+ SnapshotManager.instance.addSnapshot(snapshot);
+
+ return tableSnapshots;
+ }
+
+ @Override
+ public SnapshotTaskType getTaskType()
+ {
+ return SnapshotTaskType.RELOAD;
+ }
+ }
+
+ void addSnapshot(TableSnapshot snapshot)
+ {
+ logger.debug("Adding snapshot {}", snapshot);
+ snapshots.add(snapshot);
+ }
+
+ List<TableSnapshot> getSnapshots()
+ {
+ return snapshots;
+ }
+
+ public void resumeSnapshotCleanup()
+ {
+ if (cleanupTaskFuture == null)
+ {
+ logger.info("Scheduling expired snapshots cleanup with
initialDelaySeconds={} and cleanupPeriodSeconds={}",
+ initialDelaySeconds, cleanupPeriodSeconds);
+
+ cleanupTaskFuture =
snapshotCleanupExecutor.scheduleWithFixedDelay(SnapshotManager.instance::clearExpiredSnapshots,
+
initialDelaySeconds,
+
cleanupPeriodSeconds,
+
SECONDS);
+ }
+ }
+
+ private void pauseSnapshotCleanup()
{
- expiringSnapshots.clear();
if (cleanupTaskFuture != null)
{
cleanupTaskFuture.cancel(false);
cleanupTaskFuture = null;
}
}
- public synchronized void addSnapshot(TableSnapshot snapshot)
+ /**
+ * Deletes snapshot and removes it from manager.
+ *
+ * @param snapshot snapshot to clear
+ */
+ void clearSnapshot(TableSnapshot snapshot)
+ {
+ executeTask(new ClearSnapshotTask(this, s -> s.equals(snapshot),
true));
+ }
+
+ /**
+ * Returns list of snapshots of given keyspace
+ *
+ * @param keyspace keyspace of a snapshot
+ * @return list of snapshots of given keyspace.
+ */
+ public List<TableSnapshot> getSnapshots(String keyspace)
+ {
+ return getSnapshots(snapshot ->
snapshot.getKeyspaceName().equals(keyspace));
+ }
+
+ /**
+ * Return snapshots based on given parameters.
+ *
+ * @param skipExpiring if expiring snapshots should be skipped
+ * @param includeEphemeral if ephemeral snapshots should be included
+ * @return snapshots based on given parameters
+ */
+ public List<TableSnapshot> getSnapshots(boolean skipExpiring, boolean
includeEphemeral)
{
- // We currently only care about expiring snapshots
- if (snapshot.isExpiring())
+ return getSnapshots(s -> (!skipExpiring || !s.isExpiring()) &&
(includeEphemeral || !s.isEphemeral()));
+ }
+
+ /**
+ * Returns all snapshots passing the given predicate.
+ *
+ * @param predicate predicate to filter all snapshots of
+ * @return list of snapshots passing the predicate
+ */
+ public List<TableSnapshot> getSnapshots(Predicate<TableSnapshot> predicate)
+ {
+ return new GetSnapshotsTask(this, predicate, true).call();
+ }
+
+ /**
+ * Returns a snapshot or empty optional based on the given parameters.
+ *
+ * @param keyspace keyspace of a snapshot
+ * @param table table of a snapshot
+ * @param tag name of a snapshot
+ * @return empty optional if there is not such snapshot, non-empty
otherwise
+ */
+ public Optional<TableSnapshot> getSnapshot(String keyspace, String table,
String tag)
+ {
+ List<TableSnapshot> foundSnapshots = new GetSnapshotsTask(this,
+ snapshot ->
snapshot.getKeyspaceName().equals(keyspace) &&
+
snapshot.getTableName().equals(table) &&
+
snapshot.getTag().equals(tag) || (tag != null && tag.isEmpty()),
+ true).call();
+
+ if (foundSnapshots.isEmpty())
+ return Optional.empty();
+ else
+ return Optional.of(foundSnapshots.get(0));
+ }
+
+ /**
+ * Clear snapshots of given tag from given keyspace. Does not remove
ephemeral snapshots.
+ * <p>
+ *
+ * @param tag snapshot name
+ * @param keyspace keyspace to clear all snapshots of a given tag of
+ */
+ public void clearSnapshots(String tag, String keyspace)
+ {
+ clearSnapshots(tag, Set.of(keyspace),
Clock.Global.currentTimeMillis());
+ }
+
+ /**
+ * Removes a snapshot.
+ * <p>
+ *
+ * @param keyspace keyspace of a snapshot to remove
+ * @param table table of a snapshot to remove
+ * @param tag name of a snapshot to remove.
+ */
+ public void clearSnapshot(String keyspace, String table, String tag)
+ {
+ executeTask(new ClearSnapshotTask(this,
+ snapshot ->
snapshot.getKeyspaceName().equals(keyspace)
+ &&
snapshot.getTableName().equals(table)
+ &&
snapshot.getTag().equals(tag),
+ true));
+ }
+
+ /**
+ * Removes all snapshots for given keyspace and table.
+ *
+ * @param keyspace keyspace to remove snapshots for
+ * @param table table in a given keyspace to remove snapshots for
+ */
+ public void clearAllSnapshots(String keyspace, String table)
+ {
+ executeTask(new ClearSnapshotTask(this,
+ snapshot ->
snapshot.getKeyspaceName().equals(keyspace)
+ &&
snapshot.getTableName().equals(table),
+ true));
+ }
+
+ /**
+ * Clears all snapshots, expiring and ephemeral as well.
+ */
+ public void clearAllSnapshots()
+ {
+ executeTask(new ClearSnapshotTask(this, snapshot -> true, true));
+ }
+
+ /**
+ * Clear snapshots based on a given predicate
+ *
+ * @param predicate predicate to filter snapshots on
+ */
+ public void clearSnapshot(Predicate<TableSnapshot> predicate)
+ {
+ executeTask(new ClearSnapshotTask(this, predicate, true));
+ }
+
+ /**
+ * Clears all ephemeral snapshots in a node.
+ */
+ public void clearEphemeralSnapshots()
+ {
+ executeTask(new ClearSnapshotTask(this, TableSnapshot::isEphemeral,
true));
+ }
+
+ /**
+ * Clears all expired snapshots in a node.
+ */
+ public void clearExpiredSnapshots()
+ {
+ Instant now = FBUtilities.now();
+ executeTask(new ClearSnapshotTask(this, s -> s.isExpired(now), true));
+ }
+
+ /**
+ * Clear snapshots of given tag from given keyspaces.
+ * <p>
+ * If tag is not present / is empty, all snapshots are considered to be
cleared.
+ * If keyspaces are empty, all snapshots of given tag and older than
maxCreatedAt are removed.
+ *
+ * @param tag optional tag of snapshot to clear
+ * @param keyspaces keyspaces to remove snapshots for
+ * @param maxCreatedAt clear all such snapshots which were created before
this timestamp
+ */
+ private void clearSnapshots(String tag, Set<String> keyspaces, long
maxCreatedAt)
+ {
+ executeTask(new ClearSnapshotTask(this, getClearSnapshotPredicate(tag,
keyspaces, maxCreatedAt, false), true));
+ }
+
+ public List<TableSnapshot> takeSnapshot(SnapshotOptions options)
+ {
+ return executeTask(new TakeSnapshotTask(this, options));
+ }
+
+ // Super methods
+
+ @Override
+ public void takeSnapshot(String tag, String... entities)
+ {
+ takeSnapshot(SnapshotOptions.userSnapshot(tag, Map.of(), entities));
+ }
+
+ @Override
+ public void takeSnapshot(String tag, Map<String, String> optMap, String...
entities) throws IOException
+ {
+ try
+ {
+ takeSnapshot(SnapshotOptions.userSnapshot(tag, optMap, entities));
+ }
+ catch (SnapshotException ex)
{
- logger.debug("Adding expiring snapshot {}", snapshot);
- expiringSnapshots.add(snapshot);
+ // to be compatible with deprecated methods in StorageService
+ throw new IOException(ex);
}
}
- public synchronized Set<TableSnapshot> loadSnapshots(String keyspace)
+ @Override
+ public void clearSnapshot(String tag, Map<String, Object> options,
String... keyspaceNames)
{
- return snapshotLoader.loadSnapshots(keyspace);
+ executeTask(new ClearSnapshotTask(this,
getPredicateForCleanedSnapshots(tag, options, keyspaceNames), true));
}
- public synchronized Set<TableSnapshot> loadSnapshots()
+ @Override
+ public Map<String, TabularData> listSnapshots(Map<String, String> options)
{
- return snapshotLoader.loadSnapshots();
+ return new ListSnapshotsTask(this, options, true).call();
}
- @VisibleForTesting
- protected synchronized void addSnapshots(Collection<TableSnapshot>
snapshots)
+ @Override
+ public long getTrueSnapshotSize()
{
- logger.debug("Adding snapshots: {}.", Joiner.on(",
").join(snapshots.stream().map(TableSnapshot::getId).collect(toList())));
- snapshots.forEach(this::addSnapshot);
+ return new TrueSnapshotSizeTask(this, s -> true).call();
}
- // TODO: Support pausing snapshot cleanup
- @VisibleForTesting
- synchronized void resumeSnapshotCleanup()
+ @Override
+ public long getTrueSnapshotsSize(String keyspace)
{
- if (cleanupTaskFuture == null)
+ return new TrueSnapshotSizeTask(this, s ->
s.getKeyspaceName().equals(keyspace)).call();
+ }
+
+ @Override
+ public long getTrueSnapshotsSize(String keyspace, String table)
+ {
+ return new TrueSnapshotSizeTask(this, s ->
s.getKeyspaceName().equals(keyspace) && s.getTableName().equals(table)).call();
+ }
+
+ @Override
+ public void setSnapshotLinksPerSecond(long throttle)
+ {
+ logger.info("Setting snapshot throttle to {}", throttle);
+ DatabaseDescriptor.setSnapshotLinksPerSecond(throttle);
+ }
+
+ @Override
+ public long getSnapshotLinksPerSecond()
+ {
+ return DatabaseDescriptor.getSnapshotLinksPerSecond();
+ }
+
+ @Override
+ public void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof TruncationNotification)
{
- logger.info("Scheduling expired snapshot cleanup with
initialDelaySeconds={} and cleanupPeriodSeconds={}",
- initialDelaySeconds, cleanupPeriodSeconds);
- cleanupTaskFuture =
executor.scheduleWithFixedDelay(this::clearExpiredSnapshots,
initialDelaySeconds,
-
cleanupPeriodSeconds, TimeUnit.SECONDS);
+ TruncationNotification truncationNotification =
(TruncationNotification) notification;
+ ColumnFamilyStore cfs = truncationNotification.cfs;
+ if (!truncationNotification.disableSnapshot &&
cfs.isAutoSnapshotEnabled())
+ {
+ SnapshotOptions opts =
SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.TRUNCATE,
cfs.getKeyspaceTableName())
Review Comment:
I think we are missing the cfs here.
```suggestion
SnapshotOptions opts =
SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.TRUNCATE,
cfs.getKeyspaceTableName()).cfs(cfs)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
