This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c44eb23725c virtual storage improvements (#18683)
c44eb23725c is described below
commit c44eb23725c53c1cc4cf3444aeec6e697d1225a3
Author: Clint Wylie <[email protected]>
AuthorDate: Fri Oct 24 11:54:14 2025 -0700
virtual storage improvements (#18683)
changes:
* in virtual storage mode, it is now impossible for a segment to be
'missing' after a caller has obtained a `DataSegment`. Now, the caller can
still mount and use a segment (downloading it if necessary) even if the
coordinator has issued a drop command
* to support there being no such thing as a missing segment in vsf mode,
drops of weakly held cache entries in a storage location are now a no-op.
instead, these cache entries will remain on disk until an eviction needs to
reclaim the space for some other entry
---
.../segment/loading/SegmentLocalCacheManager.java | 81 ++++++++++++++-
.../druid/segment/loading/StorageLocation.java | 37 +++++--
.../SegmentLocalCacheManagerConcurrencyTest.java | 42 +++++++-
.../loading/SegmentLocalCacheManagerTest.java | 110 ++++++++++++++++++++-
.../druid/segment/loading/StorageLocationTest.java | 31 +++---
5 files changed, 269 insertions(+), 32 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java
b/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java
index 104304dce6a..67c30e779e7 100644
---
a/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java
+++
b/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java
@@ -62,6 +62,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.function.Supplier;
@@ -284,12 +285,38 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
{
final File segmentInfoCacheFile = new File(getEffectiveInfoDir(),
segment.getId().toString());
if (!segmentInfoCacheFile.exists()) {
- jsonMapper.writeValue(segmentInfoCacheFile, segment);
+ FileUtils.writeAtomically(
+ segmentInfoCacheFile,
+ out -> {
+ jsonMapper.writeValue(out, segment);
+ return null;
+ }
+ );
}
}
@Override
public void removeInfoFile(final DataSegment segment)
+ {
+ final Runnable delete = () -> deleteSegmentInfoFile(segment);
+ final SegmentCacheEntryIdentifier entryId = new
SegmentCacheEntryIdentifier(segment.getId());
+ boolean isCached = false;
+ // defer deleting until the unmount operation of the cache entry, if
possible, so that if the process stops before
+ // the segment files are deleted, they can be properly managed on startup
(since the info entry still exists)
+ for (StorageLocation location : locations) {
+ final SegmentCacheEntry cacheEntry = location.getCacheEntry(entryId);
+ if (cacheEntry != null) {
+ isCached = isCached || cacheEntry.setOnUnmount(delete);
+ }
+ }
+
+ // otherwise we are probably deleting for cleanup reasons, so try it
anyway if it wasn't present in any location
+ if (!isCached) {
+ delete.run();
+ }
+ }
+
+ private void deleteSegmentInfoFile(DataSegment segment)
{
final File segmentInfoCacheFile = new File(getEffectiveInfoDir(),
segment.getId().toString());
if (!segmentInfoCacheFile.delete()) {
@@ -310,7 +337,6 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
location.addWeakReservationHoldIfExists(cacheEntryIdentifier);
try {
if (hold != null) {
-
if (hold.getEntry().isMounted()) {
Optional<Segment> segment = hold.getEntry().acquireReference();
if (segment.isPresent()) {
@@ -362,6 +388,16 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
);
try {
if (hold != null) {
+ // write the segment info file if it doesn't exist. this can
happen if we are loading after a drop
+ final File segmentInfoCacheFile = new
File(getEffectiveInfoDir(), dataSegment.getId().toString());
+ if (!segmentInfoCacheFile.exists()) {
+ FileUtils.writeAtomically(segmentInfoCacheFile, out -> {
+ jsonMapper.writeValue(out, dataSegment);
+ return null;
+ });
+ hold.getEntry().setOnUnmount(() ->
deleteSegmentInfoFile(dataSegment));
+ }
+
return new AcquireSegmentAction(
makeOnDemandLoadSupplier(hold.getEntry(), location),
hold
@@ -421,7 +457,23 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
public void load(final DataSegment dataSegment) throws
SegmentLoadingException
{
if (config.isVirtualStorage()) {
- // no-op, we'll do a load when someone asks for the segment
+ // virtual storage doesn't do anything with loading immediately, but
check to see if the segment is already cached
+ // and if so, clear out the onUnmount action
+ final ReferenceCountingLock lock = lock(dataSegment);
+ synchronized (lock) {
+ try {
+ final SegmentCacheEntryIdentifier cacheEntryIdentifier = new
SegmentCacheEntryIdentifier(dataSegment.getId());
+ for (StorageLocation location : locations) {
+ final SegmentCacheEntry cacheEntry =
location.getCacheEntry(cacheEntryIdentifier);
+ if (cacheEntry != null) {
+ cacheEntry.clearOnUnmount();
+ }
+ }
+ }
+ finally {
+ unlock(dataSegment, lock);
+ }
+ }
return;
}
final SegmentCacheEntry cacheEntry = new SegmentCacheEntry(dataSegment);
@@ -456,6 +508,7 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
final SegmentCacheEntry entry = location.getCacheEntry(id);
if (entry != null) {
entry.lazyLoadCallback = loadFailed;
+ entry.clearOnUnmount();
entry.mount(location);
}
}
@@ -658,6 +711,7 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
final SegmentCacheEntry entry =
location.getCacheEntry(cacheEntry.id);
if (entry != null) {
entry.lazyLoadCallback = segmentLoadFailCallback;
+ entry.clearOnUnmount();
entry.mount(location);
return entry;
}
@@ -679,6 +733,7 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
final SegmentCacheEntry entry =
location.getCacheEntry(cacheEntry.id);
if (entry != null) {
entry.lazyLoadCallback = segmentLoadFailCallback;
+ entry.clearOnUnmount();
entry.mount(location);
return entry;
}
@@ -771,6 +826,7 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
private StorageLocation location;
private File storageDir;
private ReferenceCountedSegmentProvider referenceProvider;
+ private final AtomicReference<Runnable> onUnmount = new
AtomicReference<>();
private SegmentCacheEntry(final DataSegment dataSegment)
{
@@ -921,6 +977,11 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
storageDir = null;
location = null;
}
+
+ final Runnable onUnmountRunnable = onUnmount.get();
+ if (onUnmountRunnable != null) {
+ onUnmountRunnable.run();
+ }
}
}
finally {
@@ -936,6 +997,20 @@ public class SegmentLocalCacheManager implements
SegmentCacheManager
return referenceProvider.acquireReference();
}
+ public synchronized boolean setOnUnmount(Runnable runnable)
+ {
+ if (location == null) {
+ return false;
+ }
+ onUnmount.set(runnable);
+ return true;
+ }
+
+ public synchronized void clearOnUnmount()
+ {
+ onUnmount.set(null);
+ }
+
public void loadIntoPageCache()
{
if (!isMounted()) {
diff --git
a/server/src/main/java/org/apache/druid/segment/loading/StorageLocation.java
b/server/src/main/java/org/apache/druid/segment/loading/StorageLocation.java
index 15a9f6eb477..5d51de581a4 100644
--- a/server/src/main/java/org/apache/druid/segment/loading/StorageLocation.java
+++ b/server/src/main/java/org/apache/druid/segment/loading/StorageLocation.java
@@ -419,23 +419,18 @@ public class StorageLocation
}
/**
- * Removes an item from {@link #staticCacheEntries} or {@link
#weakCacheEntries}, reducing {@link #currSizeBytes}
- * by {@link CacheEntry#getSize()}
+ * Removes an item from {@link #staticCacheEntries}, reducing {@link
#currSizeBytes} by {@link CacheEntry#getSize()}.
+ * If the cache entry exists in {@link #weakCacheEntries}, it is left in
place to be removed by
+ * {@link #reclaim(long)} instead.
*/
public void release(CacheEntry entry)
{
lock.writeLock().lock();
try {
-
if (staticCacheEntries.containsKey(entry.getId())) {
final CacheEntry toRemove = staticCacheEntries.remove(entry.getId());
toRemove.unmount();
currSizeBytes.getAndAdd(-entry.getSize());
- } else if (weakCacheEntries.containsKey(entry.getId())) {
- final WeakCacheEntry toRemove = weakCacheEntries.remove(entry.getId());
- unlinkWeakEntry(toRemove);
- toRemove.unmount();
- stats.get().unmount();
}
}
finally {
@@ -615,6 +610,32 @@ public class StorageLocation
}
}
+ /**
+ * Unmounts all static and weakly held cache entries and resets stats and
size tracking. Currently only for testing.
+ */
+ @VisibleForTesting
+ public void reset()
+ {
+ lock.writeLock().lock();
+ try {
+ for (CacheEntry entry : staticCacheEntries.values()) {
+ entry.unmount();
+ }
+ staticCacheEntries.clear();
+ while (head != null) {
+ head.unmount();
+ head = head.next;
+ }
+ weakCacheEntries.clear();
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ currSizeBytes.set(0);
+ currWeakSizeBytes.set(0);
+ resetStats();
+ }
+
public Stats getStats()
{
return stats.get();
diff --git
a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerConcurrencyTest.java
b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerConcurrencyTest.java
index 298e9ad0a04..366b3cbbc33 100644
---
a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerConcurrencyTest.java
+++
b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerConcurrencyTest.java
@@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableMap;
import org.apache.druid.error.DruidException;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.concurrent.Execs;
@@ -78,6 +79,8 @@ class SegmentLocalCacheManagerConcurrencyTest
private File localSegmentCacheFolder;
private File otherLocalSegmentCacheFolder;
+ private SegmentLoaderConfig loaderConfig;
+ private SegmentLoaderConfig vsfLoaderConfig;
private SegmentLocalCacheManager manager;
private SegmentLocalCacheManager virtualStorageManager;
private StorageLocation location;
@@ -127,8 +130,21 @@ class SegmentLocalCacheManagerConcurrencyTest
locations.add(locationConfig);
locations.add(locationConfig2);
- final SegmentLoaderConfig loaderConfig = new
SegmentLoaderConfig().withLocations(locations);
- final SegmentLoaderConfig vsfLoaderConfig = new SegmentLoaderConfig()
+ loaderConfig = new SegmentLoaderConfig()
+ {
+ @Override
+ public List<StorageLocationConfig> getLocations()
+ {
+ return locations;
+ }
+
+ @Override
+ public File getInfoDir()
+ {
+ return new File(tempDir, "info");
+ }
+ };
+ vsfLoaderConfig = new SegmentLoaderConfig()
{
@Override
public List<StorageLocationConfig> getLocations()
@@ -147,6 +163,12 @@ class SegmentLocalCacheManagerConcurrencyTest
{
return Runtime.getRuntime().availableProcessors();
}
+
+ @Override
+ public File getInfoDir()
+ {
+ return new File(tempDir, "info");
+ }
};
final List<StorageLocation> storageLocations =
loaderConfig.toStorageLocations();
location = storageLocations.get(0);
@@ -165,6 +187,8 @@ class SegmentLocalCacheManagerConcurrencyTest
TestIndex.INDEX_IO,
jsonMapper
);
+ manager.getCachedSegments();
+ virtualStorageManager.getCachedSegments();
executorService = Execs.multiThreaded(
10,
"segment-loader-local-cache-manager-concurrency-test-%d"
@@ -346,10 +370,10 @@ class SegmentLocalCacheManagerConcurrencyTest
final File localStorageFolder = new File(tempDir, "local_storage_folder");
final Interval interval = Intervals.of("2019-01-01/P1D");
-
makeSegmentsToLoad(segmentCount, localStorageFolder, interval,
segmentsToWeakLoad);
for (boolean sleepy : new boolean[]{true, false}) {
+ // use different segments for each run, otherwise the 2nd run is all
cache hits
testWeakLoad(iterations, segmentCount, concurrentReads, true, sleepy,
true);
}
}
@@ -562,8 +586,8 @@ class SegmentLocalCacheManagerConcurrencyTest
for (DataSegment segment : segmentsToWeakLoad) {
virtualStorageManager.drop(segment);
}
- location.resetStats();
- location2.resetStats();
+ location.reset();
+ location2.reset();
for (int i = 0; i < iterations; i++) {
int segment = random ? ThreadLocalRandom.current().nextInt(segmentCount)
: i % segmentCount;
currentBatch.add(segmentsToWeakLoad.get(segment));
@@ -619,6 +643,14 @@ class SegmentLocalCacheManagerConcurrencyTest
}
assertNoLooseEnds();
+
+ try {
+ FileUtils.deleteDirectory(location.getPath());
+ FileUtils.deleteDirectory(location2.getPath());
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
private BatchResult testWeakBatch(int iteration, List<DataSegment>
currentBatch, boolean sleepy)
diff --git
a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java
b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java
index 04eafa41ed7..290de1934f2 100644
---
a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java
+++
b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java
@@ -893,6 +893,17 @@ public class SegmentLocalCacheManagerTest extends
InitializedNullHandlingTest
{
return true;
}
+
+ @Override
+ public File getInfoDir()
+ {
+ try {
+ return tmpFolder.newFolder();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
};
final List<StorageLocation> storageLocations =
loaderConfig.toStorageLocations();
SegmentLocalCacheManager manager = new SegmentLocalCacheManager(
@@ -920,7 +931,8 @@ public class SegmentLocalCacheManagerTest extends
InitializedNullHandlingTest
segmentAction.close();
manager.drop(segmentToLoad);
- Assert.assertNull(manager.getSegmentFiles(segmentToLoad));
+ // drop doesn't really drop, segments hang out until evicted
+ Assert.assertNotNull(manager.getSegmentFiles(segmentToLoad));
// can actually load them again because load doesn't really do anything
AcquireSegmentAction segmentActionAfterDrop =
manager.acquireSegment(segmentToLoad);
@@ -952,6 +964,17 @@ public class SegmentLocalCacheManagerTest extends
InitializedNullHandlingTest
{
return true;
}
+
+ @Override
+ public File getInfoDir()
+ {
+ try {
+ return tmpFolder.newFolder();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
};
final List<StorageLocation> storageLocations =
loaderConfig.toStorageLocations();
SegmentLocalCacheManager manager = new SegmentLocalCacheManager(
@@ -979,7 +1002,8 @@ public class SegmentLocalCacheManagerTest extends
InitializedNullHandlingTest
segmentAction.close();
manager.drop(segmentToBootstrap);
- Assert.assertNull(manager.getSegmentFiles(segmentToBootstrap));
+ // drop doesn't really drop, segments hang out until evicted
+ Assert.assertNotNull(manager.getSegmentFiles(segmentToBootstrap));
// can actually load them again because bootstrap doesn't really do
anything unless the segment is already
// present in the cache
@@ -1072,6 +1096,88 @@ public class SegmentLocalCacheManagerTest extends
InitializedNullHandlingTest
Assert.assertNull(manager.getSegmentFiles(segmentToLoad));
}
+ @Test
+ public void testGetSegmentVirtualStorageMountAfterDrop() throws Exception
+ {
+ final StorageLocationConfig locationConfig = new
StorageLocationConfig(localSegmentCacheDir, 10L, null);
+ final SegmentLoaderConfig loaderConfig = new SegmentLoaderConfig()
+ {
+ @Override
+ public List<StorageLocationConfig> getLocations()
+ {
+ return ImmutableList.of(locationConfig);
+ }
+
+ @Override
+ public boolean isVirtualStorage()
+ {
+ return true;
+ }
+
+ @Override
+ public File getInfoDir()
+ {
+ try {
+ return tmpFolder.newFolder();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ final List<StorageLocation> storageLocations =
loaderConfig.toStorageLocations();
+ SegmentLocalCacheManager manager = new SegmentLocalCacheManager(
+ storageLocations,
+ loaderConfig,
+ new LeastBytesUsedStorageLocationSelectorStrategy(storageLocations),
+ TestHelper.getTestIndexIO(jsonMapper, ColumnConfig.DEFAULT),
+ jsonMapper
+ );
+
+ final DataSegment segmentToLoad =
makeTestDataSegment(segmentDeepStorageDir);
+ createSegmentZipInLocation(segmentDeepStorageDir, TEST_DATA_RELATIVE_PATH);
+
+ manager.load(segmentToLoad);
+ Assert.assertNull(manager.getSegmentFiles(segmentToLoad));
+
Assert.assertFalse(manager.acquireCachedSegment(segmentToLoad).isPresent());
+ AcquireSegmentAction segmentAction = manager.acquireSegment(segmentToLoad);
+
+ // now drop it before we actually load it, but dropping a weakly held
reference does not remove the entry from the
+ // cache, deferring it until eviction
+ manager.drop(segmentToLoad);
+
+ // however, we also have a hold, so it will not be evicted
+ final DataSegment cannotLoad = makeTestDataSegment(segmentDeepStorageDir,
1, TEST_DATA_RELATIVE_PATH_2);
+ Assert.assertThrows(DruidException.class, () ->
manager.acquireSegment(cannotLoad));
+
+ // and we can still mount and use the segment we are holding
+ ReferenceCountedObjectProvider<Segment> referenceProvider =
segmentAction.getSegmentFuture().get();
+ Assert.assertNotNull(referenceProvider);
+ Optional<Segment> theSegment = referenceProvider.acquireReference();
+ Assert.assertTrue(theSegment.isPresent());
+ Assert.assertNotNull(manager.getSegmentFiles(segmentToLoad));
+ Assert.assertEquals(segmentToLoad.getId(), theSegment.get().getId());
+ Assert.assertEquals(segmentToLoad.getInterval(),
theSegment.get().getDataInterval());
+ theSegment.get().close();
+ segmentAction.close();
+ Assert.assertNotNull(manager.getSegmentFiles(segmentToLoad));
+
+ // now that the hold has been released, we can load the other segment and
evict the one that was held
+ createSegmentZipInLocation(segmentDeepStorageDir,
TEST_DATA_RELATIVE_PATH_2);
+ manager.load(cannotLoad);
+ AcquireSegmentAction segmentActionAfterDrop =
manager.acquireSegment(cannotLoad);
+ ReferenceCountedObjectProvider<Segment> referenceProviderDrop =
segmentActionAfterDrop.getSegmentFuture().get();
+ Optional<Segment> theSegmentAfterDrop =
referenceProviderDrop.acquireReference();
+ Assert.assertTrue(theSegmentAfterDrop.isPresent());
+ Assert.assertNotNull(manager.getSegmentFiles(cannotLoad));
+ Assert.assertEquals(cannotLoad.getId(), theSegmentAfterDrop.get().getId());
+ Assert.assertEquals(cannotLoad.getInterval(),
theSegmentAfterDrop.get().getDataInterval());
+ Assert.assertNull(manager.getSegmentFiles(segmentToLoad));
+
+ theSegmentAfterDrop.get().close();
+ segmentActionAfterDrop.close();
+ }
+
@Test
public void testIfTombstoneIsLoaded() throws IOException,
SegmentLoadingException
{
diff --git
a/server/src/test/java/org/apache/druid/segment/loading/StorageLocationTest.java
b/server/src/test/java/org/apache/druid/segment/loading/StorageLocationTest.java
index a1d459d1ec1..be10cd93cf0 100644
---
a/server/src/test/java/org/apache/druid/segment/loading/StorageLocationTest.java
+++
b/server/src/test/java/org/apache/druid/segment/loading/StorageLocationTest.java
@@ -106,10 +106,11 @@ class StorageLocationTest
location.release(entry2);
location.release(entry3);
location.release(entry4);
- Assertions.assertFalse(location.isWeakReserved(entry1.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry2.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry3.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry4.getId()));
+ // release does not remove weak entries
+ Assertions.assertTrue(location.isWeakReserved(entry1.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry2.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry3.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry4.getId()));
}
@Test
@@ -133,10 +134,11 @@ class StorageLocationTest
location.release(entry3);
location.release(entry2);
location.release(entry1);
- Assertions.assertFalse(location.isWeakReserved(entry1.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry2.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry3.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry4.getId()));
+ // release does not remove weak entries
+ Assertions.assertTrue(location.isWeakReserved(entry1.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry2.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry3.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry4.getId()));
}
@Test
@@ -167,12 +169,13 @@ class StorageLocationTest
location.release(entry);
entries.remove(toRemove);
}
- Assertions.assertFalse(location.isWeakReserved(entry1.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry2.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry3.getId()));
- Assertions.assertFalse(location.isWeakReserved(entry4.getId()));
- Assertions.assertEquals(0, location.currentSizeBytes());
- Assertions.assertEquals(0, location.currentWeakSizeBytes());
+ // release does not remove weak entries
+ Assertions.assertTrue(location.isWeakReserved(entry1.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry2.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry3.getId()));
+ Assertions.assertTrue(location.isWeakReserved(entry4.getId()));
+ Assertions.assertEquals(100, location.currentSizeBytes());
+ Assertions.assertEquals(100, location.currentWeakSizeBytes());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
