This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6913d68ba0 [Enhancement](merge-on-write) use delete bitmap to mark
delete for rows with delete sign when sequence column doesn't exist (#24011)
6913d68ba0 is described below
commit 6913d68ba0c40dfe2a2ee43600804b69c8d0bc12
Author: bobhan1 <[email protected]>
AuthorDate: Tue Sep 12 08:56:46 2023 +0800
[Enhancement](merge-on-write) use delete bitmap to mark delete for rows
with delete sign when sequence column doesn't exist (#24011)
---
be/src/olap/delete_bitmap_calculator.cpp | 3 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 41 ++++++++-
be/src/olap/tablet.cpp | 25 ++++--
be/src/olap/tablet_meta.h | 7 ++
.../test_partial_update_delete_sign.out | 34 +++++++-
.../test_delete_sign_delete_bitmap.out | 54 ++++++++++++
.../test_partial_update_delete_sign.groovy | 61 +++++++++++++-
.../test_delete_sign_delete_bitmap.groovy | 96 ++++++++++++++++++++++
8 files changed, 305 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/delete_bitmap_calculator.cpp
b/be/src/olap/delete_bitmap_calculator.cpp
index bfdb506c06..b1d9226eaf 100644
--- a/be/src/olap/delete_bitmap_calculator.cpp
+++ b/be/src/olap/delete_bitmap_calculator.cpp
@@ -192,7 +192,8 @@ Status
MergeIndexDeleteBitmapCalculator::calculate_all(DeleteBitmapPtr delete_bi
break;
}
RETURN_IF_ERROR(st);
- delete_bitmap->add({_rowset_id, loc.segment_id, 0}, loc.row_id);
+ delete_bitmap->add({_rowset_id, loc.segment_id,
DeleteBitmap::TEMP_VERSION_COMMON},
+ loc.row_id);
}
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 775d9b5a8a..0fb4ee0cd5 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -423,7 +423,8 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
if (_tablet_schema->is_strict_mode()) {
++num_rows_filtered;
// delete the invalid newly inserted row
- _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id,
_segment_id, 0},
+ _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id,
_segment_id,
+
DeleteBitmap::TEMP_VERSION_COMMON},
segment_pos);
}
@@ -447,6 +448,13 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
if (delete_sign_column_data != nullptr &&
delete_sign_column_data[block_pos] != 0) {
has_default_or_nullable = true;
use_default_or_null_flag.emplace_back(true);
+ if (!_tablet_schema->has_sequence_col() && !have_input_seq_column)
{
+ // we can directly use delete bitmap to mark the rows with
delete sign as deleted
+ // if sequence column doesn't exist to eliminate reading
delete sign columns in later reads
+ _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id,
_segment_id,
+
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
+ segment_pos);
+ }
} else {
// partial update should not contain invisible columns
use_default_or_null_flag.emplace_back(false);
@@ -457,10 +465,12 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
if (st.is<KEY_ALREADY_EXISTS>()) {
// although we need to mark delete current row, we still need to
read missing columns
// for this row, we need to ensure that each column is aligned
- _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id,
_segment_id, 0},
- segment_pos);
+ _mow_context->delete_bitmap->add(
+ {_opts.rowset_ctx->rowset_id, _segment_id,
DeleteBitmap::TEMP_VERSION_COMMON},
+ segment_pos);
} else {
- _mow_context->delete_bitmap->add({loc.rowset_id, loc.segment_id,
0}, loc.row_id);
+ _mow_context->delete_bitmap->add(
+ {loc.rowset_id, loc.segment_id,
DeleteBitmap::TEMP_VERSION_COMMON}, loc.row_id);
}
}
CHECK(use_default_or_null_flag.size() == num_rows);
@@ -638,6 +648,29 @@ Status SegmentWriter::append_block(const
vectorized::Block* block, size_t row_po
_serialize_block_to_row_column(*const_cast<vectorized::Block*>(block));
}
+ if (_opts.write_type == DataWriteType::TYPE_DIRECT &&
_opts.enable_unique_key_merge_on_write &&
+ !_tablet_schema->has_sequence_col() &&
_tablet_schema->delete_sign_idx() != -1) {
+ const vectorized::ColumnWithTypeAndName& delete_sign_column =
+ block->get_by_position(_tablet_schema->delete_sign_idx());
+ auto& delete_sign_col =
+ reinterpret_cast<const
vectorized::ColumnInt8&>(*(delete_sign_column.column));
+ if (delete_sign_col.size() >= row_pos + num_rows) {
+ const vectorized::Int8* delete_sign_column_data =
delete_sign_col.get_data().data();
+ uint32_t segment_start_pos =
+
_column_writers[_tablet_schema->delete_sign_idx()]->get_next_rowid();
+ for (size_t block_pos = row_pos, seg_pos = segment_start_pos;
+ seg_pos < segment_start_pos + num_rows; block_pos++,
seg_pos++) {
+ // we can directly use delete bitmap to mark the rows with
delete sign as deleted
+ // if sequence column doesn't exist to eliminate reading
delete sign columns in later reads
+ if (delete_sign_column_data[block_pos]) {
+
_mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
+
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
+ seg_pos);
+ }
+ }
+ }
+ }
+
_olap_data_convertor->set_source_content(block, row_pos, num_rows);
// find all row pos for short key indexes
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 7c737e4d3c..d4029f22be 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2971,7 +2971,8 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr
rowset,
Slice key = Slice(index_column->get_data_at(i).data,
index_column->get_data_at(i).size);
RowLocation loc;
// same row in segments should be filtered
- if (delete_bitmap->contains({rowset_id, seg->id(), 0}, row_id)) {
+ if (delete_bitmap->contains({rowset_id, seg->id(),
DeleteBitmap::TEMP_VERSION_COMMON},
+ row_id)) {
continue;
}
@@ -2989,7 +2990,8 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr
rowset,
// sequence id smaller than the previous one, so delete current row
if (st.is<KEY_ALREADY_EXISTS>()) {
- delete_bitmap->add({rowset_id, seg->id(), 0}, row_id);
+ delete_bitmap->add({rowset_id, seg->id(),
DeleteBitmap::TEMP_VERSION_COMMON},
+ row_id);
continue;
} else if (is_partial_update && rowset_writer != nullptr) {
// In publish version, record rows to be deleted for
concurrent update
@@ -3011,12 +3013,16 @@ Status
Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
// rowset.
// just set 0 as a unified temporary version number, and
update to
// the real version number later.
- delete_bitmap->add({loc.rowset_id, loc.segment_id, 0},
loc.row_id);
- delete_bitmap->add({rowset_id, seg->id(), 0}, row_id);
+ delete_bitmap->add(
+ {loc.rowset_id, loc.segment_id,
DeleteBitmap::TEMP_VERSION_COMMON},
+ loc.row_id);
+ delete_bitmap->add({rowset_id, seg->id(),
DeleteBitmap::TEMP_VERSION_COMMON},
+ row_id);
continue;
}
// when st = ok
- delete_bitmap->add({loc.rowset_id, loc.segment_id, 0}, loc.row_id);
+ delete_bitmap->add({loc.rowset_id, loc.segment_id,
DeleteBitmap::TEMP_VERSION_COMMON},
+ loc.row_id);
}
remaining -= num_read;
}
@@ -3717,8 +3723,9 @@ Status Tablet::calc_delete_bitmap_between_segments(
void Tablet::add_sentinel_mark_to_delete_bitmap(DeleteBitmap* delete_bitmap,
const RowsetIdUnorderedSet&
rowsetids) {
for (const auto& rowsetid : rowsetids) {
- delete_bitmap->add({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0},
- DeleteBitmap::ROWSET_SENTINEL_MARK);
+ delete_bitmap->add(
+ {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID,
DeleteBitmap::TEMP_VERSION_COMMON},
+ DeleteBitmap::ROWSET_SENTINEL_MARK);
}
}
@@ -3739,8 +3746,8 @@ Status
Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, in
std::vector<RowsetSharedPtr>*
rowsets) {
RowsetIdUnorderedSet missing_ids;
for (const auto& rowsetid : rowset_ids) {
- if (!delete_bitmap->delete_bitmap.contains(
- {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0})) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid,
DeleteBitmap::INVALID_SEGMENT_ID,
+
DeleteBitmap::TEMP_VERSION_COMMON})) {
missing_ids.insert(rowsetid);
}
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index c0dcdc249a..42fec6489b 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -340,6 +340,13 @@ public:
constexpr static inline uint32_t ROWSET_SENTINEL_MARK =
std::numeric_limits<uint32_t>::max() - 1;
+ // When a delete bitmap is merged into tablet's delete bitmap, the version
of entries in the delete bitmap
+ // will be replaced to the correspoding correct version. So before we
finally merge a delete bitmap into
+ // tablet's delete bitmap we can use arbitary version number in BitmapKey.
Here we define some version numbers
+ // for specific usage during this periods to avoid conflicts
+ constexpr static inline uint64_t TEMP_VERSION_COMMON = 0;
+ constexpr static inline uint64_t TEMP_VERSION_FOR_DELETE_SIGN = 1;
+
/**
*
* @param tablet_id the tablet which this delete bitmap associates with
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
index f66f4b6873..0f1cd09e7d 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
@@ -10,7 +10,7 @@
2 2 2 2 2
4 4 4 4 4
--- !with_delete_sign --
+-- !1 --
1 \N \N \N \N 1
1 1 1 1 1 0
2 2 2 2 2 0
@@ -20,3 +20,35 @@
5 \N \N \N \N 1
5 5 5 5 5 0
+-- !2 --
+2 2 2 2 2 0
+4 4 4 4 4 0
+
+-- !sql --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !after_delete --
+2 2 2 2 2
+4 4 4 4 4
+
+-- !1 --
+1 \N \N \N \N 1
+1 1 1 1 1 0
+2 2 2 2 2 0
+3 \N \N \N \N 1
+3 3 3 3 3 0
+4 4 4 4 4 0
+5 \N \N \N \N 1
+5 5 5 5 5 0
+
+-- !2 --
+1 \N \N \N \N 1
+2 2 2 2 2 0
+3 \N \N \N \N 1
+4 4 4 4 4 0
+5 \N \N \N \N 1
+
diff --git
a/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
b/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
new file mode 100644
index 0000000000..687aeab54a
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
@@ -0,0 +1,54 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !after_delete --
+2 2 2 2 2
+4 4 4 4 4
+
+-- !1 --
+1 1 1 1 1 0
+1 1 1 1 1 1
+2 2 2 2 2 0
+3 3 3 3 3 0
+3 3 3 3 3 1
+4 4 4 4 4 0
+5 5 5 5 5 0
+5 5 5 5 5 1
+
+-- !2 --
+2 2 2 2 2 0
+4 4 4 4 4 0
+
+-- !sql --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !after_delete --
+2 2 2 2 2
+4 4 4 4 4
+
+-- !1 --
+1 1 1 1 1 0
+1 1 1 1 1 1
+2 2 2 2 2 0
+3 3 3 3 3 0
+3 3 3 3 3 1
+4 4 4 4 4 0
+5 5 5 5 5 0
+5 5 5 5 5 1
+
+-- !2 --
+1 1 1 1 1 1
+2 2 2 2 2 0
+3 3 3 3 3 1
+4 4 4 4 4 0
+5 5 5 5 5 1
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
index 1aa651c48c..f8369f0ed3 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
@@ -55,6 +55,65 @@ suite('test_partial_update_delete_sign') {
sql "set skip_storage_engine_merge=true;"
sql "set skip_delete_bitmap=true;"
sql "sync"
- qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ // skip_delete_bitmap=true, skip_delete_sign=true
+ qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ // skip_delete_bitmap=false, skip_delete_sign=true
+ qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
sql "drop table if exists ${tableName1};"
+
+
+ sql "set skip_delete_sign=false;"
+ sql "set skip_storage_engine_merge=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ def tableName2 = "test_partial_update_delete_sign2"
+ sql "DROP TABLE IF EXISTS ${tableName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'c4'
+ );"""
+
+ sql "insert into ${tableName2}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+ streamLoad {
+ table "${tableName2}"
+
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'partial_columns', 'true'
+ set 'columns', 'k1,__DORIS_DELETE_SIGN__'
+
+ file 'delete_sign.csv'
+ time 10000 // limit inflight 10s
+ }
+ sql "sync"
+ qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ // skip_delete_bitmap=true, skip_delete_sign=true
+ qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ // skip_delete_bitmap=false, skip_delete_sign=true
+ qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName2};"
}
diff --git
a/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
new file mode 100644
index 0000000000..f9b89c1eea
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_delete_sign_delete_bitmap') {
+
+ def tableName1 = "test_delete_sign_delete_bitmap1"
+ sql "DROP TABLE IF EXISTS ${tableName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1"
+ );"""
+
+ sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ // sql "insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
select k1,c1,c2,c3,c4,1 from ${tableName1} where k1 in (1,3,5);"
+ sql """insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
+ sql "sync"
+ qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ // skip_delete_bitmap=true, skip_delete_sign=true
+ qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ // skip_delete_bitmap=false, skip_delete_sign=true
+ qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName1};"
+
+
+ sql "set skip_delete_sign=false;"
+ sql "set skip_storage_engine_merge=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ def tableName2 = "test_delete_sign_delete_bitmap2"
+ sql "DROP TABLE IF EXISTS ${tableName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "function_column.sequence_col" = 'c4'
+ );"""
+
+ sql "insert into ${tableName2}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+ sql """insert into ${tableName2}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
+ sql "sync"
+ qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+ sql "set skip_delete_sign=true;"
+ sql "set skip_storage_engine_merge=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ // skip_delete_bitmap=true, skip_delete_sign=true
+ qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ // skip_delete_bitmap=false, skip_delete_sign=true
+ qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ sql "drop table if exists ${tableName2};"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
