[
https://issues.apache.org/jira/browse/CASSANDRA-20487?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
David Capwell updated CASSANDRA-20487:
--------------------------------------
Description:
This was found in https://github.com/apache/cassandra/pull/4006
SHA=7fe995c3b8787bc895c62b9614d408161231259a
Test=org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkWithoutReadRepairTest
{code}
accord.utils.Property$PropertyError: Property error detected:
Seed = -6517659374622108140
Examples = 1
Pure = true
Steps: 400
Values:
State:
Setup:
CREATE KEYSPACE IF NOT EXISTS ks1 WITH replication = {'class':
'SimpleStrategy', 'replication_factor': 3};
CREATE TABLE ks1.tbl (
pk0 ascii,
ck0 boolean,
s0 list<frozen<list<boolean>>> static,
v0 frozen<list<frozen<map<ascii, varint>>>>,
v1 frozen<tuple<tinyint, frozen<map<varint, uuid>>,
frozen<tuple<ascii, double>>>>,
PRIMARY KEY (pk0, ck0)
) WITH CLUSTERING ORDER BY (ck0 ASC)
AND additional_write_policy = '99p'
AND allow_auto_snapshot = true
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'NONE', 'rows_per_partition': 'ALL'}
AND cdc = false
AND comment = ''
AND compaction = {'base_shard_count': '6', 'class':
'org.apache.cassandra.db.compaction.UnifiedCompactionStrategy',
'max_sstables_to_compact': '26', 'min_sstable_size': '9160542B',
'only_purge_repaired_tombstones': 'true', 'overlap_inclusion_method': 'NONE',
'target_sstable_size': '33118701B', 'unchecked_tombstone_compaction': 'false'}
AND compression = {'enabled': 'false'}
AND memtable = 'default'
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND extensions = {}
AND gc_grace_seconds = 864000
AND incremental_backups = true
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair = 'NONE'
AND speculative_retry = '99p';:
org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkBase.MultiNodeState
History:
17: DELETE FROM ks1.tbl USING TIMESTAMP 6 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node3
22: DELETE FROM ks1.tbl USING TIMESTAMP 7 WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
30: DELETE FROM ks1.tbl USING TIMESTAMP 11 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node2
39: UPDATE ks1.tbl USING TIMESTAMP 13 SET v0=[{'
k\u0017\u0008\u0005#y': 599384}], v1=(39, {172317:
00000000-0000-4400-a400-000000000000}, ('_+', -1.1980516747794813E130)) WHERE
pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = false -- on node1
40: INSERT INTO ks1.tbl (pk0, ck0, v0, s0, v1) VALUES
('G@\Poo*X\u000D\u000C', false, null, [[true, false]], (-10, {-69:
00000000-0000-4600-b900-000000000000}, ('\u00149+E', -3.70925155072014E170)))
USING TIMESTAMP 14 -- on node2
55: DELETE FROM ks1.tbl USING TIMESTAMP 18 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node3
95: INSERT INTO ks1.tbl (pk0, ck0, s0, v0, v1) VALUES
('G@\Poo*X\u000D\u000C', false, [[true, true], [true], [true, false]], [{'VF}':
2788132, 'f': -12948532}, {'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0,
'\u001E@:V>I\u0004eM': 391287278, 'Z|6%{R({\u001E?': 509}], (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))) USING TIMESTAMP 23 -- on node2
98: DELETE s0, s0 FROM ks1.tbl USING TIMESTAMP 24 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node1
179: INSERT INTO ks1.tbl (pk0, ck0, v0, v1) VALUES
('G@\Poo*X\u000D\u000C', true, [{'R\u0011': 0}], (-123, {0:
00000000-0000-4600-8f00-000000000000, 13643:
00000000-0000-4700-8800-000000000000}, ('M\u0011$j>\u0018',
9.187280044879176E-293))) USING TIMESTAMP 44 -- on node3
198: UPDATE ks1.tbl USING TIMESTAMP 50 SET s0=[[false]],
v0=[{'"*\u0010D\u0011y|G\u001Bp': 97612, '+\u0002ps': 0}, {'\u0000': -19321,
'\u0005\u0012<': 0, '/kW': 0}, {'CU5^\u0007w': 0, 'N^$\u0019$': 0,
'\u007F,]\u0013pEj\u00124D': 0}], v1=(-101, {0:
00000000-0000-4300-bb00-000000000000, 1: 00000000-0000-4600-ae00-000000000000},
('$1\u0012;\u001A+z', 1.5127319478448008E108)) WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node1
203: DELETE FROM ks1.tbl USING TIMESTAMP 51 WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
209: UPDATE ks1.tbl USING TIMESTAMP 54 SET s0 -= [[true, false,
true], [true], [false]], v0=[{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}], v1=null WHERE pk0 = 'G@\Poo*X\u000D\u000C'
AND ck0 = true -- on node2
214: SELECT * FROM ks1.tbl PER PARTITION LIMIT 889 -- full
table scan, on node3, fetch size 5000
Caused by: java.lang.AssertionError: Unexpected rows found:
pk0 | ck0 | s0 | v0
| v1
'G@\Poo*X\u000D\u000C' | false | [[false]] | [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'G@\Poo*X\u000D\u000C' | true | [[false]] | [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
Missing rows:
pk0 | ck0 | s0 | v0
| v1
'G@\Poo*X\u000D\u000C' | false | null | [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'G@\Poo*X\u000D\u000C' | true | null | [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
Possible column conflicts:
', false, null, [{'VF}': 2788132, 'f': -12948532}, {'-+ciS': 346}, {'': 0,
'@:V>IeM': 391287278, 'Z|6%{R({?': 509}], (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))]
Diff (expected over actual):
s0
null
[[false]]
', true, null, [{' P(': 0, ' BD': 12, '2L ''Z': 0}], null]
Diff (expected over actual):
s0
null
[[false]]
Expected:
pk0 | ck0 | s0
| v0
| v1
'7F\u00105\u001Dl' | false | [[false, false], [false, false], [false,
false, false]] | [{'\u0000\': -31621949}, {'\u00061': -1163531,
'4': 0, '|\u001B\u000CU': 76103}, {'@S': 2299}]
| (-107, {-194598: 00000000-0000-4a00-8700-000000000000, 0:
00000000-0000-4900-a000-000000000000, 33460:
00000000-0000-4500-a200-000000000000}, ('F&8<', 9.796742789142033E-236))
'~\u000A ' | false | [[false, true], [false, false, true]]
| null
| (-116, {0: 00000000-0000-4800-8300-000000000000, 3600001:
00000000-0000-4b00-9e00-000000000000}, ('Z%[', 1.1696299929752362E-275))
'qw@R=n' | true | [[false], [true], [false]]
| null
| (-104, {-1457: 00000000-0000-4000-8e00-000000000000, -248:
00000000-0000-4b00-ba00-000000000000}, ('{', -5.382269316559007E110))
'-v\u000BD' | false | [[false, true, false]]
| [{'\u000Ex': 0, '\u0017\u000C{}7K8>It': 3}]
| (-21, {-299629: 00000000-0000-4300-8700-000000000000},
('\u0000|\u0015T\u00156c\u001D', 6.835547878892394E184))
'qw@R=n' | false | [[false], [true], [false]]
| [{'\u0011\u0000S\u000E\u0005lXE\u001C[': 169}]
| null
'G@\Poo*X\u000D\u000C' | false | null
| [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'\u0011\u001F/-' | false | null
| [{'H\u0005\u001B#': 0}, {'n\u0008\u00168': 0}]
| null
'b\u001C:Y' | true | null
| null
| (24, {-141621475: 00000000-0000-4400-b600-000000000000, 0:
00000000-0000-4900-ba00-000000000000, 50:
00000000-0000-4700-bf00-000000000000}, ('''l]k\u007F', 2.9915069264120252E-40))
'T|~\u0012V\u0005\u001C' | true | [[false, false, true], [false, false, true],
[true], [false], [false]] | [{'\u001BOI(': -29, ' \u001B[jys': 1006}]
| (-7, {-21122: 00000000-0000-4000-be00-000000000000, 72790:
00000000-0000-4700-a300-000000000000}, ('9G7o9O\u007FV\u0005',
-1.9390170589559507E156))
'T|~\u0012V\u0005\u001C' | false | [[false, false, true], [false, false, true],
[true], [false], [false]] | [{'\u0013+K\u0010\u001B': 0, 'mSa\u007F\u000Cs':
-657465}]
| (96, {-229: 00000000-0000-4500-b600-000000000000, 542:
00000000-0000-4900-8400-000000000000}, ('GHpb)C0', 1.0460981817778407E-51))
'G@\Poo*X\u000D\u000C' | true | null
| [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
'-v\u000BD' | true | [[false, true, false]]
| [{'6e\u0004': -115482896, 'fT\u0017r\u0019CdW':
488359, 'f\': 11659}, {'o02': -679505}, {'\u001D\u001Fm~@CAL!': 6}]
| null
{code}
In debugger I see that the -= does generate a tombstone and that this makes it
to the Memtable.
To see if this was localized to table scan I altered the test to do a single
row read after the write, and this found the same issue
{code}
263: SELECT * FROM ks1.tbl WHERE pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = true
ALLOW FILTERING -- Select for Mutation Validation, on node1
{code}
was:
This was found in https://github.com/apache/cassandra/pull/4006,
SHA=7fe995c3b8787bc895c62b9614d408161231259a
Test=org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkWithoutReadRepairTest
{code}
accord.utils.Property$PropertyError: Property error detected:
Seed = -6517659374622108140
Examples = 1
Pure = true
Steps: 400
Values:
State:
Setup:
CREATE KEYSPACE IF NOT EXISTS ks1 WITH replication = {'class':
'SimpleStrategy', 'replication_factor': 3};
CREATE TABLE ks1.tbl (
pk0 ascii,
ck0 boolean,
s0 list<frozen<list<boolean>>> static,
v0 frozen<list<frozen<map<ascii, varint>>>>,
v1 frozen<tuple<tinyint, frozen<map<varint, uuid>>,
frozen<tuple<ascii, double>>>>,
PRIMARY KEY (pk0, ck0)
) WITH CLUSTERING ORDER BY (ck0 ASC)
AND additional_write_policy = '99p'
AND allow_auto_snapshot = true
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'NONE', 'rows_per_partition': 'ALL'}
AND cdc = false
AND comment = ''
AND compaction = {'base_shard_count': '6', 'class':
'org.apache.cassandra.db.compaction.UnifiedCompactionStrategy',
'max_sstables_to_compact': '26', 'min_sstable_size': '9160542B',
'only_purge_repaired_tombstones': 'true', 'overlap_inclusion_method': 'NONE',
'target_sstable_size': '33118701B', 'unchecked_tombstone_compaction': 'false'}
AND compression = {'enabled': 'false'}
AND memtable = 'default'
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND extensions = {}
AND gc_grace_seconds = 864000
AND incremental_backups = true
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair = 'NONE'
AND speculative_retry = '99p';:
org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkBase.MultiNodeState
History:
17: DELETE FROM ks1.tbl USING TIMESTAMP 6 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node3
22: DELETE FROM ks1.tbl USING TIMESTAMP 7 WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
30: DELETE FROM ks1.tbl USING TIMESTAMP 11 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node2
39: UPDATE ks1.tbl USING TIMESTAMP 13 SET v0=[{'
k\u0017\u0008\u0005#y': 599384}], v1=(39, {172317:
00000000-0000-4400-a400-000000000000}, ('_+', -1.1980516747794813E130)) WHERE
pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = false -- on node1
40: INSERT INTO ks1.tbl (pk0, ck0, v0, s0, v1) VALUES
('G@\Poo*X\u000D\u000C', false, null, [[true, false]], (-10, {-69:
00000000-0000-4600-b900-000000000000}, ('\u00149+E', -3.70925155072014E170)))
USING TIMESTAMP 14 -- on node2
55: DELETE FROM ks1.tbl USING TIMESTAMP 18 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node3
95: INSERT INTO ks1.tbl (pk0, ck0, s0, v0, v1) VALUES
('G@\Poo*X\u000D\u000C', false, [[true, true], [true], [true, false]], [{'VF}':
2788132, 'f': -12948532}, {'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0,
'\u001E@:V>I\u0004eM': 391287278, 'Z|6%{R({\u001E?': 509}], (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))) USING TIMESTAMP 23 -- on node2
98: DELETE s0, s0 FROM ks1.tbl USING TIMESTAMP 24 WHERE pk0 =
'G@\Poo*X\u000D\u000C' -- on node1
179: INSERT INTO ks1.tbl (pk0, ck0, v0, v1) VALUES
('G@\Poo*X\u000D\u000C', true, [{'R\u0011': 0}], (-123, {0:
00000000-0000-4600-8f00-000000000000, 13643:
00000000-0000-4700-8800-000000000000}, ('M\u0011$j>\u0018',
9.187280044879176E-293))) USING TIMESTAMP 44 -- on node3
198: UPDATE ks1.tbl USING TIMESTAMP 50 SET s0=[[false]],
v0=[{'"*\u0010D\u0011y|G\u001Bp': 97612, '+\u0002ps': 0}, {'\u0000': -19321,
'\u0005\u0012<': 0, '/kW': 0}, {'CU5^\u0007w': 0, 'N^$\u0019$': 0,
'\u007F,]\u0013pEj\u00124D': 0}], v1=(-101, {0:
00000000-0000-4300-bb00-000000000000, 1: 00000000-0000-4600-ae00-000000000000},
('$1\u0012;\u001A+z', 1.5127319478448008E108)) WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node1
203: DELETE FROM ks1.tbl USING TIMESTAMP 51 WHERE pk0 =
'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
209: UPDATE ks1.tbl USING TIMESTAMP 54 SET s0 -= [[true, false,
true], [true], [false]], v0=[{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}], v1=null WHERE pk0 = 'G@\Poo*X\u000D\u000C'
AND ck0 = true -- on node2
214: SELECT * FROM ks1.tbl PER PARTITION LIMIT 889 -- full
table scan, on node3, fetch size 5000
Caused by: java.lang.AssertionError: Unexpected rows found:
pk0 | ck0 | s0 | v0
| v1
'G@\Poo*X\u000D\u000C' | false | [[false]] | [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'G@\Poo*X\u000D\u000C' | true | [[false]] | [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
Missing rows:
pk0 | ck0 | s0 | v0
| v1
'G@\Poo*X\u000D\u000C' | false | null | [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'G@\Poo*X\u000D\u000C' | true | null | [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
Possible column conflicts:
', false, null, [{'VF}': 2788132, 'f': -12948532}, {'-+ciS': 346}, {'': 0,
'@:V>IeM': 391287278, 'Z|6%{R({?': 509}], (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))]
Diff (expected over actual):
s0
null
[[false]]
', true, null, [{' P(': 0, ' BD': 12, '2L ''Z': 0}], null]
Diff (expected over actual):
s0
null
[[false]]
Expected:
pk0 | ck0 | s0
| v0
| v1
'7F\u00105\u001Dl' | false | [[false, false], [false, false], [false,
false, false]] | [{'\u0000\': -31621949}, {'\u00061': -1163531,
'4': 0, '|\u001B\u000CU': 76103}, {'@S': 2299}]
| (-107, {-194598: 00000000-0000-4a00-8700-000000000000, 0:
00000000-0000-4900-a000-000000000000, 33460:
00000000-0000-4500-a200-000000000000}, ('F&8<', 9.796742789142033E-236))
'~\u000A ' | false | [[false, true], [false, false, true]]
| null
| (-116, {0: 00000000-0000-4800-8300-000000000000, 3600001:
00000000-0000-4b00-9e00-000000000000}, ('Z%[', 1.1696299929752362E-275))
'qw@R=n' | true | [[false], [true], [false]]
| null
| (-104, {-1457: 00000000-0000-4000-8e00-000000000000, -248:
00000000-0000-4b00-ba00-000000000000}, ('{', -5.382269316559007E110))
'-v\u000BD' | false | [[false, true, false]]
| [{'\u000Ex': 0, '\u0017\u000C{}7K8>It': 3}]
| (-21, {-299629: 00000000-0000-4300-8700-000000000000},
('\u0000|\u0015T\u00156c\u001D', 6.835547878892394E184))
'qw@R=n' | false | [[false], [true], [false]]
| [{'\u0011\u0000S\u000E\u0005lXE\u001C[': 169}]
| null
'G@\Poo*X\u000D\u000C' | false | null
| [{'VF}': 2788132, 'f': -12948532},
{'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
00000000-0000-4e00-9600-000000000000, -117652:
00000000-0000-4100-bb00-000000000000, 0: 00000000-0000-4000-b100-000000000000},
('''gUu', 5.352521969766626E-196))
'\u0011\u001F/-' | false | null
| [{'H\u0005\u001B#': 0}, {'n\u0008\u00168': 0}]
| null
'b\u001C:Y' | true | null
| null
| (24, {-141621475: 00000000-0000-4400-b600-000000000000, 0:
00000000-0000-4900-ba00-000000000000, 50:
00000000-0000-4700-bf00-000000000000}, ('''l]k\u007F', 2.9915069264120252E-40))
'T|~\u0012V\u0005\u001C' | true | [[false, false, true], [false, false, true],
[true], [false], [false]] | [{'\u001BOI(': -29, ' \u001B[jys': 1006}]
| (-7, {-21122: 00000000-0000-4000-be00-000000000000, 72790:
00000000-0000-4700-a300-000000000000}, ('9G7o9O\u007FV\u0005',
-1.9390170589559507E156))
'T|~\u0012V\u0005\u001C' | false | [[false, false, true], [false, false, true],
[true], [false], [false]] | [{'\u0013+K\u0010\u001B': 0, 'mSa\u007F\u000Cs':
-657465}]
| (96, {-229: 00000000-0000-4500-b600-000000000000, 542:
00000000-0000-4900-8400-000000000000}, ('GHpb)C0', 1.0460981817778407E-51))
'G@\Poo*X\u000D\u000C' | true | null
| [{' \u0011P(': 0, ' BD\u0012': 12,
'2\u000BL\u001E\u0013 ''Z': 0}]
| null
'-v\u000BD' | true | [[false, true, false]]
| [{'6e\u0004': -115482896, 'fT\u0017r\u0019CdW':
488359, 'f\': 11659}, {'o02': -679505}, {'\u001D\u001Fm~@CAL!': 6}]
| null
{code}
In debugger I see that the -= does generate a tombstone and that this makes it
to the Memtable.
To see if this was localized to table scan I altered the test to do a single
row read after the write, and this found the same issue
{code}
263: SELECT * FROM ks1.tbl WHERE pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = true
ALLOW FILTERING -- Select for Mutation Validation, on node1
{code}
> tombstone from list discard (-=) not observed from viewers
> ----------------------------------------------------------
>
> Key: CASSANDRA-20487
> URL: https://issues.apache.org/jira/browse/CASSANDRA-20487
> Project: Apache Cassandra
> Issue Type: Bug
> Components: CQL/Semantics
> Reporter: David Capwell
> Priority: Normal
> Fix For: 5.x
>
>
> This was found in https://github.com/apache/cassandra/pull/4006
> SHA=7fe995c3b8787bc895c62b9614d408161231259a
> Test=org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkWithoutReadRepairTest
> {code}
> accord.utils.Property$PropertyError: Property error detected:
> Seed = -6517659374622108140
> Examples = 1
> Pure = true
> Steps: 400
> Values:
> State:
> Setup:
> CREATE KEYSPACE IF NOT EXISTS ks1 WITH replication = {'class':
> 'SimpleStrategy', 'replication_factor': 3};
> CREATE TABLE ks1.tbl (
> pk0 ascii,
> ck0 boolean,
> s0 list<frozen<list<boolean>>> static,
> v0 frozen<list<frozen<map<ascii, varint>>>>,
> v1 frozen<tuple<tinyint, frozen<map<varint, uuid>>,
> frozen<tuple<ascii, double>>>>,
> PRIMARY KEY (pk0, ck0)
> ) WITH CLUSTERING ORDER BY (ck0 ASC)
> AND additional_write_policy = '99p'
> AND allow_auto_snapshot = true
> AND bloom_filter_fp_chance = 0.01
> AND caching = {'keys': 'NONE', 'rows_per_partition': 'ALL'}
> AND cdc = false
> AND comment = ''
> AND compaction = {'base_shard_count': '6', 'class':
> 'org.apache.cassandra.db.compaction.UnifiedCompactionStrategy',
> 'max_sstables_to_compact': '26', 'min_sstable_size': '9160542B',
> 'only_purge_repaired_tombstones': 'true', 'overlap_inclusion_method': 'NONE',
> 'target_sstable_size': '33118701B', 'unchecked_tombstone_compaction': 'false'}
> AND compression = {'enabled': 'false'}
> AND memtable = 'default'
> AND crc_check_chance = 1.0
> AND default_time_to_live = 0
> AND extensions = {}
> AND gc_grace_seconds = 864000
> AND incremental_backups = true
> AND max_index_interval = 2048
> AND memtable_flush_period_in_ms = 0
> AND min_index_interval = 128
> AND read_repair = 'NONE'
> AND speculative_retry = '99p';:
> org.apache.cassandra.distributed.test.cql3.MultiNodeTableWalkBase.MultiNodeState
> History:
> 17: DELETE FROM ks1.tbl USING TIMESTAMP 6 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' -- on node3
> 22: DELETE FROM ks1.tbl USING TIMESTAMP 7 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
> 30: DELETE FROM ks1.tbl USING TIMESTAMP 11 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' -- on node2
> 39: UPDATE ks1.tbl USING TIMESTAMP 13 SET v0=[{'
> k\u0017\u0008\u0005#y': 599384}], v1=(39, {172317:
> 00000000-0000-4400-a400-000000000000}, ('_+', -1.1980516747794813E130)) WHERE
> pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = false -- on node1
> 40: INSERT INTO ks1.tbl (pk0, ck0, v0, s0, v1) VALUES
> ('G@\Poo*X\u000D\u000C', false, null, [[true, false]], (-10, {-69:
> 00000000-0000-4600-b900-000000000000}, ('\u00149+E', -3.70925155072014E170)))
> USING TIMESTAMP 14 -- on node2
> 55: DELETE FROM ks1.tbl USING TIMESTAMP 18 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' -- on node3
> 95: INSERT INTO ks1.tbl (pk0, ck0, s0, v0, v1) VALUES
> ('G@\Poo*X\u000D\u000C', false, [[true, true], [true], [true, false]],
> [{'VF}': 2788132, 'f': -12948532}, {'-+ciS\u001A\u0011\u001C': 346},
> {'\u001A': 0, '\u001E@:V>I\u0004eM': 391287278, 'Z|6%{R({\u001E?': 509}],
> (-90, {-92140986: 00000000-0000-4e00-9600-000000000000, -117652:
> 00000000-0000-4100-bb00-000000000000, 0:
> 00000000-0000-4000-b100-000000000000}, ('''gUu', 5.352521969766626E-196)))
> USING TIMESTAMP 23 -- on node2
> 98: DELETE s0, s0 FROM ks1.tbl USING TIMESTAMP 24 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' -- on node1
> 179: INSERT INTO ks1.tbl (pk0, ck0, v0, v1) VALUES
> ('G@\Poo*X\u000D\u000C', true, [{'R\u0011': 0}], (-123, {0:
> 00000000-0000-4600-8f00-000000000000, 13643:
> 00000000-0000-4700-8800-000000000000}, ('M\u0011$j>\u0018',
> 9.187280044879176E-293))) USING TIMESTAMP 44 -- on node3
> 198: UPDATE ks1.tbl USING TIMESTAMP 50 SET s0=[[false]],
> v0=[{'"*\u0010D\u0011y|G\u001Bp': 97612, '+\u0002ps': 0}, {'\u0000': -19321,
> '\u0005\u0012<': 0, '/kW': 0}, {'CU5^\u0007w': 0, 'N^$\u0019$': 0,
> '\u007F,]\u0013pEj\u00124D': 0}], v1=(-101, {0:
> 00000000-0000-4300-bb00-000000000000, 1:
> 00000000-0000-4600-ae00-000000000000}, ('$1\u0012;\u001A+z',
> 1.5127319478448008E108)) WHERE pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = true
> -- on node1
> 203: DELETE FROM ks1.tbl USING TIMESTAMP 51 WHERE pk0 =
> 'G@\Poo*X\u000D\u000C' AND ck0 = true -- on node3
> 209: UPDATE ks1.tbl USING TIMESTAMP 54 SET s0 -= [[true, false,
> true], [true], [false]], v0=[{' \u0011P(': 0, ' BD\u0012': 12,
> '2\u000BL\u001E\u0013 ''Z': 0}], v1=null WHERE pk0 = 'G@\Poo*X\u000D\u000C'
> AND ck0 = true -- on node2
> 214: SELECT * FROM ks1.tbl PER PARTITION LIMIT 889 -- full
> table scan, on node3, fetch size 5000
> Caused by: java.lang.AssertionError: Unexpected rows found:
> pk0 | ck0 | s0 | v0
>
> | v1
>
>
> 'G@\Poo*X\u000D\u000C' | false | [[false]] | [{'VF}': 2788132, 'f':
> -12948532}, {'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0,
> '\u001E@:V>I\u0004eM': 391287278, 'Z|6%{R({\u001E?': 509}] | (-90,
> {-92140986: 00000000-0000-4e00-9600-000000000000, -117652:
> 00000000-0000-4100-bb00-000000000000, 0:
> 00000000-0000-4000-b100-000000000000}, ('''gUu', 5.352521969766626E-196))
> 'G@\Poo*X\u000D\u000C' | true | [[false]] | [{' \u0011P(': 0, ' BD\u0012':
> 12, '2\u000BL\u001E\u0013 ''Z': 0}]
> | null
>
>
> Missing rows:
> pk0 | ck0 | s0 | v0
>
> | v1
>
>
> 'G@\Poo*X\u000D\u000C' | false | null | [{'VF}': 2788132, 'f': -12948532},
> {'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
> 391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
> 00000000-0000-4e00-9600-000000000000, -117652:
> 00000000-0000-4100-bb00-000000000000, 0:
> 00000000-0000-4000-b100-000000000000}, ('''gUu', 5.352521969766626E-196))
> 'G@\Poo*X\u000D\u000C' | true | null | [{' \u0011P(': 0, ' BD\u0012': 12,
> '2\u000BL\u001E\u0013 ''Z': 0}]
> | null
>
>
> Possible column conflicts:
> ', false, null, [{'VF}': 2788132, 'f': -12948532}, {'-+ciS': 346}, {'': 0,
> '@:V>IeM': 391287278, 'Z|6%{R({?': 509}], (-90, {-92140986:
> 00000000-0000-4e00-9600-000000000000, -117652:
> 00000000-0000-4100-bb00-000000000000, 0:
> 00000000-0000-4000-b100-000000000000}, ('''gUu', 5.352521969766626E-196))]
> Diff (expected over actual):
> s0
> null
> [[false]]
> ', true, null, [{' P(': 0, ' BD': 12, '2L ''Z': 0}], null]
> Diff (expected over actual):
> s0
> null
> [[false]]
> Expected:
> pk0 | ck0 | s0
> | v0
>
> | v1
>
>
> '7F\u00105\u001Dl' | false | [[false, false], [false, false], [false,
> false, false]] | [{'\u0000\': -31621949}, {'\u00061':
> -1163531, '4': 0, '|\u001B\u000CU': 76103}, {'@S': 2299}]
> | (-107, {-194598:
> 00000000-0000-4a00-8700-000000000000, 0:
> 00000000-0000-4900-a000-000000000000, 33460:
> 00000000-0000-4500-a200-000000000000}, ('F&8<', 9.796742789142033E-236))
> '~\u000A ' | false | [[false, true], [false, false, true]]
> | null
>
> | (-116, {0: 00000000-0000-4800-8300-000000000000, 3600001:
> 00000000-0000-4b00-9e00-000000000000}, ('Z%[', 1.1696299929752362E-275))
>
> 'qw@R=n' | true | [[false], [true], [false]]
> | null
>
> | (-104, {-1457: 00000000-0000-4000-8e00-000000000000, -248:
> 00000000-0000-4b00-ba00-000000000000}, ('{', -5.382269316559007E110))
>
> '-v\u000BD' | false | [[false, true, false]]
> | [{'\u000Ex': 0, '\u0017\u000C{}7K8>It': 3}]
>
> | (-21, {-299629: 00000000-0000-4300-8700-000000000000},
> ('\u0000|\u0015T\u00156c\u001D', 6.835547878892394E184))
>
> 'qw@R=n' | false | [[false], [true], [false]]
> | [{'\u0011\u0000S\u000E\u0005lXE\u001C[': 169}]
>
> | null
>
>
> 'G@\Poo*X\u000D\u000C' | false | null
> | [{'VF}': 2788132, 'f': -12948532},
> {'-+ciS\u001A\u0011\u001C': 346}, {'\u001A': 0, '\u001E@:V>I\u0004eM':
> 391287278, 'Z|6%{R({\u001E?': 509}] | (-90, {-92140986:
> 00000000-0000-4e00-9600-000000000000, -117652:
> 00000000-0000-4100-bb00-000000000000, 0:
> 00000000-0000-4000-b100-000000000000}, ('''gUu', 5.352521969766626E-196))
> '\u0011\u001F/-' | false | null
> | [{'H\u0005\u001B#': 0}, {'n\u0008\u00168': 0}]
>
> | null
>
>
> 'b\u001C:Y' | true | null
> | null
>
> | (24, {-141621475: 00000000-0000-4400-b600-000000000000, 0:
> 00000000-0000-4900-ba00-000000000000, 50:
> 00000000-0000-4700-bf00-000000000000}, ('''l]k\u007F',
> 2.9915069264120252E-40))
> 'T|~\u0012V\u0005\u001C' | true | [[false, false, true], [false, false,
> true], [true], [false], [false]] | [{'\u001BOI(': -29, ' \u001B[jys': 1006}]
>
> | (-7, {-21122: 00000000-0000-4000-be00-000000000000,
> 72790: 00000000-0000-4700-a300-000000000000}, ('9G7o9O\u007FV\u0005',
> -1.9390170589559507E156))
> 'T|~\u0012V\u0005\u001C' | false | [[false, false, true], [false, false,
> true], [true], [false], [false]] | [{'\u0013+K\u0010\u001B': 0,
> 'mSa\u007F\u000Cs': -657465}]
> | (96, {-229:
> 00000000-0000-4500-b600-000000000000, 542:
> 00000000-0000-4900-8400-000000000000}, ('GHpb)C0', 1.0460981817778407E-51))
>
> 'G@\Poo*X\u000D\u000C' | true | null
> | [{' \u0011P(': 0, ' BD\u0012': 12,
> '2\u000BL\u001E\u0013 ''Z': 0}]
> | null
>
>
> '-v\u000BD' | true | [[false, true, false]]
> | [{'6e\u0004': -115482896, 'fT\u0017r\u0019CdW':
> 488359, 'f\': 11659}, {'o02': -679505}, {'\u001D\u001Fm~@CAL!': 6}]
> | null
>
>
> {code}
> In debugger I see that the -= does generate a tombstone and that this makes
> it to the Memtable.
> To see if this was localized to table scan I altered the test to do a single
> row read after the write, and this found the same issue
> {code}
> 263: SELECT * FROM ks1.tbl WHERE pk0 = 'G@\Poo*X\u000D\u000C' AND ck0 = true
> ALLOW FILTERING -- Select for Mutation Validation, on node1
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
