[
https://issues.apache.org/jira/browse/CASSANDRA-14136?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16305216#comment-16305216
]
Aditya Bharadwaj edited comment on CASSANDRA-14136 at 1/3/18 8:58 PM:
----------------------------------------------------------------------
Looks related. Seen this issue 38 times in the last 1 month, in all but 1
scenario, it seems to have come while writing to a secondary index.
This is the exception
bq. java.lang.RuntimeException: Last written key
DecoratedKey(-6638873113115166967, 81e77da2723b483d8c0d49f800c1e288) >= current
key DecoratedKey(-8794293631676762023,
9130cbbaa8e911e79641aba7018ec35280ee0000000800000000000007b6000000080000000000005bcc00000006d8add8acd9850000000ad983d8a8d98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000200000001000400026172000000000000343600000000007e4d080000000000001e06000000000000000182ed830ffc0590395ad535f900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bce080cd985d8aad988d8b3d8b7d8a9083dcccccd01081091408324a8e911e7ae1b578dff8303d380f00000000800000000000007b6000000080000000000005bce00000006d8add8acd9850000000cd985d8aad988d8b3d8b7d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000300000001000400026172000000000000343600000000007e4d080000000000001e08000000000000000182e98314fc0590395c213df900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bd0080ad8b5d8bad98ad8b1d8a9083dcccccd010810914d7b73a8e911e7ae1b578dff8303d380ee0000000800000000000007b6000000080000000000005bd000000006d8add8acd9850000000ad8b5d8bad98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000400000001000400026172000000000000343600000000007e4d3a0000000000001e22000000000000000183148310fc0889665cb9adf90400050030080850697a7a61487574080632303732363408046e756c6c0c080543727573740c0c0c0c080a435530303231353738320c0c0c0c080000000000009c400c080000000000000000080000000000000001084099999a0c0800000000000000000812d985d8a7d8b1d8acd8a7d8b1d98ad8aad8a70801410800000000007e4d3a0c0c0c08104d6172676865726974612043727573740c080c4f6d6e4372742d4d726774610c0815d986d988d8b920d8a7d984d8b9d8acd9)
writing into
/mnt/DATA/cassandra/data/product/productdetails_by_storeid_variants-ec590ad0108611e7a92033b648576005/mc-25-big-Data.db
Even in this scenario, this table has a secondary index though.
{quote}CREATE TABLE prduct.productdetails_by_storeid_variants (
merchantid uuid,
languagecode text,
storeid bigint,
productid bigint,
variantproductid bigint,
quantity bigint,
brand text,
brandid text,
bundlegroups text,
catalogcode text,
deliverymode text,
deliverytime text,
h1catname text,
h2catname text,
h3catname text,
h4catname text,
h5catname text,
hierarchy1category text,
hierarchy2category text,
hierarchy3category text,
hierarchy4category text,
hierarchy5category text,
image text,
inventory bigint,
largeimage text,
longdescription text,
maximumorderquantity bigint,
minimumorderquantity bigint,
mrp float,
offerdesc text,
primaryproductid bigint,
producttitle text,
producttype text,
refid bigint,
seodescription text,
seokeywords text,
seopagetitle text,
seourlkey text,
shortdescription text,
sku text,
smallimage text,
tags text,
variantproducts list<frozen<variant>>,
variantproperty text,
variantpropertyvalueid bigint,
variantvalue text,
webprice float,
PRIMARY KEY (merchantid, languagecode, storeid, productid,
variantproductid, quantity)
) WITH CLUSTERING ORDER BY (languagecode ASC, storeid ASC, productid ASC,
variantproductid ASC, quantity ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 86400
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
CREATE INDEX productdetails_by_storeid_variants_refid_idx ON
product.productdetails_by_storeid_variants (refid);{quote}
was (Author: adityabharadwaj):
Looks related. Seen this issue 38 times in the last 1 month, in all but 1
scenario, it seems to have come while writing to a secondary index.
This is the exception
bq. java.lang.RuntimeException: Last written key
DecoratedKey(-6638873113115166967, 81e77da2723b483d8c0d49f800c1e288) >= current
key DecoratedKey(-8794293631676762023,
9130cbbaa8e911e79641aba7018ec35280ee0000000800000000000007b6000000080000000000005bcc00000006d8add8acd9850000000ad983d8a8d98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000200000001000400026172000000000000343600000000007e4d080000000000001e06000000000000000182ed830ffc0590395ad535f900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bce080cd985d8aad988d8b3d8b7d8a9083dcccccd01081091408324a8e911e7ae1b578dff8303d380f00000000800000000000007b6000000080000000000005bce00000006d8add8acd9850000000cd985d8aad988d8b3d8b7d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000300000001000400026172000000000000343600000000007e4d080000000000001e08000000000000000182e98314fc0590395c213df900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bd0080ad8b5d8bad98ad8b1d8a9083dcccccd010810914d7b73a8e911e7ae1b578dff8303d380ee0000000800000000000007b6000000080000000000005bd000000006d8add8acd9850000000ad8b5d8bad98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000400000001000400026172000000000000343600000000007e4d3a0000000000001e22000000000000000183148310fc0889665cb9adf90400050030080850697a7a61487574080632303732363408046e756c6c0c080543727573740c0c0c0c080a435530303231353738320c0c0c0c080000000000009c400c080000000000000000080000000000000001084099999a0c0800000000000000000812d985d8a7d8b1d8acd8a7d8b1d98ad8aad8a70801410800000000007e4d3a0c0c0c08104d6172676865726974612043727573740c080c4f6d6e4372742d4d726774610c0815d986d988d8b920d8a7d984d8b9d8acd9)
writing into
/mnt/DATA/cassandra/data/martjack/productdetails_by_storeid_variants-ec590ad0108611e7a92033b648576005/mc-25-big-Data.db
Even in this scenario, this table has a secondary index though.
{quote}CREATE TABLE martjack.productdetails_by_storeid_variants (
merchantid uuid,
languagecode text,
storeid bigint,
productid bigint,
variantproductid bigint,
quantity bigint,
brand text,
brandid text,
bundlegroups text,
catalogcode text,
deliverymode text,
deliverytime text,
h1catname text,
h2catname text,
h3catname text,
h4catname text,
h5catname text,
hierarchy1category text,
hierarchy2category text,
hierarchy3category text,
hierarchy4category text,
hierarchy5category text,
image text,
inventory bigint,
largeimage text,
longdescription text,
maximumorderquantity bigint,
minimumorderquantity bigint,
mrp float,
offerdesc text,
primaryproductid bigint,
producttitle text,
producttype text,
refid bigint,
seodescription text,
seokeywords text,
seopagetitle text,
seourlkey text,
shortdescription text,
sku text,
smallimage text,
tags text,
variantproducts list<frozen<variant>>,
variantproperty text,
variantpropertyvalueid bigint,
variantvalue text,
webprice float,
PRIMARY KEY (merchantid, languagecode, storeid, productid,
variantproductid, quantity)
) WITH CLUSTERING ORDER BY (languagecode ASC, storeid ASC, productid ASC,
variantproductid ASC, quantity ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 86400
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
CREATE INDEX productdetails_by_storeid_variants_refid_idx ON
martjack.productdetails_by_storeid_variants (refid);{quote}
> MemtableFlushWriter DecoratedKey Exception
> ------------------------------------------
>
> Key: CASSANDRA-14136
> URL: https://issues.apache.org/jira/browse/CASSANDRA-14136
> Project: Cassandra
> Issue Type: Bug
> Components: Core, Secondary Indexes
> Environment: CentOS Linux release 7.3.1611 (Core) , 3 Node cluster
> with Replication Factor or 2
> Reporter: Aditya Bharadwaj
> Attachments: debug.log.gz, system.log.gz
>
>
> Running into this issue on my cluster periodically for different tables.
> After this error is encountered, all the post flushes stop and eventually the
> system runs out of memory.
> On a restart all the commit logs get played normally and things go back to
> normal.
> I'm unable to understand the scenario, but the issue is recreating every few
> days.
> {code}DEBUG [MemtableFlushWriter:884] 2017-12-26 18:19:40,883
> Memtable.java:401 - Completed flushing
> /mnt/DATA/cassandra/data/products/products_by_hierarchy5storeid_pascdesc-411cabe0632411e7b25a1b665c06298b/.id
> x_hierarchy1category/mc-2050-big-Data.db (508.127KiB) for commitlog position
> ReplayPosition(segmentId=1513929386900,
> position=19110822)
> DEBUG [MemtableFlushWriter:884] 2017-12-26 18:19:41,150 Memtable.java:368 -
> Writing
> Memtable-products_by_hierarchy5storeid_pascdesc.idx_hierarchy3category@551487729(545.926KiB
> serialized bytes, 324073 ops, 0%/0% of on/off-heap limit)
> ERROR [MemtableFlushWriter:884] 2017-12-26 18:19:41,316
> CassandraDaemon.java:205 - Exception in thread
> Thread[MemtableFlushWriter:884,5,main]
> java.lang.RuntimeException: Last written key DecoratedKey(CU00328612,
> 43553030333238363132) >= current key DecoratedKey(^@^@^@^@^@^@^@^@^@^@,
> 43553030333238363838) writing into
> /mnt/DATA/cassandra/data/products/products_by_hierarchy5storeid_pascdesc-411cabe0632411e7b25a1b665c06298b/.idx_hierarchy3category/mc-2134-big-Data.db
> at
> org.apache.cassandra.io.sstable.format.big.BigTableWriter.beforeAppend(BigTableWriter.java:106)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> org.apache.cassandra.io.sstable.format.big.BigTableWriter.append(BigTableWriter.java:145)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter.append(SimpleSSTableMultiWriter.java:45)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> org.apache.cassandra.io.sstable.SSTableTxnWriter.append(SSTableTxnWriter.java:52)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> org.apache.cassandra.db.Memtable.writeSortedContents(Memtable.java:394)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at org.apache.cassandra.db.Memtable.flush(Memtable.java:332)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1054)
> ~[apache-cassandra-3.0.9.jar:3.0.9]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> ~[na:1.8.0_112]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> ~[na:1.8.0_112]
> at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_112]
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
