cassandra git commit: Fix queries on static compact tables
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 df65a6c68 -> 235032050
Fix queries on static compact tables
patch by slebresne; reviewed by iamaleskey for CASSANDRA-10093
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23503205
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23503205
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23503205
Branch: refs/heads/cassandra-3.0
Commit: 2350320501bcd26085181b929f4718cb12d08b41
Parents: df65a6c
Author: Sylvain Lebresne
Authored: Mon Aug 17 15:00:41 2015 +0200
Committer: Sylvain Lebresne
Committed: Tue Aug 18 09:04:24 2015 +0200
--
CHANGES.txt | 1 +
.../cql3/restrictions/StatementRestrictions.java | 11 ++-
.../cassandra/cql3/statements/SelectStatement.java | 5 -
src/java/org/apache/cassandra/db/Clustering.java | 1 +
4 files changed, 16 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index b4b568c..54a6a07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.0-beta1
+ * Fix query on static compact tables (CASSANDRA-10093)
* Fix race during construction of commit log (CASSANDRA-10049)
* Add option to only purge repaired tombstones (CASSANDRA-6434)
* Change authorization handling for MVs (CASSANDRA-9927)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
--
diff --git
a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 1a3b083..36f11ad 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.btree.BTreeSet;
import static
org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static
org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -478,6 +479,12 @@ public final class StatementRestrictions
*/
public NavigableSet getClusteringColumns(QueryOptions options)
throws InvalidRequestException
{
+// If this is a names command and the table is a static compact one,
then as far as CQL is concerned we have
+// only a single row which internally correspond to the static parts.
In which case we want to return an empty
+// set (since that's what ClusteringIndexNamesFilter expects).
+if (cfm.isStaticCompactTable())
+return BTreeSet.empty(cfm.comparator);
+
return clusteringColumnsRestrictions.valuesAsClustering(options);
}
@@ -513,7 +520,9 @@ public final class StatementRestrictions
*/
public boolean isColumnRange()
{
-// For static compact tables we need to ignore the fake clustering
column.
+// For static compact tables we want to ignore the fake clustering
column (note that if we weren't special casing,
+// this would mean a 'SELECT *' on a static compact table would query
whole partitions, even though we'll only return
+// the static part as far as CQL is concerned. This is thus mostly an
optimization to use the query-by-name path).
int numberOfClusteringColumns = cfm.isStaticCompactTable() ? 0 :
cfm.clusteringColumns().size();
// it is a range query if it has at least one the column alias for
which no relation is defined or is not EQ.
return clusteringColumnsRestrictions.size() < numberOfClusteringColumns
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
--
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 9acbb35..01ce8e6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -484,7 +484,10 @@ public class SelectStatement implements CQLStatement
else
{
NavigableSet clusterings = getRequestedRows(options);
-if (clusterings.isEmp
[2/2] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/30a704e0 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/30a704e0 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/30a704e0 Branch: refs/heads/trunk Commit: 30a704e01b18f2a6446db28a3d00cd4551fe59ca Parents: e939773 2350320 Author: Sylvain Lebresne Authored: Tue Aug 18 09:05:30 2015 +0200 Committer: Sylvain Lebresne Committed: Tue Aug 18 09:05:30 2015 +0200 -- CHANGES.txt | 1 + .../cql3/restrictions/StatementRestrictions.java | 11 ++- .../cassandra/cql3/statements/SelectStatement.java | 5 - src/java/org/apache/cassandra/db/Clustering.java | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/30a704e0/CHANGES.txt -- diff --cc CHANGES.txt index 5d8797f,54a6a07..eba9e36 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,8 -1,5 +1,9 @@@ +3.2 + * Add transparent data encryption core classes (CASSANDRA-9945) + + 3.0.0-beta1 + * Fix query on static compact tables (CASSANDRA-10093) * Fix race during construction of commit log (CASSANDRA-10049) * Add option to only purge repaired tombstones (CASSANDRA-6434) * Change authorization handling for MVs (CASSANDRA-9927)
[1/2] cassandra git commit: Fix queries on static compact tables
Repository: cassandra
Updated Branches:
refs/heads/trunk e93977310 -> 30a704e01
Fix queries on static compact tables
patch by slebresne; reviewed by iamaleskey for CASSANDRA-10093
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23503205
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23503205
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23503205
Branch: refs/heads/trunk
Commit: 2350320501bcd26085181b929f4718cb12d08b41
Parents: df65a6c
Author: Sylvain Lebresne
Authored: Mon Aug 17 15:00:41 2015 +0200
Committer: Sylvain Lebresne
Committed: Tue Aug 18 09:04:24 2015 +0200
--
CHANGES.txt | 1 +
.../cql3/restrictions/StatementRestrictions.java | 11 ++-
.../cassandra/cql3/statements/SelectStatement.java | 5 -
src/java/org/apache/cassandra/db/Clustering.java | 1 +
4 files changed, 16 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index b4b568c..54a6a07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.0-beta1
+ * Fix query on static compact tables (CASSANDRA-10093)
* Fix race during construction of commit log (CASSANDRA-10049)
* Add option to only purge repaired tombstones (CASSANDRA-6434)
* Change authorization handling for MVs (CASSANDRA-9927)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
--
diff --git
a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 1a3b083..36f11ad 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.btree.BTreeSet;
import static
org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
import static
org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
@@ -478,6 +479,12 @@ public final class StatementRestrictions
*/
public NavigableSet getClusteringColumns(QueryOptions options)
throws InvalidRequestException
{
+// If this is a names command and the table is a static compact one,
then as far as CQL is concerned we have
+// only a single row which internally correspond to the static parts.
In which case we want to return an empty
+// set (since that's what ClusteringIndexNamesFilter expects).
+if (cfm.isStaticCompactTable())
+return BTreeSet.empty(cfm.comparator);
+
return clusteringColumnsRestrictions.valuesAsClustering(options);
}
@@ -513,7 +520,9 @@ public final class StatementRestrictions
*/
public boolean isColumnRange()
{
-// For static compact tables we need to ignore the fake clustering
column.
+// For static compact tables we want to ignore the fake clustering
column (note that if we weren't special casing,
+// this would mean a 'SELECT *' on a static compact table would query
whole partitions, even though we'll only return
+// the static part as far as CQL is concerned. This is thus mostly an
optimization to use the query-by-name path).
int numberOfClusteringColumns = cfm.isStaticCompactTable() ? 0 :
cfm.clusteringColumns().size();
// it is a range query if it has at least one the column alias for
which no relation is defined or is not EQ.
return clusteringColumnsRestrictions.size() < numberOfClusteringColumns
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23503205/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
--
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 9acbb35..01ce8e6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -484,7 +484,10 @@ public class SelectStatement implements CQLStatement
else
{
NavigableSet clusterings = getRequestedRows(options);
-if (clusterings.isEmpty() && !selecti
[jira] [Commented] (CASSANDRA-10094) Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM failure
[
https://issues.apache.org/jira/browse/CASSANDRA-10094?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700858#comment-14700858
]
Benedict commented on CASSANDRA-10094:
--
I'm not sure what went wrong, to be honest. I committed the 2.2 patch and
merged upwards, but I was notified 3.0 and trunk were still failing, so figured
I'd messed up by not taking the pre-merged 3.0 when merging upwards beyond 2.2,
so applied this only as a correction. I guess I had really applied the prior
version of the patch. I'll apply the backport shortly.
> Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM
> failure
> ---
>
> Key: CASSANDRA-10094
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10094
> Project: Cassandra
> Issue Type: Bug
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 2.2.x
>
> Attachments: 10094-2.2.txt
>
>
> Error:
> {noformat}
> junit.framework.AssertionFailedError:
> at
> org.apache.cassandra.db.CommitLogFailurePolicyTest.testCommitLogFailureBeforeInitialization_mustKillJVM(CommitLogFailurePolicyTest.java:149)
> {noformat}
> [Failure
> History|http://cassci.datastax.com/view/cassandra-2.2/job/cassandra-2.2_utest_win32/lastCompletedBuild/testReport/org.apache.cassandra.db/CommitLogFailurePolicyTest/testCommitLogFailureBeforeInitialization_mustKillJVM/history/]:
> Consistent since build #85
> Env: CI only. Cannot repro locally
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-7066) Simplify (and unify) cleanup of compaction leftovers
[
https://issues.apache.org/jira/browse/CASSANDRA-7066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700888#comment-14700888
]
Benedict commented on CASSANDRA-7066:
-
I think it would be nice for clarity here to make {{FileUtils.readLines}} and
{{FileUtils.append}} accept a charset, so that we can ensure they are mirrored.
My initial reaction was that these were dependent on the system defaults and
that this change would lead to a bug.
> Simplify (and unify) cleanup of compaction leftovers
>
>
> Key: CASSANDRA-7066
> URL: https://issues.apache.org/jira/browse/CASSANDRA-7066
> Project: Cassandra
> Issue Type: Improvement
> Components: Core
>Reporter: Benedict
>Assignee: Stefania
>Priority: Minor
> Labels: benedict-to-commit, compaction
> Fix For: 3.0 alpha 1
>
> Attachments: 7066.txt
>
>
> Currently we manage a list of in-progress compactions in a system table,
> which we use to cleanup incomplete compactions when we're done. The problem
> with this is that 1) it's a bit clunky (and leaves us in positions where we
> can unnecessarily cleanup completed files, or conversely not cleanup files
> that have been superceded); and 2) it's only used for a regular compaction -
> no other compaction types are guarded in the same way, so can result in
> duplication if we fail before deleting the replacements.
> I'd like to see each sstable store in its metadata its direct ancestors, and
> on startup we simply delete any sstables that occur in the union of all
> ancestor sets. This way as soon as we finish writing we're capable of
> cleaning up any leftovers, so we never get duplication. It's also much easier
> to reason about.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Created] (CASSANDRA-10114) Allow count(*) and count(1) to be use as normal aggregation
Benjamin Lerer created CASSANDRA-10114:
--
Summary: Allow count(*) and count(1) to be use as normal
aggregation
Key: CASSANDRA-10114
URL: https://issues.apache.org/jira/browse/CASSANDRA-10114
Project: Cassandra
Issue Type: Bug
Reporter: Benjamin Lerer
Assignee: Benjamin Lerer
Priority: Minor
Fix For: 2.2.x
For the following query:
{code}
SELECT count(*), max(timestamp), min(timestamp) FROM myData WHERE id = ?
{code}
Cassandra will throw a {{InvalidSyntaxException}}.
We should allow count(*) and count(1) to be queried with other aggregations or
columns
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10072) "Replica(s) failed to execute read" on simple select on stress-created table with >1 nodes
[
https://issues.apache.org/jira/browse/CASSANDRA-10072?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700898#comment-14700898
]
Marcus Eriksson commented on CASSANDRA-10072:
-
The script works on my machine (see below)
Do you run the correct python driver? (iirc 3.0 requires the cassandra-testing
branch of python-driver) If you do, you will need to attach more logs etc
{code}
oss/cassandra [trunk●] » sh repro.sh
trunk, 1 nodes
select-failure does not appear to be a valid cluster (use ccm list to view
valid clusters)
http://git-wip-us.apache.org/repos/asf/cassandra.git git:trunk
Cloning Cassandra...
Cloning Cassandra (from local cache)
Checking out requested branch (trunk)
Compiling Cassandra trunk ...
Current cluster is now: select-failure
Failed to connect over JMX; not collecting these stats
[Row(key='0P37709P21',
C0='\x85\x18/\xdf\xe7\xf8c\x06\xcdX\xce\x81\xaaS\xcc\xa2L\x198A\xd6\xae\x8a\x91djL0M\xd9\xf4x\x92\xf9',
C1='\x1f\\\x9f\x8c0\xa6,\xe4\x81^\x93m\xe8\x14QN\xa3>\xd4\xd8_\xa3?\x90"-X\xa5\xa0\xa4\x9b8\xd2\xfe',
C2='"U\x10\x93+\xd8\x81+F\x10\x81vS\xb7\x9c\x86U,\x99\xae\xfb\x17\x7fL\xef\xa4\x85\n\x919\xdbinn',
C3='\xd7\xe0\x99\xa5\x9d\xa5Y\xe09`\x0fn,\x0e\xde\x94\xba\xda\x8c\xfe]\x8dO\xf2mh\xffY};?h\xf2\xb4',
C4='\xfb\xdd\x9c\xec\x02O\xcb\xdeC\x83\x99g\x04u\xaa\x89\x00\xd8\x0e\x0e\xd3\xd0\xc31-\x9dJ\xe7\x92A!Mt\xc3')]
trunk, 2 nodes
http://git-wip-us.apache.org/repos/asf/cassandra.git git:trunk
Fetching Cassandra updates...
Current cluster is now: select-failure
Failed to connect over JMX; not collecting these stats
[Row(key='0P37709P21',
C0='\x85\x18/\xdf\xe7\xf8c\x06\xcdX\xce\x81\xaaS\xcc\xa2L\x198A\xd6\xae\x8a\x91djL0M\xd9\xf4x\x92\xf9',
C1='\x1f\\\x9f\x8c0\xa6,\xe4\x81^\x93m\xe8\x14QN\xa3>\xd4\xd8_\xa3?\x90"-X\xa5\xa0\xa4\x9b8\xd2\xfe',
C2='"U\x10\x93+\xd8\x81+F\x10\x81vS\xb7\x9c\x86U,\x99\xae\xfb\x17\x7fL\xef\xa4\x85\n\x919\xdbinn',
C3='\xd7\xe0\x99\xa5\x9d\xa5Y\xe09`\x0fn,\x0e\xde\x94\xba\xda\x8c\xfe]\x8dO\xf2mh\xffY};?h\xf2\xb4',
C4='\xfb\xdd\x9c\xec\x02O\xcb\xdeC\x83\x99g\x04u\xaa\x89\x00\xd8\x0e\x0e\xd3\xd0\xc31-\x9dJ\xe7\x92A!Mt\xc3')]
cassandra-2.2, 1 nodes
http://git-wip-us.apache.org/repos/asf/cassandra.git git:cassandra-2.2
Fetching Cassandra updates...
Cloning Cassandra (from local cache)
Checking out requested branch (cassandra-2.2)
Compiling Cassandra cassandra-2.2 ...
Current cluster is now: select-failure
Failed to connect over JMX; not collecting these stats
[Row(key='0P37709P21',
C0='\x85\x18/\xdf\xe7\xf8c\x06\xcdX\xce\x81\xaaS\xcc\xa2L\x198A\xd6\xae\x8a\x91djL0M\xd9\xf4x\x92\xf9',
C1='\x1f\\\x9f\x8c0\xa6,\xe4\x81^\x93m\xe8\x14QN\xa3>\xd4\xd8_\xa3?\x90"-X\xa5\xa0\xa4\x9b8\xd2\xfe',
C2='"U\x10\x93+\xd8\x81+F\x10\x81vS\xb7\x9c\x86U,\x99\xae\xfb\x17\x7fL\xef\xa4\x85\n\x919\xdbinn',
C3='\xd7\xe0\x99\xa5\x9d\xa5Y\xe09`\x0fn,\x0e\xde\x94\xba\xda\x8c\xfe]\x8dO\xf2mh\xffY};?h\xf2\xb4',
C4='\xfb\xdd\x9c\xec\x02O\xcb\xdeC\x83\x99g\x04u\xaa\x89\x00\xd8\x0e\x0e\xd3\xd0\xc31-\x9dJ\xe7\x92A!Mt\xc3')]
cassandra-2.2, 2 nodes
http://git-wip-us.apache.org/repos/asf/cassandra.git git:cassandra-2.2
Fetching Cassandra updates...
Current cluster is now: select-failure
Failed to connect over JMX; not collecting these stats
[Row(key='0P37709P21',
C0='\x85\x18/\xdf\xe7\xf8c\x06\xcdX\xce\x81\xaaS\xcc\xa2L\x198A\xd6\xae\x8a\x91djL0M\xd9\xf4x\x92\xf9',
C1='\x1f\\\x9f\x8c0\xa6,\xe4\x81^\x93m\xe8\x14QN\xa3>\xd4\xd8_\xa3?\x90"-X\xa5\xa0\xa4\x9b8\xd2\xfe',
C2='"U\x10\x93+\xd8\x81+F\x10\x81vS\xb7\x9c\x86U,\x99\xae\xfb\x17\x7fL\xef\xa4\x85\n\x919\xdbinn',
C3='\xd7\xe0\x99\xa5\x9d\xa5Y\xe09`\x0fn,\x0e\xde\x94\xba\xda\x8c\xfe]\x8dO\xf2mh\xffY};?h\xf2\xb4',
C4='\xfb\xdd\x9c\xec\x02O\xcb\xdeC\x83\x99g\x04u\xaa\x89\x00\xd8\x0e\x0e\xd3\xd0\xc31-\x9dJ\xe7\x92A!Mt\xc3')]
cassandra-3.0, 1 nodes
http://git-wip-us.apache.org/repos/asf/cassandra.git git:cassandra-3.0
Fetching Cassandra updates...
Cloning Cassandra (from local cache)
Checking out requested branch (cassandra-3.0)
Compiling Cassandra cassandra-3.0 ...
Current cluster is now: select-failure
Failed to connect over JMX; not collecting these stats
[Row(key='0P37709P21',
C0='\x85\x18/\xdf\xe7\xf8c\x06\xcdX\xce\x81\xaaS\xcc\xa2L\x198A\xd6\xae\x8a\x91djL0M\xd9\xf4x\x92\xf9',
C1='\x1f\\\x9f\x8c0\xa6,\xe4\x81^\x93m\xe8\x14QN\xa3>\xd4\xd8_\xa3?\x90"-X\xa5\xa0\xa4\x9b8\xd2\xfe',
C2='"U\x10\x93+\xd8\x81+F\x10\x81vS\xb7\x9c\x86U,\x9
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700906#comment-14700906
]
Benedict commented on CASSANDRA-10109:
--
Reading the Windows documentation it does look like there really are no
guarantees. At least, it states absolutely no guarantees. The files are each
returned by independent method calls, leaving me with even less confidence if
they don't state it outright.
So, yes, I'd say the best thing we can do is to read all of the files in the
directory into a buffer, then read all of the transaction files we encounter,
and then classify the transaction files based on presence of all old files. If
some are missing, then we assume the transaction has been committed; if all are
present we assume it's in progress. We should make behaviour in the case of
missing files optional, as in many cases missing the new files would be fine, I
suspect. There's always the risk that a file is deleted after it's found and
returned, so the user has to be robust to that. All we care about is ensuring
we return a consistent snapshot, or a subset thereof.
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-8716) "java.util.concurrent.ExecutionException: java.lang.AssertionError: Memory was freed" when running cleanup
[
https://issues.apache.org/jira/browse/CASSANDRA-8716?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700907#comment-14700907
]
Benedict commented on CASSANDRA-8716:
-
It's a very small patch. You can checkout 2.0.12, apply the patch, and build
yourself a snapshot in the meantime.
> "java.util.concurrent.ExecutionException: java.lang.AssertionError: Memory
> was freed" when running cleanup
> --
>
> Key: CASSANDRA-8716
> URL: https://issues.apache.org/jira/browse/CASSANDRA-8716
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Environment: Centos 6.6, Cassandra 2.0.12, Oracle JDK 1.7.0_67
>Reporter: Imri Zvik
>Assignee: Robert Stupp
>Priority: Minor
> Labels: qa-resolved
> Fix For: 2.0.13
>
> Attachments: 8716.txt, system.log.gz
>
>
> {code}Error occurred during cleanup
> java.util.concurrent.ExecutionException: java.lang.AssertionError: Memory was
> freed
> at java.util.concurrent.FutureTask.report(FutureTask.java:122)
> at java.util.concurrent.FutureTask.get(FutureTask.java:188)
> at
> org.apache.cassandra.db.compaction.CompactionManager.performAllSSTableOperation(CompactionManager.java:234)
> at
> org.apache.cassandra.db.compaction.CompactionManager.performCleanup(CompactionManager.java:272)
> at
> org.apache.cassandra.db.ColumnFamilyStore.forceCleanup(ColumnFamilyStore.java:1115)
> at
> org.apache.cassandra.service.StorageService.forceKeyspaceCleanup(StorageService.java:2177)
> at sun.reflect.GeneratedMethodAccessor29.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75)
> at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
> at
> com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
> at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
> at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1487)
> at
> javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:97)
> at
> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1328)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1420)
> at
> javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:848)
> at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322)
> at sun.rmi.transport.Transport$1.run(Transport.java:177)
> at sun.rmi.transport.Transport$1.run(Transport.java:174)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.rmi.transport.Transport.serviceCall(Transport.java:173)
> at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:556)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:811)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:670)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.AssertionError: Memory was freed
> at org.apache.cassandra.io.util.Memory.checkPosition(Memory.java:259)
> at org.apache.cassandra.io.util.Memory.getInt(Memory.java:211)
> at
> org.apache.cassandra.io.sstable.I
[jira] [Commented] (CASSANDRA-10055) High CPU load for Cassandra 2.1.8
[ https://issues.apache.org/jira/browse/CASSANDRA-10055?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700912#comment-14700912 ] Benedict commented on CASSANDRA-10055: -- If you could attach a profiler to the process and collect CPU profiles over a short period that exhibits this problem it might help. It's possible the issue exists only for a short time horizon. > High CPU load for Cassandra 2.1.8 > - > > Key: CASSANDRA-10055 > URL: https://issues.apache.org/jira/browse/CASSANDRA-10055 > Project: Cassandra > Issue Type: Bug > Components: Config > Environment: Prod >Reporter: vijay > Attachments: dstst-lcdn.log, dstst-lcdn2.log, dstst-lcdn3.log, > dstst-lcdn4.log, dstst-lcdn5.log, dstst-lcdn6.log, js.log, js2.log, js3.log, > js4.log, js5.log, js6.log, top-bHn1-2.log, top-bHn1-3.log, top-bHn1-4.log, > top-bHn1-5.log, top-bHn1-6.log, top-bHn1.log > > > We are seeing High CPU Load about 80% to 100% in Cassandra 2.1.8 when doing > Data ingest, we did not had this issue in 2.0.x version of Cassandra > we tested this in different Cloud platforms and results are same. > CPU: Tested with M3 2xlarge AWS instances > Ingest rate: Injecting 1 million Inserts each insert is of 1000bytes > no other Operations are happening except inserts in Cassandra > let me know if more info is needed. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9901) Make AbstractType.isByteOrderComparable abstract
[
https://issues.apache.org/jira/browse/CASSANDRA-9901?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700916#comment-14700916
]
Benedict commented on CASSANDRA-9901:
-
bq. In fact, I'm not even really sure a warning is necessary in the first place.
I'm fine with dropping it.
Otherwise I'm fine with these proposals. Or, if you prefer, I don't mind having
an abstract {{isByteOrderComparable()}} and memoizing the result of it to a
boolean property during construction.
> Make AbstractType.isByteOrderComparable abstract
>
>
> Key: CASSANDRA-9901
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9901
> Project: Cassandra
> Issue Type: Improvement
> Components: Core
>Reporter: Benedict
>Assignee: Benedict
> Fix For: 3.0 beta 2
>
>
> I can't recall _precisely_ what was agreed at the NGCC, but I'm reasonably
> sure we agreed to make this method abstract, put some javadoc explaining we
> may require fields to yield true in the near future, and potentially log a
> warning on startup if a user-defined type returns false.
> This should make it into 3.0, IMO, so that we can look into migrating to
> byte-order comparable types in the post-3.0 world.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[ https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Stefania updated CASSANDRA-8630: Attachment: mmaped_uncomp_hotspot.png > Faster sequential IO (on compaction, streaming, etc) > > > Key: CASSANDRA-8630 > URL: https://issues.apache.org/jira/browse/CASSANDRA-8630 > Project: Cassandra > Issue Type: Improvement > Components: Core, Tools >Reporter: Oleg Anastasyev >Assignee: Stefania > Labels: compaction, performance > Fix For: 3.x > > Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png, > flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz, > mmaped_uncomp_hotspot.png > > > When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot > of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int). > This is because default implementations of readShort,readLong, etc as well as > their matching write* are implemented with numerous calls of byte by byte > read and write. > This makes a lot of syscalls as well. > A quick microbench shows than just reimplementation of these methods in > either way gives 8x speed increase. > A patch attached implements RandomAccessReader.read and > SequencialWriter.write methods in more efficient way. > I also eliminated some extra byte copies in CompositeType.split and > ColumnNameHelper.maxComponents, which were on my profiler's hotspot method > list during tests. > A stress tests on my laptop show that this patch makes compaction 25-30% > faster on uncompressed sstables and 15% faster for compressed ones. > A deployment to production shows much less CPU load for compaction. > (I attached a cpu load graph from one of our production, orange is niced CPU > load - i.e. compaction; yellow is user - i.e. not compaction related tasks) -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9901) Make AbstractType.isByteOrderComparable abstract
[
https://issues.apache.org/jira/browse/CASSANDRA-9901?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700929#comment-14700929
]
Sylvain Lebresne commented on CASSANDRA-9901:
-
bq. Or, if you prefer, I don't mind having an abstract
{{isByteOrderComparable()}} and memoizing the result of it to a boolean
property during construction.
Nah, I'm fine with the enum after all. I do like the option of being able to
declare the type as incomparable to enforce it's never used in a clustering.
> Make AbstractType.isByteOrderComparable abstract
>
>
> Key: CASSANDRA-9901
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9901
> Project: Cassandra
> Issue Type: Improvement
> Components: Core
>Reporter: Benedict
>Assignee: Benedict
> Fix For: 3.0 beta 2
>
>
> I can't recall _precisely_ what was agreed at the NGCC, but I'm reasonably
> sure we agreed to make this method abstract, put some javadoc explaining we
> may require fields to yield true in the near future, and potentially log a
> warning on startup if a user-defined type returns false.
> This should make it into 3.0, IMO, so that we can look into migrating to
> byte-order comparable types in the post-3.0 world.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-10043) A NullPointerException is thrown if the column name is unknown for an IN relation
[
https://issues.apache.org/jira/browse/CASSANDRA-10043?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Robert Stupp updated CASSANDRA-10043:
-
Reviewer: Robert Stupp
> A NullPointerException is thrown if the column name is unknown for an IN
> relation
> -
>
> Key: CASSANDRA-10043
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10043
> Project: Cassandra
> Issue Type: Bug
>Reporter: Benjamin Lerer
>Assignee: Benjamin Lerer
> Attachments: 10043-2.2.txt, 10043-3.0.txt
>
>
> {code}
> cqlsh:test> create table newTable (a int, b int, c int, primary key(a, b));
> cqlsh:test> select * from newTable where d in (1, 2);
> ServerError: message="java.lang.NullPointerException">
> {code}
> The problem seems to occur only for {{IN}} restrictions
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9893) Fix upgrade tests from #9704 that are still failing
[ https://issues.apache.org/jira/browse/CASSANDRA-9893?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700943#comment-14700943 ] Sylvain Lebresne commented on CASSANDRA-9893: - Did we ever committed Tyler's branch to the dtests, and or have it run by cascci in any way? If not, can someone have a look at it ASAP so we can get on top of upgrade failures are quickly as possible? [~enigmacurry] [~philipthompson] > Fix upgrade tests from #9704 that are still failing > --- > > Key: CASSANDRA-9893 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9893 > Project: Cassandra > Issue Type: Bug >Reporter: Sylvain Lebresne >Assignee: Blake Eggleston > Fix For: 3.0 beta 2 > > > The first things to do on this ticket would be to commit Tyler's branch > (https://github.com/thobbs/cassandra-dtest/tree/8099-backwards-compat) to the > dtests so cassci run them. I've had to do a few minor modifications to have > them run locally so someone which access to cassci should do it and make sure > it runs properly. > Once we have that, we should fix any test that isn't passing. I've ran the > tests locally and I had 8 failures. for 2 of them, it sounds plausible that > they'll get fixed by the patch of CASSANDRA-9775, though that's just a guess. > The rest where test that timeouted without a particular error in the log, > and running some of them individually, they passed. So we'll have to see if > it's just my machine being overly slow when running them all. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[
https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700948#comment-14700948
]
Stefania commented on CASSANDRA-8630:
-
The slowness with the uncompressed mmaped segments is caused by the rate
limiter, which ultimately comes from the compaction throughput, see
_mmaped_uncomp_hotspot.png_ attached. Whereas before we were simply looping on
a sorted list of mmaped segments and returning a {{ByteBufferDataInput}} for
each one of them, now we have a sorted map of segments that are swapped in or
out by the RAR rebuffer method. Because previously we would apply the rate
limiter to the rebuffer method, mmaped segments became much slower.
If we apply the rate limiter only just before reading, as opposite to every
time rebuffer is called, here are the results:
||Version||Run 1||Run 2||Run 3||Rounded AVG||
|8630 comp|17.48|16.77|16.26|17|
|8630 uncomp|15.51|17.5|17.7|17|
|TRUNK comp|17.95|17.64|17.72|18|
|TRUNK uncomp|20.81|20.01|18.81|20|
I am not sure I understand fully why the compressed case was not affected as
much, these segments are pretty big also for the uncompressed case. I also
would like to know if there is a way to have flight recorder look at the total
time rather than just the CPU time, without [visual
vm|https://visualvm.java.net] I would not have been able to find this.
> Faster sequential IO (on compaction, streaming, etc)
>
>
> Key: CASSANDRA-8630
> URL: https://issues.apache.org/jira/browse/CASSANDRA-8630
> Project: Cassandra
> Issue Type: Improvement
> Components: Core, Tools
>Reporter: Oleg Anastasyev
>Assignee: Stefania
> Labels: compaction, performance
> Fix For: 3.x
>
> Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png,
> flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz,
> mmaped_uncomp_hotspot.png
>
>
> When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot
> of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int).
> This is because default implementations of readShort,readLong, etc as well as
> their matching write* are implemented with numerous calls of byte by byte
> read and write.
> This makes a lot of syscalls as well.
> A quick microbench shows than just reimplementation of these methods in
> either way gives 8x speed increase.
> A patch attached implements RandomAccessReader.read and
> SequencialWriter.write methods in more efficient way.
> I also eliminated some extra byte copies in CompositeType.split and
> ColumnNameHelper.maxComponents, which were on my profiler's hotspot method
> list during tests.
> A stress tests on my laptop show that this patch makes compaction 25-30%
> faster on uncompressed sstables and 15% faster for compressed ones.
> A deployment to production shows much less CPU load for compaction.
> (I attached a cpu load graph from one of our production, orange is niced CPU
> load - i.e. compaction; yellow is user - i.e. not compaction related tasks)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10082) Transactional classes shouldn't also implement streams, channels, etc
[ https://issues.apache.org/jira/browse/CASSANDRA-10082?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700955#comment-14700955 ] Benedict commented on CASSANDRA-10082: -- bq. as it seems a confusingly arbitrary method call for changing the semantics. Your comments don't address this. The method you've introduced has the following problems with it: # It is called getOutputStream, and SequentialWriter is already an OutputStream # It gives you an OS with different semantics, but backed by the same SW, _both of which exist at the same time_ #* This is especially implied as OK, given the "get" semantics, as though it were a property, despite its effectively overriding the transactionality of the SW # In no way does it let the user know there has been a change in semantics My point of making a "general purpose" utility was to avoid all of this, by creating a wrapper class that clearly modifies the semantics of a Tranasctional OutputStream, returning you a non-transactional one. It could live in SW if you prefer, but it should make it very clear what is happening, and should _take as its argument_ the SW to make it very clear the SW no longer exists as a separate entity, but has been encapsulated. It's not a major modification. That all said, I would also really much rather we decoupled the sstable-specific behaviour of SW from those we might employ elsewhere, and avoid this problem further up the hierarchy. The whole shebang needs a refactor. This is fine as a stop gap, but it's all considerably uglier than it should be. > Transactional classes shouldn't also implement streams, channels, etc > - > > Key: CASSANDRA-10082 > URL: https://issues.apache.org/jira/browse/CASSANDRA-10082 > Project: Cassandra > Issue Type: Improvement >Reporter: Blake Eggleston >Assignee: Blake Eggleston > Attachments: > 0001-replacing-SequentialWriter-OutputStream-extension-wi.patch > > > Since the close method on the Transactional interface means "abort if commit > hasn't been called", mixing Transactional and AutoCloseable interfaces where > close means "we're done here" is pretty much never the right thing to do. > The only class that does this is SequentialWriter. It's not used in a way > that causes a problem, but it's still a potential hazard for future > development. > The attached patch replaces the SequentialWriter OutputStream implementation > with a wrapper class that implements the expected behavior on close, and adds > a warning to the Transactional interface. It also adds a unit test that > demonstrates the problem without the fix. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[1/2] cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 26ff15070 -> 755a6b244
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/cassandra-2.1
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:47:04 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
-@VisibleForTesting
-long lastExpiredCheck;
+private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
Set uncompacting = cfs.getUncompactingSSTables();
Set expired = Collections.emptySet();
-// we only check for expired sstables every 10 minutes due to it being
an expensive operation
-if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+// we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION =
TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
= 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY =
"timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY
= "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays *
timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ?
DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+e
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2
Merge branch 'cassandra-2.1' into cassandra-2.2 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/09147a8b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/09147a8b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/09147a8b Branch: refs/heads/cassandra-2.2 Commit: 09147a8b6c2ced97a05d185c02bedbf1a59978e1 Parents: ee186b6 755a6b2 Author: Marcus Eriksson Authored: Tue Aug 18 10:51:56 2015 +0200 Committer: Marcus Eriksson Committed: Tue Aug 18 10:51:56 2015 +0200 -- .../DateTieredCompactionStrategy.java | 7 +++ .../DateTieredCompactionStrategyOptions.java| 21 .../DateTieredCompactionStrategyTest.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java --
cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.0 f53aacba8 -> 17042f19b
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/cassandra-2.0
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:47:04 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
-@VisibleForTesting
-long lastExpiredCheck;
+private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
Set uncompacting = cfs.getUncompactingSSTables();
Set expired = Collections.emptySet();
-// we only check for expired sstables every 10 minutes due to it being
an expensive operation
-if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+// we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION =
TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
= 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY =
"timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY
= "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays *
timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ?
DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+e
[1/3] cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 ee186b6b0 -> 09147a8b6
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/cassandra-2.2
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:47:04 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
-@VisibleForTesting
-long lastExpiredCheck;
+private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
Set uncompacting = cfs.getUncompactingSSTables();
Set expired = Collections.emptySet();
-// we only check for expired sstables every 10 minutes due to it being
an expensive operation
-if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+// we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION =
TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
= 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY =
"timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY
= "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays *
timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ?
DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+e
[2/2] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/755a6b24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/755a6b24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/755a6b24
Branch: refs/heads/cassandra-2.1
Commit: 755a6b244c7c0db34044f03b9a62c2070e033a78
Parents: 26ff150 17042f1
Author: Marcus Eriksson
Authored: Tue Aug 18 10:51:38 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:51:38 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 41c304b,2e43352..a8e2aff
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -36,11 -36,9 +36,10 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger =
LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
-protected DateTieredCompactionStrategyOptions options;
+private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+private final Set sstables = new HashSet<>();
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@@ -79,14 -80,14 +78,14 @@@
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (!isEnabled() || cfs.getSSTables().isEmpty())
+if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
-Set uncompacting = cfs.getUncompactingSSTables();
+Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cea835f,6d9bfc2..f05bf44
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -305,12 -305,10 +305,12 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
+for (SSTableReader sstable : cfs.getSSTables())
+dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis()/1000));
assertNotNull(t);
[4/4] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6c69f07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6c69f07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6c69f07c
Branch: refs/heads/cassandra-3.0
Commit: 6c69f07c2ed3a24398a55811f8c8cad326c3e2ce
Parents: 2350320 09147a8
Author: Marcus Eriksson
Authored: Tue Aug 18 10:56:06 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:56:06 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a908884,0956962..ad14a9c
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -85,17 -81,17 +84,17 @@@ public class DateTieredCompactionStrate
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (cfs.getSSTables().isEmpty())
+if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE)))
return Collections.emptyList();
-Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
+Set uncompacting =
ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains));
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
-expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting),
gcBefore);
lastExpiredCheck = System.currentTimeMillis();
}
Set candidates =
Sets.newHashSet(filterSuspectSSTables(uncompacting));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 7b3b9e7,0159c83..003a1cf
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -319,8 -316,9 +319,9 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
-for (SSTableReader sstable : cfs.getSSTables())
+for (SSTableReader sstable : cfs.getLiveSSTables())
dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
[2/4] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/755a6b24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/755a6b24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/755a6b24
Branch: refs/heads/cassandra-3.0
Commit: 755a6b244c7c0db34044f03b9a62c2070e033a78
Parents: 26ff150 17042f1
Author: Marcus Eriksson
Authored: Tue Aug 18 10:51:38 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:51:38 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 41c304b,2e43352..a8e2aff
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -36,11 -36,9 +36,10 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger =
LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
-protected DateTieredCompactionStrategyOptions options;
+private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+private final Set sstables = new HashSet<>();
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@@ -79,14 -80,14 +78,14 @@@
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (!isEnabled() || cfs.getSSTables().isEmpty())
+if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
-Set uncompacting = cfs.getUncompactingSSTables();
+Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cea835f,6d9bfc2..f05bf44
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -305,12 -305,10 +305,12 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
+for (SSTableReader sstable : cfs.getSSTables())
+dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis()/1000));
assertNotNull(t);
[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/755a6b24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/755a6b24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/755a6b24
Branch: refs/heads/cassandra-2.2
Commit: 755a6b244c7c0db34044f03b9a62c2070e033a78
Parents: 26ff150 17042f1
Author: Marcus Eriksson
Authored: Tue Aug 18 10:51:38 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:51:38 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 41c304b,2e43352..a8e2aff
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -36,11 -36,9 +36,10 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger =
LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
-protected DateTieredCompactionStrategyOptions options;
+private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+private final Set sstables = new HashSet<>();
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@@ -79,14 -80,14 +78,14 @@@
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (!isEnabled() || cfs.getSSTables().isEmpty())
+if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
-Set uncompacting = cfs.getUncompactingSSTables();
+Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cea835f,6d9bfc2..f05bf44
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -305,12 -305,10 +305,12 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
+for (SSTableReader sstable : cfs.getSSTables())
+dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis()/1000));
assertNotNull(t);
[1/4] cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 235032050 -> 6c69f07c2
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/cassandra-3.0
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:47:04 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
-@VisibleForTesting
-long lastExpiredCheck;
+private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
Set uncompacting = cfs.getUncompactingSSTables();
Set expired = Collections.emptySet();
-// we only check for expired sstables every 10 minutes due to it being
an expensive operation
-if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+// we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION =
TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
= 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY =
"timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY
= "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays *
timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ?
DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+e
[4/5] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6c69f07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6c69f07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6c69f07c
Branch: refs/heads/trunk
Commit: 6c69f07c2ed3a24398a55811f8c8cad326c3e2ce
Parents: 2350320 09147a8
Author: Marcus Eriksson
Authored: Tue Aug 18 10:56:06 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:56:06 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a908884,0956962..ad14a9c
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -85,17 -81,17 +84,17 @@@ public class DateTieredCompactionStrate
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (cfs.getSSTables().isEmpty())
+if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE)))
return Collections.emptyList();
-Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
+Set uncompacting =
ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains));
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
-expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting),
gcBefore);
lastExpiredCheck = System.currentTimeMillis();
}
Set candidates =
Sets.newHashSet(filterSuspectSSTables(uncompacting));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 7b3b9e7,0159c83..003a1cf
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -319,8 -316,9 +319,9 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
-for (SSTableReader sstable : cfs.getSSTables())
+for (SSTableReader sstable : cfs.getLiveSSTables())
dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
[1/5] cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/trunk 30a704e01 -> 91e3d3421
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/trunk
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:47:04 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
-@VisibleForTesting
-long lastExpiredCheck;
+private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends
AbstractCompactionStrategy
Set uncompacting = cfs.getUncompactingSSTables();
Set expired = Collections.emptySet();
-// we only check for expired sstables every 10 minutes due to it being
an expensive operation
-if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+// we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
--
diff --git
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION =
TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
= 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY =
"timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY
= "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays *
timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ?
DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+expiredSSTableChe
[3/5] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2
Merge branch 'cassandra-2.1' into cassandra-2.2 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/09147a8b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/09147a8b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/09147a8b Branch: refs/heads/trunk Commit: 09147a8b6c2ced97a05d185c02bedbf1a59978e1 Parents: ee186b6 755a6b2 Author: Marcus Eriksson Authored: Tue Aug 18 10:51:56 2015 +0200 Committer: Marcus Eriksson Committed: Tue Aug 18 10:51:56 2015 +0200 -- .../DateTieredCompactionStrategy.java | 7 +++ .../DateTieredCompactionStrategyOptions.java| 21 .../DateTieredCompactionStrategyTest.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java --
[3/4] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2
Merge branch 'cassandra-2.1' into cassandra-2.2 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/09147a8b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/09147a8b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/09147a8b Branch: refs/heads/cassandra-3.0 Commit: 09147a8b6c2ced97a05d185c02bedbf1a59978e1 Parents: ee186b6 755a6b2 Author: Marcus Eriksson Authored: Tue Aug 18 10:51:56 2015 +0200 Committer: Marcus Eriksson Committed: Tue Aug 18 10:51:56 2015 +0200 -- .../DateTieredCompactionStrategy.java | 7 +++ .../DateTieredCompactionStrategyOptions.java| 21 .../DateTieredCompactionStrategyTest.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java --
[5/5] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/91e3d342 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/91e3d342 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/91e3d342 Branch: refs/heads/trunk Commit: 91e3d34210e593f3ae03070865d1f346f3b3bac7 Parents: 30a704e 6c69f07 Author: Marcus Eriksson Authored: Tue Aug 18 10:59:53 2015 +0200 Committer: Marcus Eriksson Committed: Tue Aug 18 10:59:53 2015 +0200 -- .../DateTieredCompactionStrategy.java | 7 +++ .../DateTieredCompactionStrategyOptions.java| 21 .../DateTieredCompactionStrategyTest.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) --
[2/5] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/755a6b24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/755a6b24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/755a6b24
Branch: refs/heads/trunk
Commit: 755a6b244c7c0db34044f03b9a62c2070e033a78
Parents: 26ff150 17042f1
Author: Marcus Eriksson
Authored: Tue Aug 18 10:51:38 2015 +0200
Committer: Marcus Eriksson
Committed: Tue Aug 18 10:51:38 2015 +0200
--
.../DateTieredCompactionStrategy.java | 7 +++
.../DateTieredCompactionStrategyOptions.java| 21
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 41c304b,2e43352..a8e2aff
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -36,11 -36,9 +36,10 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger =
LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
-protected DateTieredCompactionStrategyOptions options;
+private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+private final Set sstables = new HashSet<>();
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
{
@@@ -79,14 -80,14 +78,14 @@@
*/
private List getNextBackgroundSSTables(final int gcBefore)
{
-if (!isEnabled() || cfs.getSSTables().isEmpty())
+if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
-Set uncompacting = cfs.getUncompactingSSTables();
+Set uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
Set expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it
being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default)
due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter
what.
expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
--
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cea835f,6d9bfc2..f05bf44
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -305,12 -305,10 +305,12 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60;
+
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
+for (SSTableReader sstable : cfs.getSSTables())
+dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis()/1000));
assertNotNull(t);
[jira] [Commented] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[
https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700970#comment-14700970
]
Benedict commented on CASSANDRA-8630:
-
Probably due to the burst-limit of the rate limiter. If you rebuffer all 100
sstables at once, you're perhaps exceeding the default 1s burst limit, and so
find yourself sleeping often. Sleeping has a coarse granularity, so sleeping
may last longer than intended by the {{RateLimiter}}.
We should probably test this with rate limiting off, anyway, though. To see
what unthrottled throughput looks like.
> Faster sequential IO (on compaction, streaming, etc)
>
>
> Key: CASSANDRA-8630
> URL: https://issues.apache.org/jira/browse/CASSANDRA-8630
> Project: Cassandra
> Issue Type: Improvement
> Components: Core, Tools
>Reporter: Oleg Anastasyev
>Assignee: Stefania
> Labels: compaction, performance
> Fix For: 3.x
>
> Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png,
> flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz,
> mmaped_uncomp_hotspot.png
>
>
> When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot
> of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int).
> This is because default implementations of readShort,readLong, etc as well as
> their matching write* are implemented with numerous calls of byte by byte
> read and write.
> This makes a lot of syscalls as well.
> A quick microbench shows than just reimplementation of these methods in
> either way gives 8x speed increase.
> A patch attached implements RandomAccessReader.read and
> SequencialWriter.write methods in more efficient way.
> I also eliminated some extra byte copies in CompositeType.split and
> ColumnNameHelper.maxComponents, which were on my profiler's hotspot method
> list during tests.
> A stress tests on my laptop show that this patch makes compaction 25-30%
> faster on uncompressed sstables and 15% faster for compressed ones.
> A deployment to production shows much less CPU load for compaction.
> (I attached a cpu load graph from one of our production, orange is niced CPU
> load - i.e. compaction; yellow is user - i.e. not compaction related tasks)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-7066) Simplify (and unify) cleanup of compaction leftovers
[
https://issues.apache.org/jira/browse/CASSANDRA-7066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700967#comment-14700967
]
Stefania commented on CASSANDRA-7066:
-
Thanks for spotting this, I hadn't noticed it. See if this [amended
commit|https://github.com/stef1927/cassandra/commit/f5d6f310d2f3136728d4773b996d824f588039a2]
is better. Instead of passing an argument I exported the charset from
{{FileUtils}} since it shouldn't really change.
Probably best to wait for CI:
http://cassci.datastax.com/view/Dev/view/stef1927/job/stef1927-7066-b-dtest/
http://cassci.datastax.com/view/Dev/view/stef1927/job/stef1927-7066-b-testall/
> Simplify (and unify) cleanup of compaction leftovers
>
>
> Key: CASSANDRA-7066
> URL: https://issues.apache.org/jira/browse/CASSANDRA-7066
> Project: Cassandra
> Issue Type: Improvement
> Components: Core
>Reporter: Benedict
>Assignee: Stefania
>Priority: Minor
> Labels: benedict-to-commit, compaction
> Fix For: 3.0 alpha 1
>
> Attachments: 7066.txt
>
>
> Currently we manage a list of in-progress compactions in a system table,
> which we use to cleanup incomplete compactions when we're done. The problem
> with this is that 1) it's a bit clunky (and leaves us in positions where we
> can unnecessarily cleanup completed files, or conversely not cleanup files
> that have been superceded); and 2) it's only used for a regular compaction -
> no other compaction types are guarded in the same way, so can result in
> duplication if we fail before deleting the replacements.
> I'd like to see each sstable store in its metadata its direct ancestors, and
> on startup we simply delete any sstables that occur in the union of all
> ancestor sets. This way as soon as we finish writing we're capable of
> cleaning up any leftovers, so we never get duplication. It's also much easier
> to reason about.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[3/6] cassandra git commit: Fix CommitLogFailurePolicyTest
Fix CommitLogFailurePolicyTest
patch by paulo motta for CASSANDRA-10094
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d61c026f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d61c026f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d61c026f
Branch: refs/heads/trunk
Commit: d61c026fd723390e9d916e7a18ff9bdad019ee2c
Parents: 09147a8
Author: Benedict Elliott Smith
Authored: Mon Aug 17 09:52:13 2015 +0100
Committer: Benedict Elliott Smith
Committed: Tue Aug 18 10:10:22 2015 +0100
--
.../db/CommitLogFailurePolicyTest.java | 64 ++--
1 file changed, 5 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d61c026f/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
--
diff --git a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
index cca6503..0ecab3c 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db;
-import java.io.File;
-
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +26,6 @@ import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.service.CassandraDaemon;
@@ -38,7 +35,6 @@ import org.apache.cassandra.utils.KillerForTests;
public class CommitLogFailurePolicyTest
{
-
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
@@ -95,57 +91,20 @@ public class CommitLogFailurePolicyTest
}
@Test
-public void testCommitFailurePolicy_mustDieIfNotStartedUp()
-{
-//startup was not completed successfuly (since method completeSetup()
was not called)
-CassandraDaemon daemon = new CassandraDaemon();
-StorageService.instance.registerDaemon(daemon);
-
-KillerForTests killerForTests = new KillerForTests();
-JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
-Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
-try
-{
-//even though policy is ignore, JVM must die because Daemon has
not finished initializing
-
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-CommitLog.handleCommitError("Testing die policy", new Throwable());
-Assert.assertTrue(killerForTests.wasKilled());
-Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed
quietly due to startup failure
-}
-finally
-{
-DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
-JVMStabilityInspector.replaceKiller(originalKiller);
-}
-}
-
-@Test
-public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws
Exception
+public void testCommitFailurePolicy_ignore_beforeStartup()
{
//startup was not completed successfuly (since method completeSetup()
was not called)
CassandraDaemon daemon = new CassandraDaemon();
StorageService.instance.registerDaemon(daemon);
-//let's make the commit log directory non-writable
-File commitLogDir = new
File(DatabaseDescriptor.getCommitLogLocation());
-commitLogDir.setWritable(false);
-
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
try
{
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-
-//now let's create a commit log segment manager and wait for it to
fail
-new CommitLogSegmentManager(CommitLog.instance);
-
-//busy wait since commitlogsegmentmanager spawns another thread
-int retries = 0;
-while (!killerForTests.wasKilled() && retries++ < 5)
-Thread.sleep(10);
-
-//since failure was before CassandraDaemon startup, the JVM must
be killed
+CommitLog.handleCommitError("Testing ignore policy", new
Throwable());
+//even thou
[5/6] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2923011b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2923011b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2923011b Branch: refs/heads/cassandra-3.0 Commit: 2923011b7aadc2368b87065cee12dd428d8c875d Parents: 6c69f07 d61c026 Author: Benedict Elliott Smith Authored: Tue Aug 18 10:10:58 2015 +0100 Committer: Benedict Elliott Smith Committed: Tue Aug 18 10:10:58 2015 +0100 -- --
[2/6] cassandra git commit: Fix CommitLogFailurePolicyTest
Fix CommitLogFailurePolicyTest
patch by paulo motta for CASSANDRA-10094
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d61c026f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d61c026f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d61c026f
Branch: refs/heads/cassandra-3.0
Commit: d61c026fd723390e9d916e7a18ff9bdad019ee2c
Parents: 09147a8
Author: Benedict Elliott Smith
Authored: Mon Aug 17 09:52:13 2015 +0100
Committer: Benedict Elliott Smith
Committed: Tue Aug 18 10:10:22 2015 +0100
--
.../db/CommitLogFailurePolicyTest.java | 64 ++--
1 file changed, 5 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d61c026f/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
--
diff --git a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
index cca6503..0ecab3c 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db;
-import java.io.File;
-
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +26,6 @@ import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.service.CassandraDaemon;
@@ -38,7 +35,6 @@ import org.apache.cassandra.utils.KillerForTests;
public class CommitLogFailurePolicyTest
{
-
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
@@ -95,57 +91,20 @@ public class CommitLogFailurePolicyTest
}
@Test
-public void testCommitFailurePolicy_mustDieIfNotStartedUp()
-{
-//startup was not completed successfuly (since method completeSetup()
was not called)
-CassandraDaemon daemon = new CassandraDaemon();
-StorageService.instance.registerDaemon(daemon);
-
-KillerForTests killerForTests = new KillerForTests();
-JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
-Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
-try
-{
-//even though policy is ignore, JVM must die because Daemon has
not finished initializing
-
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-CommitLog.handleCommitError("Testing die policy", new Throwable());
-Assert.assertTrue(killerForTests.wasKilled());
-Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed
quietly due to startup failure
-}
-finally
-{
-DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
-JVMStabilityInspector.replaceKiller(originalKiller);
-}
-}
-
-@Test
-public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws
Exception
+public void testCommitFailurePolicy_ignore_beforeStartup()
{
//startup was not completed successfuly (since method completeSetup()
was not called)
CassandraDaemon daemon = new CassandraDaemon();
StorageService.instance.registerDaemon(daemon);
-//let's make the commit log directory non-writable
-File commitLogDir = new
File(DatabaseDescriptor.getCommitLogLocation());
-commitLogDir.setWritable(false);
-
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
try
{
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-
-//now let's create a commit log segment manager and wait for it to
fail
-new CommitLogSegmentManager(CommitLog.instance);
-
-//busy wait since commitlogsegmentmanager spawns another thread
-int retries = 0;
-while (!killerForTests.wasKilled() && retries++ < 5)
-Thread.sleep(10);
-
-//since failure was before CassandraDaemon startup, the JVM must
be killed
+CommitLog.handleCommitError("Testing ignore policy", new
Throwable());
+//e
[6/6] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/234a15aa Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/234a15aa Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/234a15aa Branch: refs/heads/trunk Commit: 234a15aae256a186bfc0f67c4c3e70a87a80dfbc Parents: 91e3d34 2923011 Author: Benedict Elliott Smith Authored: Tue Aug 18 10:11:07 2015 +0100 Committer: Benedict Elliott Smith Committed: Tue Aug 18 10:11:07 2015 +0100 -- --
[4/6] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2923011b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2923011b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2923011b Branch: refs/heads/trunk Commit: 2923011b7aadc2368b87065cee12dd428d8c875d Parents: 6c69f07 d61c026 Author: Benedict Elliott Smith Authored: Tue Aug 18 10:10:58 2015 +0100 Committer: Benedict Elliott Smith Committed: Tue Aug 18 10:10:58 2015 +0100 -- --
[1/6] cassandra git commit: Fix CommitLogFailurePolicyTest
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 09147a8b6 -> d61c026fd
refs/heads/cassandra-3.0 6c69f07c2 -> 2923011b7
refs/heads/trunk 91e3d3421 -> 234a15aae
Fix CommitLogFailurePolicyTest
patch by paulo motta for CASSANDRA-10094
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d61c026f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d61c026f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d61c026f
Branch: refs/heads/cassandra-2.2
Commit: d61c026fd723390e9d916e7a18ff9bdad019ee2c
Parents: 09147a8
Author: Benedict Elliott Smith
Authored: Mon Aug 17 09:52:13 2015 +0100
Committer: Benedict Elliott Smith
Committed: Tue Aug 18 10:10:22 2015 +0100
--
.../db/CommitLogFailurePolicyTest.java | 64 ++--
1 file changed, 5 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d61c026f/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
--
diff --git a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
index cca6503..0ecab3c 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogFailurePolicyTest.java
@@ -19,8 +19,6 @@
package org.apache.cassandra.db;
-import java.io.File;
-
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +26,6 @@ import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.service.CassandraDaemon;
@@ -38,7 +35,6 @@ import org.apache.cassandra.utils.KillerForTests;
public class CommitLogFailurePolicyTest
{
-
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
@@ -95,57 +91,20 @@ public class CommitLogFailurePolicyTest
}
@Test
-public void testCommitFailurePolicy_mustDieIfNotStartedUp()
-{
-//startup was not completed successfuly (since method completeSetup()
was not called)
-CassandraDaemon daemon = new CassandraDaemon();
-StorageService.instance.registerDaemon(daemon);
-
-KillerForTests killerForTests = new KillerForTests();
-JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
-Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
-try
-{
-//even though policy is ignore, JVM must die because Daemon has
not finished initializing
-
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-CommitLog.handleCommitError("Testing die policy", new Throwable());
-Assert.assertTrue(killerForTests.wasKilled());
-Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed
quietly due to startup failure
-}
-finally
-{
-DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
-JVMStabilityInspector.replaceKiller(originalKiller);
-}
-}
-
-@Test
-public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws
Exception
+public void testCommitFailurePolicy_ignore_beforeStartup()
{
//startup was not completed successfuly (since method completeSetup()
was not called)
CassandraDaemon daemon = new CassandraDaemon();
StorageService.instance.registerDaemon(daemon);
-//let's make the commit log directory non-writable
-File commitLogDir = new
File(DatabaseDescriptor.getCommitLogLocation());
-commitLogDir.setWritable(false);
-
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller =
JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy =
DatabaseDescriptor.getCommitFailurePolicy();
try
{
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
-
-//now let's create a commit log segment manager and wait for it to
fail
-new CommitLogSegmentManager(CommitLog.instance);
-
-//busy wait since commitlogsegmentmanager spawns another thread
-int retries = 0;
-while (!killerForTests.wasKilled() && retries++ < 5)
-Thread.sleep(10);
-
-
[jira] [Commented] (CASSANDRA-10112) Apply disk_failure_policy to transaction logs
[
https://issues.apache.org/jira/browse/CASSANDRA-10112?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14700986#comment-14700986
]
Benedict commented on CASSANDRA-10112:
--
I think in the event of failure we should just print out all of the information
we have, and let the operator decide what to do. Listing the transaction logs
that are corrupted, along with a brief summary of their contents. It would be
nice to print one row for each row in the problematic transaction log,
informing the user of the file that's referenced, if it exists, and if the row
was corroborated by checksum. We could also output a brief explanation of the
transaction logs themselves, or reference the NEWS.txt. I think that should
give them all the information they need to make a rational decision.
> Apply disk_failure_policy to transaction logs
> -
>
> Key: CASSANDRA-10112
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10112
> Project: Cassandra
> Issue Type: Improvement
>Reporter: Stefania
>Assignee: Stefania
>
> Transaction logs were introduced by CASSANDRA-7066 and are read during
> start-up. In case of file system errors, such as disk corruption, we
> currently log a panic error and leave the sstable files and transaction logs
> as they are; this is to avoid rolling back a transaction (i.e. deleting
> files) by mistake.
> We should instead look at the {{disk_failure_policy}} and refuse to start
> unless the failure policy is {{ignore}}.
> We should also consider stashing files that cannot be read during startup,
> either transaction logs or sstables, by moving them to a dedicated
> sub-folder.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9854) Make CompactionController.maybeRefreshOverlaps public
[ https://issues.apache.org/jira/browse/CASSANDRA-9854?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701003#comment-14701003 ] Mike Adamson commented on CASSANDRA-9854: - [~jbellis] This is a separate requirement. It is specifically for the requirement in the description, so that 3rd party strategies can exist in separate packages. It is the only method used by compaction tasks that is package protected. > Make CompactionController.maybeRefreshOverlaps public > - > > Key: CASSANDRA-9854 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9854 > Project: Cassandra > Issue Type: Improvement > Components: Core >Reporter: Mike Adamson >Assignee: Mike Adamson > Fix For: 3.0 beta 2 > > Attachments: 9854.txt > > > This method is currently package protected and means that 3rd party > compaction strategies outside of the compaction package cannot use it. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9749) CommitLogReplayer continues startup after encountering errors
[
https://issues.apache.org/jira/browse/CASSANDRA-9749?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701021#comment-14701021
]
Branimir Lambov commented on CASSANDRA-9749:
Let me try to rephrase and simplify Jonathan's question:
The original idea of this ticket was to let the commit log failure policy
dictate what is done in the case of replay failures. Now that CASSANDRA-8515 is
in place, this means that regardless of the failure policy the node will always
refuse to start in the case of replay failures (unless they are in the last
section of the last segment). This can be overridded using the
{{cassandra.commitlog.ignorereplayerrors}} flag.
The question now is whether this is a good or sufficient solution, or should I
invest time to override the CASSANDRA-8515 override so that the original
failure policy can apply?
My personal opinion is that it _is_ a proper solution. Any scenario I could
imagine benefiting from e.g. the 'stop' policy at log replay would also be
better served by the same policy at log startup.
> CommitLogReplayer continues startup after encountering errors
> -
>
> Key: CASSANDRA-9749
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9749
> Project: Cassandra
> Issue Type: Bug
>Reporter: Blake Eggleston
>Assignee: Branimir Lambov
> Fix For: 2.2.x
>
> Attachments: 9749-coverage.tgz
>
>
> There are a few places where the commit log recovery method either skips
> sections or just returns when it encounters errors.
> Specifically if it can't read the header here:
> https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java#L298
> Or if there are compressor problems here:
> https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java#L314
> and here:
> https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java#L366
> Whether these are user-fixable or not, I think we should require more direct
> user intervention (ie: fix what's wrong, or remove the bad file and restart)
> since we're basically losing data.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9917) MVs should validate gc grace seconds on the tables involved
[
https://issues.apache.org/jira/browse/CASSANDRA-9917?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701115#comment-14701115
]
Paulo Motta commented on CASSANDRA-9917:
There are 3 possible failure scenarios we want to protect against, by limiting
the {{gc_grace_seconds}} of tables involved in a batchlog/MV:
1. batchlog reaching the destination node alive (< 1s)
2. protect against minor spikes and gc pauses (few minutes)
3. medium to major outages (> 1h)
By using a large treshold we can protect against 1-3, but we can't define the
optimal value (since this varies from cluster to cluster) nor use
{{max_hints_windows_in_ms}} (due to the valid reasons given by Alexey). By
validating that the value is not {{0}}, we only protect against 1, and users
might workaround this limitation by setting the gc_grace_seconds to 1, which
can still lead to problems.
I think 2 is pretty common and can save us some headache in the future, so how
about using a min treshold of 10 minutes to protect against 1 and 2? We can
either hard code this, or leave the unpublished property
{{cassandra.min_batchlog_ttl}} if advanced operators want to modify that
constant. We can still properly document the effects of gc gs in the MV
documentation to protect against 3.
> MVs should validate gc grace seconds on the tables involved
> ---
>
> Key: CASSANDRA-9917
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9917
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Paulo Motta
> Labels: materializedviews
> Fix For: 3.0 beta 2
>
>
> For correctness reasons (potential resurrection of dropped values), batchlog
> entries are TTLs with the lowest gc grace second of all the tables involved
> in a batch.
> It means that if gc gs is set to 0 in one of the tables, the batchlog entry
> will be dead on arrival, and never replayed.
> We should probably warn against such LOGGED writes taking place, in general,
> but for MVs, we must validate that gc gs on the base table (and on the MV
> table, if we should allow altering gc gs there at all), is never set too low,
> or else.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10105) Windows dtest 3.0: TestOfflineTools failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10105?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701138#comment-14701138
]
Paulo Motta commented on CASSANDRA-10105:
-
I got this problem in some other test locally too. I guess this should be a
WARN and not an ERROR, since SecureDirectoryStream is not supported by Windows.
> Windows dtest 3.0: TestOfflineTools failures
>
>
> Key: CASSANDRA-10105
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10105
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Joshua McKenzie
> Labels: Windows
> Fix For: 3.0.x
>
>
> offline_tools_test.py:TestOfflineTools.sstablelevelreset_test
> offline_tools_test.py:TestOfflineTools.sstableofflinerelevel_test
> Both tests fail with the following:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 15:55:05,060 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> Failure history:
> [consistent|http://cassci.datastax.com/view/cassandra-3.0/job/cassandra-3.0_dtest_win32/17/testReport/junit/jmx_test/TestJMX/netstats_test/history/]
> Env: ci and local
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Assigned] (CASSANDRA-10097) Windows dtest 3.0: bootstrap_test.py:TestBootstrap.bootstrap_with_reset_bootstrap_state_test fails
[
https://issues.apache.org/jira/browse/CASSANDRA-10097?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Paulo Motta reassigned CASSANDRA-10097:
---
Assignee: Paulo Motta
> Windows dtest 3.0:
> bootstrap_test.py:TestBootstrap.bootstrap_with_reset_bootstrap_state_test
> fails
> --
>
> Key: CASSANDRA-10097
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10097
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 3.0.x
>
>
> {noformat}
> File "D:\Python27\lib\unittest\case.py", line 329, in run
> testMethod()
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\tools.py",
> line 243, in wrapped
> f(obj)
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\bootstrap_test.py",
> line 184, in bootstrap_with_reset_bootstrap_state_test
> node3.watch_log_for("Resetting bootstrap progress to start fresh",
> from_mark=mark)
> File "build\bdist.win-amd64\egg\ccmlib\node.py", line 382, in watch_log_for
> raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + "
> [" + self.name + "] Missing: " + str([e.pattern for e in tofind]) + ":\n" +
> reads)
> {noformat}
> Failure history:
> [consistent|http://cassci.datastax.com/view/cassandra-3.0/job/cassandra-3.0_dtest_win32/17/testReport/junit/bootstrap_test/TestBootstrap/bootstrap_with_reset_bootstrap_state_test/history/]
> Env: both ci and locally
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9917) MVs should validate gc grace seconds on the tables involved
[ https://issues.apache.org/jira/browse/CASSANDRA-9917?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701177#comment-14701177 ] Aleksey Yeschenko commented on CASSANDRA-9917: -- There are no shades of 'correct'. Let's not overthink it. By far the most common GC GS is the default one, which is 10 days. The second most common one is 0. There aren't many out there that are in-between. Please, let's just validate that it's not 0, since with the current implementation of BL and hints it just doesn't work (but something we should revisit once we introduce the write-once option for the tables). > MVs should validate gc grace seconds on the tables involved > --- > > Key: CASSANDRA-9917 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9917 > Project: Cassandra > Issue Type: Bug >Reporter: Aleksey Yeschenko >Assignee: Paulo Motta > Labels: materializedviews > Fix For: 3.0 beta 2 > > > For correctness reasons (potential resurrection of dropped values), batchlog > entries are TTLs with the lowest gc grace second of all the tables involved > in a batch. > It means that if gc gs is set to 0 in one of the tables, the batchlog entry > will be dead on arrival, and never replayed. > We should probably warn against such LOGGED writes taking place, in general, > but for MVs, we must validate that gc gs on the base table (and on the MV > table, if we should allow altering gc gs there at all), is never set too low, > or else. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
cassandra git commit: Fix AggregationTest post-test error messages
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 d61c026fd -> 6d9fa3755
Fix AggregationTest post-test error messages
patch by Robert Stupp; reviewed by Benjamin Lerer for CASSANDRA-10056
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6d9fa375
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6d9fa375
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6d9fa375
Branch: refs/heads/cassandra-2.2
Commit: 6d9fa3755daf6b21b1a7b2043730bcd9fb2643c4
Parents: d61c026
Author: Robert Stupp
Authored: Tue Aug 18 14:39:29 2015 +0200
Committer: Robert Stupp
Committed: Tue Aug 18 14:39:29 2015 +0200
--
.../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d9fa375/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
--
diff --git
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 6a77368..9881d73 100644
---
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -1465,7 +1465,7 @@ public class AggregationTest extends CQLTester
execute("INSERT INTO %s (a, b) VALUES (?, {f : ?})", 3, 7);
String fState = createFunction(KEYSPACE,
- "tuple",
+ myType + ',' + myType,
"CREATE FUNCTION %s (state " + myType +
", values " + myType + ") " +
"CALLED ON NULL INPUT " +
"RETURNS " + myType + " " +
@@ -1535,7 +1535,7 @@ public class AggregationTest extends CQLTester
"INITCOND ''");
String fRNON = createFunction(KEYSPACE,
- "text",
+ "text, text",
"CREATE FUNCTION %s(a text, b text) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS text " +
@@ -1551,7 +1551,7 @@ public class AggregationTest extends CQLTester
"AS 'return \"fin\" + a;'");
String aRNON = createAggregate(KEYSPACE,
- "int",
+ "text",
"CREATE AGGREGATE %s(text) " +
"SFUNC " + shortFunctionName(fRNON) + '
' +
"STYPE text " +
[1/2] cassandra git commit: Fix AggregationTest post-test error messages
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 2923011b7 -> 310378d56
Fix AggregationTest post-test error messages
patch by Robert Stupp; reviewed by Benjamin Lerer for CASSANDRA-10056
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6d9fa375
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6d9fa375
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6d9fa375
Branch: refs/heads/cassandra-3.0
Commit: 6d9fa3755daf6b21b1a7b2043730bcd9fb2643c4
Parents: d61c026
Author: Robert Stupp
Authored: Tue Aug 18 14:39:29 2015 +0200
Committer: Robert Stupp
Committed: Tue Aug 18 14:39:29 2015 +0200
--
.../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d9fa375/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
--
diff --git
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 6a77368..9881d73 100644
---
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -1465,7 +1465,7 @@ public class AggregationTest extends CQLTester
execute("INSERT INTO %s (a, b) VALUES (?, {f : ?})", 3, 7);
String fState = createFunction(KEYSPACE,
- "tuple",
+ myType + ',' + myType,
"CREATE FUNCTION %s (state " + myType +
", values " + myType + ") " +
"CALLED ON NULL INPUT " +
"RETURNS " + myType + " " +
@@ -1535,7 +1535,7 @@ public class AggregationTest extends CQLTester
"INITCOND ''");
String fRNON = createFunction(KEYSPACE,
- "text",
+ "text, text",
"CREATE FUNCTION %s(a text, b text) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS text " +
@@ -1551,7 +1551,7 @@ public class AggregationTest extends CQLTester
"AS 'return \"fin\" + a;'");
String aRNON = createAggregate(KEYSPACE,
- "int",
+ "text",
"CREATE AGGREGATE %s(text) " +
"SFUNC " + shortFunctionName(fRNON) + '
' +
"STYPE text " +
[3/3] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e214567a Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e214567a Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e214567a Branch: refs/heads/trunk Commit: e214567a8f400ea3a3479e318d48c312e9d04e8e Parents: 234a15a 310378d Author: Robert Stupp Authored: Tue Aug 18 14:39:49 2015 +0200 Committer: Robert Stupp Committed: Tue Aug 18 14:39:49 2015 +0200 -- .../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[2/3] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/310378d5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/310378d5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/310378d5 Branch: refs/heads/trunk Commit: 310378d56785fda211166b9d7a78daea0586ca9b Parents: 2923011 6d9fa37 Author: Robert Stupp Authored: Tue Aug 18 14:39:41 2015 +0200 Committer: Robert Stupp Committed: Tue Aug 18 14:39:41 2015 +0200 -- .../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/310378d5/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java --
[2/2] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/310378d5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/310378d5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/310378d5 Branch: refs/heads/cassandra-3.0 Commit: 310378d56785fda211166b9d7a78daea0586ca9b Parents: 2923011 6d9fa37 Author: Robert Stupp Authored: Tue Aug 18 14:39:41 2015 +0200 Committer: Robert Stupp Committed: Tue Aug 18 14:39:41 2015 +0200 -- .../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/310378d5/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java --
[1/3] cassandra git commit: Fix AggregationTest post-test error messages
Repository: cassandra
Updated Branches:
refs/heads/trunk 234a15aae -> e214567a8
Fix AggregationTest post-test error messages
patch by Robert Stupp; reviewed by Benjamin Lerer for CASSANDRA-10056
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6d9fa375
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6d9fa375
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6d9fa375
Branch: refs/heads/trunk
Commit: 6d9fa3755daf6b21b1a7b2043730bcd9fb2643c4
Parents: d61c026
Author: Robert Stupp
Authored: Tue Aug 18 14:39:29 2015 +0200
Committer: Robert Stupp
Committed: Tue Aug 18 14:39:29 2015 +0200
--
.../cassandra/cql3/validation/operations/AggregationTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d9fa375/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
--
diff --git
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
index 6a77368..9881d73 100644
---
a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
+++
b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java
@@ -1465,7 +1465,7 @@ public class AggregationTest extends CQLTester
execute("INSERT INTO %s (a, b) VALUES (?, {f : ?})", 3, 7);
String fState = createFunction(KEYSPACE,
- "tuple",
+ myType + ',' + myType,
"CREATE FUNCTION %s (state " + myType +
", values " + myType + ") " +
"CALLED ON NULL INPUT " +
"RETURNS " + myType + " " +
@@ -1535,7 +1535,7 @@ public class AggregationTest extends CQLTester
"INITCOND ''");
String fRNON = createFunction(KEYSPACE,
- "text",
+ "text, text",
"CREATE FUNCTION %s(a text, b text) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS text " +
@@ -1551,7 +1551,7 @@ public class AggregationTest extends CQLTester
"AS 'return \"fin\" + a;'");
String aRNON = createAggregate(KEYSPACE,
- "int",
+ "text",
"CREATE AGGREGATE %s(text) " +
"SFUNC " + shortFunctionName(fRNON) + '
' +
"STYPE text " +
[jira] [Updated] (CASSANDRA-9854) Make CompactionController.maybeRefreshOverlaps public
[ https://issues.apache.org/jira/browse/CASSANDRA-9854?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Aleksey Yeschenko updated CASSANDRA-9854: - Reviewer: Aleksey Yeschenko > Make CompactionController.maybeRefreshOverlaps public > - > > Key: CASSANDRA-9854 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9854 > Project: Cassandra > Issue Type: Improvement > Components: Core >Reporter: Mike Adamson >Assignee: Mike Adamson > Fix For: 3.0 beta 2 > > Attachments: 9854.txt > > > This method is currently package protected and means that 3rd party > compaction strategies outside of the compaction package cannot use it. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-7066) Simplify (and unify) cleanup of compaction leftovers
[
https://issues.apache.org/jira/browse/CASSANDRA-7066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701211#comment-14701211
]
Aleksey Yeschenko commented on CASSANDRA-7066:
--
On a purely organisational note - now that we have the {{Ready to Commit}}
status, we should be using it instead of x-to-commit labels (the filter would
look like reviewer=me and status=ready to commit).
> Simplify (and unify) cleanup of compaction leftovers
>
>
> Key: CASSANDRA-7066
> URL: https://issues.apache.org/jira/browse/CASSANDRA-7066
> Project: Cassandra
> Issue Type: Improvement
> Components: Core
>Reporter: Benedict
>Assignee: Stefania
>Priority: Minor
> Labels: benedict-to-commit, compaction
> Fix For: 3.0 alpha 1
>
> Attachments: 7066.txt
>
>
> Currently we manage a list of in-progress compactions in a system table,
> which we use to cleanup incomplete compactions when we're done. The problem
> with this is that 1) it's a bit clunky (and leaves us in positions where we
> can unnecessarily cleanup completed files, or conversely not cleanup files
> that have been superceded); and 2) it's only used for a regular compaction -
> no other compaction types are guarded in the same way, so can result in
> duplication if we fail before deleting the replacements.
> I'd like to see each sstable store in its metadata its direct ancestors, and
> on startup we simply delete any sstables that occur in the union of all
> ancestor sets. This way as soon as we finish writing we're capable of
> cleaning up any leftovers, so we never get duplication. It's also much easier
> to reason about.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-9970) CQL 3.3 ordering in where clause produces different results when using "IN...ORDER BY"
[
https://issues.apache.org/jira/browse/CASSANDRA-9970?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Aleksey Yeschenko updated CASSANDRA-9970:
-
Fix Version/s: (was: 2.2.x)
3.0 beta 1
2.2.1
> CQL 3.3 ordering in where clause produces different results when using
> "IN...ORDER BY"
> --
>
> Key: CASSANDRA-9970
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9970
> Project: Cassandra
> Issue Type: Bug
> Components: API
>Reporter: Marc Zbyszynski
>Assignee: Benjamin Lerer
>Priority: Minor
> Fix For: 2.2.1, 3.0 beta 1
>
> Attachments: 9970.txt, dtest-CASSANDRA-9970.txt
>
>
> This is an issue I found with
> {code}
> [cqlsh 5.0.1 | Cassandra 2.2.0 | CQL spec 3.3.0 | Native protocol v4]
> {code}
> Given this table and data:
> {code:sql}
> cqlsh> CREATE TABLE myschema.in_test (
>... col_1 int,
>... col_2 int,
>... col_3 int,
>... PRIMARY KEY ((col_1, col_2), col_3)
>... );
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 1);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 2);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 13);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 2, 10);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 2, 11);
> {code}
> This query behaves as expected (as described in
> [http://docs.datastax.com/en/cql/3.3/cql/cql_using/useQueryIN.html])
> {code:sql}
> cqlsh> select * from myschema.in_test where col_1=1 and col_2 IN (1,2) order
> by col_3 desc;
> InvalidRequest: code=2200 [Invalid query] message="Cannot page queries with
> both ORDER BY and a IN restriction on the partition key; you must either
> remove the ORDER BY or the IN and sort client side, or disable paging for
> this query"
> {code}
> But if you swap the order of the statements in the where clause, the query
> executes without error and doesn't totally honor the _order by_ clause:
> {code:sql}
> cqlsh> select * from myschema.in_test where col_2 IN (1,2) and col_1=1 order
> by col_3 desc;
> col_1 | col_2 | col_3
> ---+---+---
> 1 | 2 |11
> 1 | 2 |10
> 1 | 1 |13
> 1 | 1 | 2
> 1 | 1 | 1
> (5 rows)
> {code}
> Apologies in advance if this is working as intended, or I'm duplicating an
> existing issue. Also please let me know if you need any additional info.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-7066) Simplify (and unify) cleanup of compaction leftovers
[ https://issues.apache.org/jira/browse/CASSANDRA-7066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701222#comment-14701222 ] Benedict commented on CASSANDRA-7066: - That tag was introduced for situations where I was the committer, but not the reviewer. We really need to introduce a "committer" property, so that "Ready To Commit" has meaning for these situations. It's pretty common now for a reviewer not to have committer status. > Simplify (and unify) cleanup of compaction leftovers > > > Key: CASSANDRA-7066 > URL: https://issues.apache.org/jira/browse/CASSANDRA-7066 > Project: Cassandra > Issue Type: Improvement > Components: Core >Reporter: Benedict >Assignee: Stefania >Priority: Minor > Labels: benedict-to-commit, compaction > Fix For: 3.0 alpha 1 > > Attachments: 7066.txt > > > Currently we manage a list of in-progress compactions in a system table, > which we use to cleanup incomplete compactions when we're done. The problem > with this is that 1) it's a bit clunky (and leaves us in positions where we > can unnecessarily cleanup completed files, or conversely not cleanup files > that have been superceded); and 2) it's only used for a regular compaction - > no other compaction types are guarded in the same way, so can result in > duplication if we fail before deleting the replacements. > I'd like to see each sstable store in its metadata its direct ancestors, and > on startup we simply delete any sstables that occur in the union of all > ancestor sets. This way as soon as we finish writing we're capable of > cleaning up any leftovers, so we never get duplication. It's also much easier > to reason about. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Comment Edited] (CASSANDRA-9970) CQL 3.3 ordering in where clause produces different results when using "IN...ORDER BY"
[
https://issues.apache.org/jira/browse/CASSANDRA-9970?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14698905#comment-14698905
]
Stefania edited comment on CASSANDRA-9970 at 8/18/15 1:16 PM:
--
+1.
I found a dtest on 2.2 failing only on this branch but it is clearly unrelated
(Gossip) and therefore I opened CASSANDRA-10089 for it.
was (Author: stefania):
+1.
I found a dtest on 2.2 happening only on this branch but it is clearly
unrelated (Gossip) and therefore I opened CASSANDRA-10089 for it.
> CQL 3.3 ordering in where clause produces different results when using
> "IN...ORDER BY"
> --
>
> Key: CASSANDRA-9970
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9970
> Project: Cassandra
> Issue Type: Bug
> Components: API
>Reporter: Marc Zbyszynski
>Assignee: Benjamin Lerer
>Priority: Minor
> Fix For: 2.2.1, 3.0 beta 1
>
> Attachments: 9970.txt, dtest-CASSANDRA-9970.txt
>
>
> This is an issue I found with
> {code}
> [cqlsh 5.0.1 | Cassandra 2.2.0 | CQL spec 3.3.0 | Native protocol v4]
> {code}
> Given this table and data:
> {code:sql}
> cqlsh> CREATE TABLE myschema.in_test (
>... col_1 int,
>... col_2 int,
>... col_3 int,
>... PRIMARY KEY ((col_1, col_2), col_3)
>... );
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 1);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 2);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 1, 13);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 2, 10);
> cqlsh> INSERT INTO myschema.in_test (col_1, col_2, col_3) VALUES(1, 2, 11);
> {code}
> This query behaves as expected (as described in
> [http://docs.datastax.com/en/cql/3.3/cql/cql_using/useQueryIN.html])
> {code:sql}
> cqlsh> select * from myschema.in_test where col_1=1 and col_2 IN (1,2) order
> by col_3 desc;
> InvalidRequest: code=2200 [Invalid query] message="Cannot page queries with
> both ORDER BY and a IN restriction on the partition key; you must either
> remove the ORDER BY or the IN and sort client side, or disable paging for
> this query"
> {code}
> But if you swap the order of the statements in the where clause, the query
> executes without error and doesn't totally honor the _order by_ clause:
> {code:sql}
> cqlsh> select * from myschema.in_test where col_2 IN (1,2) and col_1=1 order
> by col_3 desc;
> col_1 | col_2 | col_3
> ---+---+---
> 1 | 2 |11
> 1 | 2 |10
> 1 | 1 |13
> 1 | 1 | 2
> 1 | 1 | 1
> (5 rows)
> {code}
> Apologies in advance if this is working as intended, or I'm duplicating an
> existing issue. Also please let me know if you need any additional info.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-10046) RangeTombstone validation missing on thrift
[
https://issues.apache.org/jira/browse/CASSANDRA-10046?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sylvain Lebresne updated CASSANDRA-10046:
-
Reviewer: Aleksey Yeschenko (was: Tyler Hobbs)
> RangeTombstone validation missing on thrift
> ---
>
> Key: CASSANDRA-10046
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10046
> Project: Cassandra
> Issue Type: Bug
>Reporter: T Jake Luciani
>Assignee: Sylvain Lebresne
>Priority: Minor
> Fix For: 3.0.0 rc1
>
>
> Currently a number of thrift tests fail due to a NPE
> {code}
> test_range_deletion
> test_batch_mutate_remove_slice_standard
> test_batch_mutate_remove_slice_of_entire_supercolumns
> {code}
> These errors all stem from the fact we aren't properly throwing a validation
> exception when a deletion with a slice that is not start == finish.
> We should add better validation to throw a clear exception sooner.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[3/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
Move 2i metadata out of system_schema.columns and ColumnDefinition Patch by Sam Tunnicliffe; reviewed by Aleksey Yeschenko and Robert Stupp for CASSANDRA-6717 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/06c130e3 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/06c130e3 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/06c130e3 Branch: refs/heads/cassandra-3.0 Commit: 06c130e3cb85577041b475084400c08c505d8f9e Parents: 310378d Author: Sam Tunnicliffe Authored: Mon Aug 10 16:09:15 2015 +0100 Committer: Sam Tunnicliffe Committed: Tue Aug 18 14:59:44 2015 +0100 -- build.xml | 4 +- ...ra-driver-core-2.2.0-rc2-SNAPSHOT-shaded.jar | Bin 2163939 -> 0 bytes ...ore-3.0.0-alpha2-188d996-SNAPSHOT-shaded.jar | Bin 0 -> 2204619 bytes ...ver-internal-only-2.6.0rc2.post0-1a480f1.zip | Bin 200621 -> 0 bytes ...iver-internal-only-3.0.0a1.post0-90a797e.zip | Bin 0 -> 201086 bytes .../org/apache/cassandra/config/CFMetaData.java | 159 -- .../cassandra/config/ColumnDefinition.java | 116 +-- .../org/apache/cassandra/config/IndexType.java | 25 --- .../cql3/functions/JavaBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/JavaUDF.java | 16 +- .../cql3/functions/ScriptBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/UDFunction.java| 12 +- .../cassandra/cql3/functions/UDHelper.java | 34 ++- .../cql3/statements/AlterTableStatement.java| 19 +- .../cql3/statements/CreateIndexStatement.java | 79 --- .../cql3/statements/DropIndexStatement.java | 79 +++ .../cassandra/cql3/statements/IndexTarget.java | 23 +- .../apache/cassandra/db/ColumnFamilyStore.java | 19 +- .../AbstractSimplePerColumnSecondaryIndex.java | 2 +- .../cassandra/db/index/SecondaryIndex.java | 92 .../db/index/SecondaryIndexManager.java | 27 +-- .../db/index/composites/CompositesIndex.java| 19 +- .../cassandra/db/index/keys/KeysIndex.java | 5 +- .../cassandra/hadoop/cql3/CqlRecordReader.java | 38 .../io/sstable/format/SSTableReader.java| 6 +- .../apache/cassandra/schema/IndexMetadata.java | 167 +++ .../org/apache/cassandra/schema/Indexes.java| 208 +++ .../cassandra/schema/KeyspaceMetadata.java | 23 ++ .../cassandra/schema/LegacySchemaMigrator.java | 106 -- .../apache/cassandra/schema/SchemaKeyspace.java | 196 ++--- .../cassandra/thrift/CassandraServer.java | 3 - .../cassandra/thrift/ThriftConversion.java | 74 ++- .../utils/NativeSSTableLoaderClient.java| 2 +- .../unit/org/apache/cassandra/SchemaLoader.java | 42 ++-- .../cassandra/config/ColumnDefinitionTest.java | 6 +- .../org/apache/cassandra/cql3/CQLTester.java| 21 +- .../cassandra/cql3/IndexQueryPagingTest.java| 2 +- .../validation/entities/UFPureScriptTest.java | 13 +- .../cql3/validation/entities/UFTest.java| 14 +- .../cassandra/db/BatchlogManagerTest.java | 4 +- .../apache/cassandra/db/DirectoriesTest.java| 12 +- .../apache/cassandra/db/RangeTombstoneTest.java | 106 ++ .../apache/cassandra/db/SecondaryIndexTest.java | 21 +- .../db/index/PerRowSecondaryIndexTest.java | 3 +- .../org/apache/cassandra/schema/DefsTest.java | 39 ++-- .../schema/LegacySchemaMigratorTest.java| 36 +++- .../cassandra/stress/util/JavaDriverClient.java | 2 +- 47 files changed, 1261 insertions(+), 617 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/build.xml -- diff --git a/build.xml b/build.xml index 06638f6..8df738c 100644 --- a/build.xml +++ b/build.xml @@ -134,13 +134,11 @@
[6/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
Move 2i metadata out of system_schema.columns and ColumnDefinition Patch by Sam Tunnicliffe; reviewed by Aleksey Yeschenko and Robert Stupp for CASSANDRA-6717 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/06c130e3 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/06c130e3 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/06c130e3 Branch: refs/heads/trunk Commit: 06c130e3cb85577041b475084400c08c505d8f9e Parents: 310378d Author: Sam Tunnicliffe Authored: Mon Aug 10 16:09:15 2015 +0100 Committer: Sam Tunnicliffe Committed: Tue Aug 18 14:59:44 2015 +0100 -- build.xml | 4 +- ...ra-driver-core-2.2.0-rc2-SNAPSHOT-shaded.jar | Bin 2163939 -> 0 bytes ...ore-3.0.0-alpha2-188d996-SNAPSHOT-shaded.jar | Bin 0 -> 2204619 bytes ...ver-internal-only-2.6.0rc2.post0-1a480f1.zip | Bin 200621 -> 0 bytes ...iver-internal-only-3.0.0a1.post0-90a797e.zip | Bin 0 -> 201086 bytes .../org/apache/cassandra/config/CFMetaData.java | 159 -- .../cassandra/config/ColumnDefinition.java | 116 +-- .../org/apache/cassandra/config/IndexType.java | 25 --- .../cql3/functions/JavaBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/JavaUDF.java | 16 +- .../cql3/functions/ScriptBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/UDFunction.java| 12 +- .../cassandra/cql3/functions/UDHelper.java | 34 ++- .../cql3/statements/AlterTableStatement.java| 19 +- .../cql3/statements/CreateIndexStatement.java | 79 --- .../cql3/statements/DropIndexStatement.java | 79 +++ .../cassandra/cql3/statements/IndexTarget.java | 23 +- .../apache/cassandra/db/ColumnFamilyStore.java | 19 +- .../AbstractSimplePerColumnSecondaryIndex.java | 2 +- .../cassandra/db/index/SecondaryIndex.java | 92 .../db/index/SecondaryIndexManager.java | 27 +-- .../db/index/composites/CompositesIndex.java| 19 +- .../cassandra/db/index/keys/KeysIndex.java | 5 +- .../cassandra/hadoop/cql3/CqlRecordReader.java | 38 .../io/sstable/format/SSTableReader.java| 6 +- .../apache/cassandra/schema/IndexMetadata.java | 167 +++ .../org/apache/cassandra/schema/Indexes.java| 208 +++ .../cassandra/schema/KeyspaceMetadata.java | 23 ++ .../cassandra/schema/LegacySchemaMigrator.java | 106 -- .../apache/cassandra/schema/SchemaKeyspace.java | 196 ++--- .../cassandra/thrift/CassandraServer.java | 3 - .../cassandra/thrift/ThriftConversion.java | 74 ++- .../utils/NativeSSTableLoaderClient.java| 2 +- .../unit/org/apache/cassandra/SchemaLoader.java | 42 ++-- .../cassandra/config/ColumnDefinitionTest.java | 6 +- .../org/apache/cassandra/cql3/CQLTester.java| 21 +- .../cassandra/cql3/IndexQueryPagingTest.java| 2 +- .../validation/entities/UFPureScriptTest.java | 13 +- .../cql3/validation/entities/UFTest.java| 14 +- .../cassandra/db/BatchlogManagerTest.java | 4 +- .../apache/cassandra/db/DirectoriesTest.java| 12 +- .../apache/cassandra/db/RangeTombstoneTest.java | 106 ++ .../apache/cassandra/db/SecondaryIndexTest.java | 21 +- .../db/index/PerRowSecondaryIndexTest.java | 3 +- .../org/apache/cassandra/schema/DefsTest.java | 39 ++-- .../schema/LegacySchemaMigratorTest.java| 36 +++- .../cassandra/stress/util/JavaDriverClient.java | 2 +- 47 files changed, 1261 insertions(+), 617 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/build.xml -- diff --git a/build.xml b/build.xml index 06638f6..8df738c 100644 --- a/build.xml +++ b/build.xml @@ -134,13 +134,11 @@
[4/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
--
diff --git
a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
index d82ebfd..8be1c96 100644
--- a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
+++ b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
@@ -142,7 +142,7 @@ public class JavaDriverClient
public ResultSet execute(String query,
org.apache.cassandra.db.ConsistencyLevel consistency)
{
-SimpleStatement stmt = new SimpleStatement(query);
+SimpleStatement stmt = getSession().newSimpleStatement(query);
stmt.setConsistencyLevel(from(consistency));
return getSession().execute(stmt);
}
[2/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/src/java/org/apache/cassandra/schema/IndexMetadata.java
--
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java
b/src/java/org/apache/cassandra/schema/IndexMetadata.java
new file mode 100644
index 000..af07cee
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.index.SecondaryIndex;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+/**
+ * An immutable representation of secondary index metadata.
+ */
+public final class IndexMetadata
+{
+public enum IndexType
+{
+KEYS, CUSTOM, COMPOSITES
+}
+
+public enum TargetType
+{
+COLUMN, ROW
+}
+
+public final String name;
+public final IndexType indexType;
+public final TargetType targetType;
+public final Map options;
+public final Set columns;
+
+private IndexMetadata(String name,
+ Map options,
+ IndexType indexType,
+ TargetType targetType,
+ Set columns)
+{
+this.name = name;
+this.options = options == null ? ImmutableMap.of() :
ImmutableMap.copyOf(options);
+this.indexType = indexType;
+this.targetType = targetType;
+this.columns = columns == null ? ImmutableSet.of() :
ImmutableSet.copyOf(columns);
+}
+
+public static IndexMetadata legacyIndex(ColumnIdentifier column,
+ String name,
+ IndexType type,
+ Map options)
+{
+return new IndexMetadata(name, options, type, TargetType.COLUMN,
Collections.singleton(column));
+}
+
+public static IndexMetadata legacyIndex(ColumnDefinition column,
+ String name,
+ IndexType type,
+ Map options)
+{
+return legacyIndex(column.name, name, type, options);
+}
+
+public static boolean isNameValid(String name)
+{
+return name != null && !name.isEmpty() && name.matches("\\w+");
+}
+
+// these will go away as part of #9459 as we enable real per-row indexes
+public static String getDefaultIndexName(String cfName, ColumnIdentifier
columnName)
+{
+return (cfName + "_" + columnName + "_idx").replaceAll("\\W", "");
+}
+
+public void validate()
+{
+if (!isNameValid(name))
+throw new ConfigurationException("Illegal index name " + name);
+
+if (indexType == null)
+throw new ConfigurationException("Index type is null for index " +
name);
+
+if (targetType == null)
+throw new ConfigurationException("Target type is null for index "
+ name);
+
+if (indexType == IndexMetadata.IndexType.CUSTOM)
+if (options == null ||
!options.containsKey(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME))
+throw new ConfigurationException(String.format("Required
option missing for index %s : %s",
+ name,
SecondaryIndex.CUSTOM_INDEX_OPTION_NAME));
+}
+
+public ColumnDefinition indexedColumn(CFMetaData cfm)
+{
+ return cfm.getColumnDefinition(columns.iterator().next());
+}
+
+public boolean isCustom()
+{
+return indexType == IndexType.CUSTOM;
+}
+
+public boolean isKeys()
+{
+retu
[7/7] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e92a4b63 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e92a4b63 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e92a4b63 Branch: refs/heads/trunk Commit: e92a4b6345c4a3a6b5fa2c14ac0746f9cd2ee9ae Parents: e214567 06c130e Author: Sam Tunnicliffe Authored: Tue Aug 18 15:03:59 2015 +0100 Committer: Sam Tunnicliffe Committed: Tue Aug 18 15:03:59 2015 +0100 -- build.xml | 4 +- ...ra-driver-core-2.2.0-rc2-SNAPSHOT-shaded.jar | Bin 2163939 -> 0 bytes ...ore-3.0.0-alpha2-188d996-SNAPSHOT-shaded.jar | Bin 0 -> 2204619 bytes ...ver-internal-only-2.6.0rc2.post0-1a480f1.zip | Bin 200621 -> 0 bytes ...iver-internal-only-3.0.0a1.post0-90a797e.zip | Bin 0 -> 201086 bytes .../org/apache/cassandra/config/CFMetaData.java | 159 -- .../cassandra/config/ColumnDefinition.java | 116 +-- .../org/apache/cassandra/config/IndexType.java | 25 --- .../cql3/functions/JavaBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/JavaUDF.java | 16 +- .../cql3/functions/ScriptBasedUDFunction.java | 2 +- .../cassandra/cql3/functions/UDFunction.java| 12 +- .../cassandra/cql3/functions/UDHelper.java | 34 ++- .../cql3/statements/AlterTableStatement.java| 19 +- .../cql3/statements/CreateIndexStatement.java | 79 --- .../cql3/statements/DropIndexStatement.java | 79 +++ .../cassandra/cql3/statements/IndexTarget.java | 23 +- .../apache/cassandra/db/ColumnFamilyStore.java | 19 +- .../AbstractSimplePerColumnSecondaryIndex.java | 2 +- .../cassandra/db/index/SecondaryIndex.java | 92 .../db/index/SecondaryIndexManager.java | 27 +-- .../db/index/composites/CompositesIndex.java| 19 +- .../cassandra/db/index/keys/KeysIndex.java | 5 +- .../cassandra/hadoop/cql3/CqlRecordReader.java | 38 .../io/sstable/format/SSTableReader.java| 6 +- .../apache/cassandra/schema/IndexMetadata.java | 167 +++ .../org/apache/cassandra/schema/Indexes.java| 208 +++ .../cassandra/schema/KeyspaceMetadata.java | 23 ++ .../cassandra/schema/LegacySchemaMigrator.java | 106 -- .../apache/cassandra/schema/SchemaKeyspace.java | 196 ++--- .../cassandra/thrift/CassandraServer.java | 3 - .../cassandra/thrift/ThriftConversion.java | 74 ++- .../utils/NativeSSTableLoaderClient.java| 2 +- .../unit/org/apache/cassandra/SchemaLoader.java | 42 ++-- .../cassandra/config/ColumnDefinitionTest.java | 6 +- .../org/apache/cassandra/cql3/CQLTester.java| 21 +- .../cassandra/cql3/IndexQueryPagingTest.java| 2 +- .../validation/entities/UFPureScriptTest.java | 13 +- .../cql3/validation/entities/UFTest.java| 14 +- .../cassandra/db/BatchlogManagerTest.java | 4 +- .../apache/cassandra/db/DirectoriesTest.java| 12 +- .../apache/cassandra/db/RangeTombstoneTest.java | 106 ++ .../apache/cassandra/db/SecondaryIndexTest.java | 21 +- .../db/index/PerRowSecondaryIndexTest.java | 3 +- .../org/apache/cassandra/schema/DefsTest.java | 39 ++-- .../schema/LegacySchemaMigratorTest.java| 36 +++- .../cassandra/stress/util/JavaDriverClient.java | 2 +- 47 files changed, 1261 insertions(+), 617 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e92a4b63/src/java/org/apache/cassandra/db/ColumnFamilyStore.java --
[1/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 310378d56 -> 06c130e3c
refs/heads/trunk e214567a8 -> e92a4b634
http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
--
diff --git
a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
index d82ebfd..8be1c96 100644
--- a/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
+++ b/tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java
@@ -142,7 +142,7 @@ public class JavaDriverClient
public ResultSet execute(String query,
org.apache.cassandra.db.ConsistencyLevel consistency)
{
-SimpleStatement stmt = new SimpleStatement(query);
+SimpleStatement stmt = getSession().newSimpleStatement(query);
stmt.setConsistencyLevel(from(consistency));
return getSession().execute(stmt);
}
[5/7] cassandra git commit: Move 2i metadata out of system_schema.columns and ColumnDefinition
http://git-wip-us.apache.org/repos/asf/cassandra/blob/06c130e3/src/java/org/apache/cassandra/schema/IndexMetadata.java
--
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java
b/src/java/org/apache/cassandra/schema/IndexMetadata.java
new file mode 100644
index 000..af07cee
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.index.SecondaryIndex;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+/**
+ * An immutable representation of secondary index metadata.
+ */
+public final class IndexMetadata
+{
+public enum IndexType
+{
+KEYS, CUSTOM, COMPOSITES
+}
+
+public enum TargetType
+{
+COLUMN, ROW
+}
+
+public final String name;
+public final IndexType indexType;
+public final TargetType targetType;
+public final Map options;
+public final Set columns;
+
+private IndexMetadata(String name,
+ Map options,
+ IndexType indexType,
+ TargetType targetType,
+ Set columns)
+{
+this.name = name;
+this.options = options == null ? ImmutableMap.of() :
ImmutableMap.copyOf(options);
+this.indexType = indexType;
+this.targetType = targetType;
+this.columns = columns == null ? ImmutableSet.of() :
ImmutableSet.copyOf(columns);
+}
+
+public static IndexMetadata legacyIndex(ColumnIdentifier column,
+ String name,
+ IndexType type,
+ Map options)
+{
+return new IndexMetadata(name, options, type, TargetType.COLUMN,
Collections.singleton(column));
+}
+
+public static IndexMetadata legacyIndex(ColumnDefinition column,
+ String name,
+ IndexType type,
+ Map options)
+{
+return legacyIndex(column.name, name, type, options);
+}
+
+public static boolean isNameValid(String name)
+{
+return name != null && !name.isEmpty() && name.matches("\\w+");
+}
+
+// these will go away as part of #9459 as we enable real per-row indexes
+public static String getDefaultIndexName(String cfName, ColumnIdentifier
columnName)
+{
+return (cfName + "_" + columnName + "_idx").replaceAll("\\W", "");
+}
+
+public void validate()
+{
+if (!isNameValid(name))
+throw new ConfigurationException("Illegal index name " + name);
+
+if (indexType == null)
+throw new ConfigurationException("Index type is null for index " +
name);
+
+if (targetType == null)
+throw new ConfigurationException("Target type is null for index "
+ name);
+
+if (indexType == IndexMetadata.IndexType.CUSTOM)
+if (options == null ||
!options.containsKey(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME))
+throw new ConfigurationException(String.format("Required
option missing for index %s : %s",
+ name,
SecondaryIndex.CUSTOM_INDEX_OPTION_NAME));
+}
+
+public ColumnDefinition indexedColumn(CFMetaData cfm)
+{
+ return cfm.getColumnDefinition(columns.iterator().next());
+}
+
+public boolean isCustom()
+{
+return indexType == IndexType.CUSTOM;
+}
+
+public boolean isKeys()
+{
+retu
[jira] [Commented] (CASSANDRA-6717) Modernize schema tables
[
https://issues.apache.org/jira/browse/CASSANDRA-6717?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701304#comment-14701304
]
Sam Tunnicliffe commented on CASSANDRA-6717:
committed the 2i changes to 3.0 in
{{06c130e3cb85577041b475084400c08c505d8f9e}}. The latest cassci runs look ok,
as in mostly the failures are also present on 3.0/trunk. The exception is
{{schema_metadata_test:TestSchemaMetadata.indexes_test}} which passes when run
with the latest driver changes (which [~aholmber] has kindly merged to the
{{cassandra-test}} branch now).
> Modernize schema tables
> ---
>
> Key: CASSANDRA-6717
> URL: https://issues.apache.org/jira/browse/CASSANDRA-6717
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Sylvain Lebresne
>Assignee: Sam Tunnicliffe
> Labels: client-impacting, doc-impacting
> Fix For: 3.0 beta 1
>
>
> There is a few problems/improvements that can be done with the way we store
> schema:
> # CASSANDRA-4988: as explained on the ticket, storing the comparator is now
> redundant (or almost, we'd need to store whether the table is COMPACT or not
> too, which we don't currently is easy and probably a good idea anyway), it
> can be entirely reconstructed from the infos in schema_columns (the same is
> true of key_validator and subcomparator, and replacing default_validator by a
> COMPACT_VALUE column in all case is relatively simple). And storing the
> comparator as an opaque string broke concurrent updates of sub-part of said
> comparator (concurrent collection addition or altering 2 separate clustering
> columns typically) so it's really worth removing it.
> # CASSANDRA-4603: it's time to get rid of those ugly json maps. I'll note
> that schema_keyspaces is a problem due to its use of COMPACT STORAGE, but I
> think we should fix it once and for-all nonetheless (see below).
> # For CASSANDRA-6382 and to allow indexing both map keys and values at the
> same time, we'd need to be able to have more than one index definition for a
> given column.
> # There is a few mismatches in table options between the one stored in the
> schema and the one used when declaring/altering a table which would be nice
> to fix. The compaction, compression and replication maps are one already
> mentioned from CASSANDRA-4603, but also for some reason
> 'dclocal_read_repair_chance' in CQL is called just 'local_read_repair_chance'
> in the schema table, and 'min/max_compaction_threshold' are column families
> option in the schema but just compaction options for CQL (which makes more
> sense).
> None of those issues are major, and we could probably deal with them
> independently but it might be simpler to just fix them all in one shot so I
> wanted to sum them all up here. In particular, the fact that
> 'schema_keyspaces' uses COMPACT STORAGE is annoying (for the replication map,
> but it may limit future stuff too) which suggest we should migrate it to a
> new, non COMPACT table. And while that's arguably a detail, it wouldn't hurt
> to rename schema_columnfamilies to schema_tables for the years to come since
> that's the prefered vernacular for CQL.
> Overall, what I would suggest is to move all schema tables to a new keyspace,
> named 'schema' for instance (or 'system_schema' but I prefer the shorter
> version), and fix all the issues above at once. Since we currently don't
> exchange schema between nodes of different versions, all we'd need to do that
> is a one shot startup migration, and overall, I think it could be simpler for
> clients to deal with one clear migration than to have to handle minor
> individual changes all over the place. I also think it's somewhat cleaner
> conceptually to have schema tables in their own keyspace since they are
> replicated through a different mechanism than other system tables.
> If we do that, we could, for instance, migrate to the following schema tables
> (details up for discussion of course):
> {noformat}
> CREATE TYPE user_type (
> name text,
> column_names list,
> column_types list
> )
> CREATE TABLE keyspaces (
> name text PRIMARY KEY,
> durable_writes boolean,
> replication map,
> user_types map
> )
> CREATE TYPE trigger_definition (
> name text,
> options map
> )
> CREATE TABLE tables (
> keyspace text,
> name text,
> id uuid,
> table_type text, // COMPACT, CQL or SUPER
> dropped_columns map,
> triggers map,
> // options
> comment text,
> compaction map,
> compression map,
> read_repair_chance double,
> dclocal_read_repair_chance double,
> gc_grace_seconds int,
> caching text,
> rows_per_partition_to_cache text,
> default_time_to_live int,
> min_index_interval int,
> max_index_interval int,
> speculative_retry text,
> populate_io_cache_on_flush boolean,
> bl
[jira] [Comment Edited] (CASSANDRA-6717) Modernize schema tables
[
https://issues.apache.org/jira/browse/CASSANDRA-6717?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701304#comment-14701304
]
Sam Tunnicliffe edited comment on CASSANDRA-6717 at 8/18/15 2:10 PM:
-
committed the 2i changes to 3.0 in
{{06c130e3cb85577041b475084400c08c505d8f9e}}. The
[latest|http://cassci.datastax.com/view/Dev/view/beobal/job/beobal-6717-testall/lastCompletedBuild/testReport/]
[cassci|http://cassci.datastax.com/view/Dev/view/beobal/job/beobal-6717-dtest/lastCompletedBuild/testReport/]
runs look ok, as in mostly the failures are also present on 3.0/trunk. The
exception is {{schema_metadata_test:TestSchemaMetadata.indexes_test}} which
passes when run with the latest driver changes (which [~aholmber] has kindly
merged to the {{cassandra-test}} branch now).
was (Author: beobal):
committed the 2i changes to 3.0 in
{{06c130e3cb85577041b475084400c08c505d8f9e}}. The latest cassci runs look ok,
as in mostly the failures are also present on 3.0/trunk. The exception is
{{schema_metadata_test:TestSchemaMetadata.indexes_test}} which passes when run
with the latest driver changes (which [~aholmber] has kindly merged to the
{{cassandra-test}} branch now).
> Modernize schema tables
> ---
>
> Key: CASSANDRA-6717
> URL: https://issues.apache.org/jira/browse/CASSANDRA-6717
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Sylvain Lebresne
>Assignee: Sam Tunnicliffe
> Labels: client-impacting, doc-impacting
> Fix For: 3.0 beta 1
>
>
> There is a few problems/improvements that can be done with the way we store
> schema:
> # CASSANDRA-4988: as explained on the ticket, storing the comparator is now
> redundant (or almost, we'd need to store whether the table is COMPACT or not
> too, which we don't currently is easy and probably a good idea anyway), it
> can be entirely reconstructed from the infos in schema_columns (the same is
> true of key_validator and subcomparator, and replacing default_validator by a
> COMPACT_VALUE column in all case is relatively simple). And storing the
> comparator as an opaque string broke concurrent updates of sub-part of said
> comparator (concurrent collection addition or altering 2 separate clustering
> columns typically) so it's really worth removing it.
> # CASSANDRA-4603: it's time to get rid of those ugly json maps. I'll note
> that schema_keyspaces is a problem due to its use of COMPACT STORAGE, but I
> think we should fix it once and for-all nonetheless (see below).
> # For CASSANDRA-6382 and to allow indexing both map keys and values at the
> same time, we'd need to be able to have more than one index definition for a
> given column.
> # There is a few mismatches in table options between the one stored in the
> schema and the one used when declaring/altering a table which would be nice
> to fix. The compaction, compression and replication maps are one already
> mentioned from CASSANDRA-4603, but also for some reason
> 'dclocal_read_repair_chance' in CQL is called just 'local_read_repair_chance'
> in the schema table, and 'min/max_compaction_threshold' are column families
> option in the schema but just compaction options for CQL (which makes more
> sense).
> None of those issues are major, and we could probably deal with them
> independently but it might be simpler to just fix them all in one shot so I
> wanted to sum them all up here. In particular, the fact that
> 'schema_keyspaces' uses COMPACT STORAGE is annoying (for the replication map,
> but it may limit future stuff too) which suggest we should migrate it to a
> new, non COMPACT table. And while that's arguably a detail, it wouldn't hurt
> to rename schema_columnfamilies to schema_tables for the years to come since
> that's the prefered vernacular for CQL.
> Overall, what I would suggest is to move all schema tables to a new keyspace,
> named 'schema' for instance (or 'system_schema' but I prefer the shorter
> version), and fix all the issues above at once. Since we currently don't
> exchange schema between nodes of different versions, all we'd need to do that
> is a one shot startup migration, and overall, I think it could be simpler for
> clients to deal with one clear migration than to have to handle minor
> individual changes all over the place. I also think it's somewhat cleaner
> conceptually to have schema tables in their own keyspace since they are
> replicated through a different mechanism than other system tables.
> If we do that, we could, for instance, migrate to the following schema tables
> (details up for discussion of course):
> {noformat}
> CREATE TYPE user_type (
> name text,
> column_names list,
> column_types list
> )
> CREATE TABLE keyspaces (
> name text PRIMARY KEY,
> durable_writes boolea
[jira] [Commented] (CASSANDRA-9893) Fix upgrade tests from #9704 that are still failing
[ https://issues.apache.org/jira/browse/CASSANDRA-9893?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701317#comment-14701317 ] Philip Thompson commented on CASSANDRA-9893: It doesn't appear to have been committed. I'll work on merging that in today. > Fix upgrade tests from #9704 that are still failing > --- > > Key: CASSANDRA-9893 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9893 > Project: Cassandra > Issue Type: Bug >Reporter: Sylvain Lebresne >Assignee: Blake Eggleston > Fix For: 3.0 beta 2 > > > The first things to do on this ticket would be to commit Tyler's branch > (https://github.com/thobbs/cassandra-dtest/tree/8099-backwards-compat) to the > dtests so cassci run them. I've had to do a few minor modifications to have > them run locally so someone which access to cassci should do it and make sure > it runs properly. > Once we have that, we should fix any test that isn't passing. I've ran the > tests locally and I had 8 failures. for 2 of them, it sounds plausible that > they'll get fixed by the patch of CASSANDRA-9775, though that's just a guess. > The rest where test that timeouted without a particular error in the log, > and running some of them individually, they passed. So we'll have to see if > it's just my machine being overly slow when running them all. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701366#comment-14701366
]
Philip Thompson commented on CASSANDRA-10109:
-
I'm running into the same issue on OSX, and on tests other than ttl_tests.
Specifically the cql_tests
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9554) Avoid digest mismatch storm on upgade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701389#comment-14701389
]
Sylvain Lebresne commented on CASSANDRA-9554:
-
I've rebased the patch on [my
branch|https://github.com/pcmanus/cassandra/commits/9554] (CI:
[utests|http://cassci.datastax.com/job/pcmanus-9554-testall/1/] and
[dtests|http://cassci.datastax.com/job/pcmanus-9554-dtest/1/] - no new failure
that I can see). I did manually tested it this time (by adding new log messages
on mismatch and manually validating that we had mismatch without the patch on
update but not with the patch) and it appears to work as expected.
We should make that automated dtests however and I think we can do that by
using tracing since mismatches are sent in traces. That said, I'm not all that
familiar with the python driver so I haven't dug into how to read traces yet,
and besides, I'm not sure what's our usual way to deal with upgrade tests so
they play well with cassci. Maybe some test engineer that is more familiar with
all this has a few cycles to write proper automated dtests for this?
([~enigmacurry], [~philipthompson]?)
Anyway, in the meantime, the code is actually ready for review.
> Avoid digest mismatch storm on upgade to 3.0
>
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-6230) Write hints to flat files instead of the system.hints
[ https://issues.apache.org/jira/browse/CASSANDRA-6230?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701427#comment-14701427 ] Aleksey Yeschenko commented on CASSANDRA-6230: -- Thanks. [dtests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-testall/] are fine now, making [dtests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-dtest/] happy still requires some work, but will hopefully deal with all of them (all broken by 6230, that is) tonight, and commit. > Write hints to flat files instead of the system.hints > - > > Key: CASSANDRA-6230 > URL: https://issues.apache.org/jira/browse/CASSANDRA-6230 > Project: Cassandra > Issue Type: Sub-task > Components: Core >Reporter: Jonathan Ellis >Assignee: Aleksey Yeschenko > Fix For: 3.0 beta 1 > > > Writing to a file would have less overhead on both hint creation and replay. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Comment Edited] (CASSANDRA-6230) Write hints to flat files instead of the system.hints
[ https://issues.apache.org/jira/browse/CASSANDRA-6230?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701427#comment-14701427 ] Aleksey Yeschenko edited comment on CASSANDRA-6230 at 8/18/15 3:31 PM: --- Thanks. [utests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-testall/] are fine now, making [dtests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-dtest/] happy still requires some work, but will hopefully deal with all of them (all broken by 6230, that is) tonight, and commit. was (Author: iamaleksey): Thanks. [dtests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-testall/] are fine now, making [dtests|http://cassci.datastax.com/job/iamaleksey-6230-3.0-dtest/] happy still requires some work, but will hopefully deal with all of them (all broken by 6230, that is) tonight, and commit. > Write hints to flat files instead of the system.hints > - > > Key: CASSANDRA-6230 > URL: https://issues.apache.org/jira/browse/CASSANDRA-6230 > Project: Cassandra > Issue Type: Sub-task > Components: Core >Reporter: Jonathan Ellis >Assignee: Aleksey Yeschenko > Fix For: 3.0 beta 1 > > > Writing to a file would have less overhead on both hint creation and replay. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10072) "Replica(s) failed to execute read" on simple select on stress-created table with >1 nodes
[
https://issues.apache.org/jira/browse/CASSANDRA-10072?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701429#comment-14701429
]
Jim Witschey commented on CASSANDRA-10072:
--
I have been running that branch of the Python driver. I'm not sure what fixed
it, but the test failures this caused on CassCI are fixed now. Just for the
record, the tests were:
{code}
largecolumn_test.py:TestLargeColumn.cleanup_test
bootstrap_test.py:TestBootstrap.decommissioned_wiped_node_can_join_test
bootstrap_test.py:TestBootstrap.killed_wiped_node_cannot_join_test
bootstrap_test.py:TestBootstrap.manual_bootstrap_test
bootstrap_test.py:TestBootstrap.read_from_bootstrapped_node_test
bootstrap_test.py:TestBootstrap.shutdown_wiped_node_cannot_join_test
consistency_test.py:TestAccuracy.test_network_topology_strategy_users
consistency_test.py:TestAccuracy.test_simple_strategy_users
consistency_test.py:TestAvailability.test_network_topology_strategy
consistency_test.py:TestAvailability.test_simple_strategy
replace_address_test.py:TestReplaceAddress.replace_first_boot_test
replace_address_test.py:TestReplaceAddress.replace_shutdown_node_test
replace_address_test.py:TestReplaceAddress.replace_stopped_node_test
replace_address_test.py:TestReplaceAddress.replace_with_reset_resume_state_test
replace_address_test.py:TestReplaceAddress.resumable_replace_test
{code}
And they stopped failing here:
http://cassci.datastax.com/view/trunk/job/trunk_dtest/485/testReport/junit/
I couldn't find any major changes to the {{cassandra-test}} branch of the
Python driver during the last few days, so I don't think that changes things.
So, who knows why it's fixed, but it seems to be. I'll close this.
> "Replica(s) failed to execute read" on simple select on stress-created table
> with >1 nodes
> --
>
> Key: CASSANDRA-10072
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10072
> Project: Cassandra
> Issue Type: Bug
>Reporter: Jim Witschey
>Assignee: Marcus Eriksson
> Fix For: 3.0 beta 2
>
> Attachments: repro.sh
>
>
> This looks similar to CASSANDRA-9825, though that was marked as resolved.
> To reproduce, create a multi-node cluster and insert values using stress.
> Then, try to run a {{SELECT}} statement on the {{keyspace1.standard1}} table
> created by stress.
> Running the select through the Python driver, I consistently get the result
> {code}
> cassandra.ReadFailure: code=1300 [Replica(s) failed to execute read]
> message="Operation failed - received 0 responses and 1 failures"
> info={'failures': 1, 'received_responses': 0, 'required_responses': 1,
> 'consistency': 'ONE'}
> {code}
> I've attached a script that reproduces it via the {{ccm}} CLI. As the script
> also demonstrates, the error doesn't show under 2.2, or when there is only 1
> node in the cluster.
> In the repro script, I've also included a commented-out insertion via cqlsh.
> If you uncomment that and comment out the stress line, the script
> demonstrates that the error doesn't occur when the table was created/inserted
> into via cqlsh.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Resolved] (CASSANDRA-10072) "Replica(s) failed to execute read" on simple select on stress-created table with >1 nodes
[
https://issues.apache.org/jira/browse/CASSANDRA-10072?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jim Witschey resolved CASSANDRA-10072.
--
Resolution: Fixed
> "Replica(s) failed to execute read" on simple select on stress-created table
> with >1 nodes
> --
>
> Key: CASSANDRA-10072
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10072
> Project: Cassandra
> Issue Type: Bug
>Reporter: Jim Witschey
>Assignee: Marcus Eriksson
> Fix For: 3.0 beta 2
>
> Attachments: repro.sh
>
>
> This looks similar to CASSANDRA-9825, though that was marked as resolved.
> To reproduce, create a multi-node cluster and insert values using stress.
> Then, try to run a {{SELECT}} statement on the {{keyspace1.standard1}} table
> created by stress.
> Running the select through the Python driver, I consistently get the result
> {code}
> cassandra.ReadFailure: code=1300 [Replica(s) failed to execute read]
> message="Operation failed - received 0 responses and 1 failures"
> info={'failures': 1, 'received_responses': 0, 'required_responses': 1,
> 'consistency': 'ONE'}
> {code}
> I've attached a script that reproduces it via the {{ccm}} CLI. As the script
> also demonstrates, the error doesn't show under 2.2, or when there is only 1
> node in the cluster.
> In the repro script, I've also included a commented-out insertion via cqlsh.
> If you uncomment that and comment out the stress line, the script
> demonstrates that the error doesn't occur when the table was created/inserted
> into via cqlsh.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9669) If sstable flushes complete out of order, on restart we can fail to replay necessary commit log records
[
https://issues.apache.org/jira/browse/CASSANDRA-9669?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701436#comment-14701436
]
Branimir Lambov commented on CASSANDRA-9669:
First round of review, focussing at the replay logic:
{{ReplayPosition.add}} is suspect, as [in extending the
end|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-2b76f3efa4aaa38339ab8f4869c9b7bfR70]
{{extend != null}} implies {{extend.getKey().compareTo(end) >= 0}} and this
makes the branch reachable only if {{end == entend.getKey()}} which I don't
think is what you want.
The other direction appears correct, but the two can't be symmetrical as we can
only key into starts. This is sufficient as {{range\[i\].key/lower <
range\[i\].value/upper < range\[i+1\].key/lower}} (which I think you should
clarify in a comment) and thus the entry we want to extend us on the right is
the one before the first start larger than our end. {{end = max(end,
floorEntry(end).value)}} should do it.
There's no need to recursively rerun, extending in either direction has no
effect on the other side so you can do the two in sequence.
{{ReplayPosition.min}} should be a bit more descriptively named, e.g.
"minUpperBound" or "firstNotCovered".
I'd call
[CommitLogReplayer.replay|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-348a1347dacf897385fb0a97116a1b5eR166]
"shouldReplay". Consider moving the containment logic (the last two lines)
into {{ReplayPosition}} to keep the details of the range format within the
class. I believe the intervals are start-inclusive, end-exclusive, thus the
[final
comparison|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-348a1347dacf897385fb0a97116a1b5eR172]
should be {{<=}}.
Could you update the JavaDoc, especially returned values (e.g.
[here|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-98f5acb96aa6d684781936c141132e2aR2498]
and
[here|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-f0a15c3588b56c5ce53ece7c48e325b5R247])?
The comment
[here|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-348a1347dacf897385fb0a97116a1b5eR333]
is outdated.
Is
[FlushRunnable.runMayThrow|https://github.com/apache/cassandra/compare/trunk...belliottsmith:9669-2.0#diff-f0a15c3588b56c5ce53ece7c48e325b5R352]
still needed? DiskAwareRunnable?
I'll continue looking at rest of the code tomorrow.
> If sstable flushes complete out of order, on restart we can fail to replay
> necessary commit log records
> ---
>
> Key: CASSANDRA-9669
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9669
> Project: Cassandra
> Issue Type: Bug
> Components: Core
>Reporter: Benedict
>Assignee: Benedict
>Priority: Critical
> Labels: correctness
> Fix For: 3.x, 2.1.x, 2.2.x, 3.0.x
>
>
> While {{postFlushExecutor}} ensures it never expires CL entries out-of-order,
> on restart we simply take the maximum replay position of any sstable on disk,
> and ignore anything prior.
> It is quite possible for there to be two flushes triggered for a given table,
> and for the second to finish first by virtue of containing a much smaller
> quantity of live data (or perhaps the disk is just under less pressure). If
> we crash before the first sstable has been written, then on restart the data
> it would have represented will disappear, since we will not replay the CL
> records.
> This looks to be a bug present since time immemorial, and also seems pretty
> serious.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-9554) Avoid digest mismatch storm on upgade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701447#comment-14701447
]
Ryan McGuire commented on CASSANDRA-9554:
-
We have some dtest code that exemplifies how to read from traces here :
https://github.com/riptano/cassandra-dtest/blob/master/replication_test.py#L49-L109
We'll talk in standup today who can take this on.
> Avoid digest mismatch storm on upgade to 3.0
>
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-10094) Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM failure
[
https://issues.apache.org/jira/browse/CASSANDRA-10094?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joshua McKenzie updated CASSANDRA-10094:
Reviewer: Benedict (was: Joshua McKenzie)
> Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM
> failure
> ---
>
> Key: CASSANDRA-10094
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10094
> Project: Cassandra
> Issue Type: Bug
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 2.2.x
>
> Attachments: 10094-2.2.txt
>
>
> Error:
> {noformat}
> junit.framework.AssertionFailedError:
> at
> org.apache.cassandra.db.CommitLogFailurePolicyTest.testCommitLogFailureBeforeInitialization_mustKillJVM(CommitLogFailurePolicyTest.java:149)
> {noformat}
> [Failure
> History|http://cassci.datastax.com/view/cassandra-2.2/job/cassandra-2.2_utest_win32/lastCompletedBuild/testReport/org.apache.cassandra.db/CommitLogFailurePolicyTest/testCommitLogFailureBeforeInitialization_mustKillJVM/history/]:
> Consistent since build #85
> Env: CI only. Cannot repro locally
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10094) Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM failure
[
https://issues.apache.org/jira/browse/CASSANDRA-10094?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701453#comment-14701453
]
Joshua McKenzie commented on CASSANDRA-10094:
-
Switching to you as reviewer Benedict - since you've already reviewed it and it
was apparently just a commit snafu.
> Windows utest 2.2: testCommitLogFailureBeforeInitialization_mustKillJVM
> failure
> ---
>
> Key: CASSANDRA-10094
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10094
> Project: Cassandra
> Issue Type: Bug
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 2.2.x
>
> Attachments: 10094-2.2.txt
>
>
> Error:
> {noformat}
> junit.framework.AssertionFailedError:
> at
> org.apache.cassandra.db.CommitLogFailurePolicyTest.testCommitLogFailureBeforeInitialization_mustKillJVM(CommitLogFailurePolicyTest.java:149)
> {noformat}
> [Failure
> History|http://cassci.datastax.com/view/cassandra-2.2/job/cassandra-2.2_utest_win32/lastCompletedBuild/testReport/org.apache.cassandra.db/CommitLogFailurePolicyTest/testCommitLogFailureBeforeInitialization_mustKillJVM/history/]:
> Consistent since build #85
> Env: CI only. Cannot repro locally
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-9554) Avoid digest mismatch storm on upgrade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ryan McGuire updated CASSANDRA-9554:
Summary: Avoid digest mismatch storm on upgrade to 3.0 (was: Avoid digest
mismatch storm on upgade to 3.0)
> Avoid digest mismatch storm on upgrade to 3.0
> -
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-9554) Avoid digest mismatch storm on upgrade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ryan McGuire updated CASSANDRA-9554:
Labels: upgrade (was: )
> Avoid digest mismatch storm on upgrade to 3.0
> -
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Labels: upgrade
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (CASSANDRA-9554) Avoid digest mismatch storm on upgrade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ryan McGuire updated CASSANDRA-9554:
Tester: Russ Hatch
> Avoid digest mismatch storm on upgrade to 3.0
> -
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Labels: upgrade
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Comment Edited] (CASSANDRA-9554) Avoid digest mismatch storm on upgrade to 3.0
[
https://issues.apache.org/jira/browse/CASSANDRA-9554?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701447#comment-14701447
]
Ryan McGuire edited comment on CASSANDRA-9554 at 8/18/15 3:50 PM:
--
We have some dtest code that exemplifies how to read from traces here :
https://github.com/riptano/cassandra-dtest/blob/master/replication_test.py#L49-L109
[~rhatch] will handle this in dtests
was (Author: enigmacurry):
We have some dtest code that exemplifies how to read from traces here :
https://github.com/riptano/cassandra-dtest/blob/master/replication_test.py#L49-L109
We'll talk in standup today who can take this on.
> Avoid digest mismatch storm on upgrade to 3.0
> -
>
> Key: CASSANDRA-9554
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9554
> Project: Cassandra
> Issue Type: Bug
>Reporter: Aleksey Yeschenko
>Assignee: Sylvain Lebresne
> Labels: upgrade
> Fix For: 3.0 beta 2
>
>
> CASSANDRA-8099, in {{UnfilteredRowIterators.digest()}}:
> {code}
> // TODO: we're not computing digest the same way that old nodes. This
> // means we'll have digest mismatches during upgrade. We should pass
> the messaging version of
> // the node this is for (which might mean computing the digest last,
> and won't work
> // for schema (where we announce the version through gossip to
> everyone))
> {code}
> In a mixed 2.1(2.2) - 3.0 clusters, we need to calculate both digest at the
> same time and keep both results, and send the appropriate one, depending on
> receiving nodes' messaging versions. Do that until
> {{MessagingService.allNodesAtLeast30()}} is true (this is not unprecedented).
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701461#comment-14701461
]
Joshua McKenzie commented on CASSANDRA-10109:
-
bq. the best thing we can do
bq. If some are missing, then we assume
bq. if all are present we assume
With the caveat that I have not read deeply into the design on CASSANDRA-7066:
the verbiage here makes me wary. Are we offering the same guarantees of
consistency and safety to a user's data on Windows (and OSX) as we are on Linux?
Are we currently discussing having and maintaining 2 separate execution paths
for Linux and other OS' when it comes to the txn logs and sstable management?
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[1/2] cassandra git commit: Fix testScrubDataDirectories on Windows
Repository: cassandra Updated Branches: refs/heads/trunk e92a4b634 -> be51176d4 Fix testScrubDataDirectories on Windows Patch by stefania; reviewed by jmckenzie for CASSANDRA-10036 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52411606 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52411606 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52411606 Branch: refs/heads/trunk Commit: 5241160652a1b24ee0a20eb3dcd7b76282087c02 Parents: 06c130e Author: Stefania Alborghetti Authored: Tue Aug 18 11:55:26 2015 -0400 Committer: Joshua McKenzie Committed: Tue Aug 18 11:55:26 2015 -0400 -- test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52411606/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java -- diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 9e36bab..78f9e3e 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -530,6 +530,8 @@ public class ColumnFamilyStoreTest String tmpDataFileName = ssTable.descriptor.tmpFilenameFor(Component.DATA); new File(dataFileName).renameTo(new File(tmpDataFileName)); +ssTable.selfRef().release(); + ColumnFamilyStore.scrubDataDirectories(cfs.metadata); List ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
cassandra git commit: Fix testScrubDataDirectories on Windows
Repository: cassandra Updated Branches: refs/heads/cassandra-3.0 06c130e3c -> 524116065 Fix testScrubDataDirectories on Windows Patch by stefania; reviewed by jmckenzie for CASSANDRA-10036 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52411606 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52411606 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52411606 Branch: refs/heads/cassandra-3.0 Commit: 5241160652a1b24ee0a20eb3dcd7b76282087c02 Parents: 06c130e Author: Stefania Alborghetti Authored: Tue Aug 18 11:55:26 2015 -0400 Committer: Joshua McKenzie Committed: Tue Aug 18 11:55:26 2015 -0400 -- test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52411606/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java -- diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 9e36bab..78f9e3e 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -530,6 +530,8 @@ public class ColumnFamilyStoreTest String tmpDataFileName = ssTable.descriptor.tmpFilenameFor(Component.DATA); new File(dataFileName).renameTo(new File(tmpDataFileName)); +ssTable.selfRef().release(); + ColumnFamilyStore.scrubDataDirectories(cfs.metadata); List ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
[2/2] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/be51176d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/be51176d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/be51176d Branch: refs/heads/trunk Commit: be51176d4ed71f63745d2f358347ffa42ddf0531 Parents: e92a4b6 5241160 Author: Joshua McKenzie Authored: Tue Aug 18 11:56:05 2015 -0400 Committer: Joshua McKenzie Committed: Tue Aug 18 11:56:05 2015 -0400 -- test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java | 2 ++ 1 file changed, 2 insertions(+) --
[jira] [Assigned] (CASSANDRA-10098) Windows dtest 3.0: commitlog_test.py:TestCommitLog.small_segment_size_test fails
[
https://issues.apache.org/jira/browse/CASSANDRA-10098?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Paulo Motta reassigned CASSANDRA-10098:
---
Assignee: Paulo Motta
> Windows dtest 3.0: commitlog_test.py:TestCommitLog.small_segment_size_test
> fails
>
>
> Key: CASSANDRA-10098
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10098
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 3.0.x
>
>
> {noformat}
> File "D:\Python27\lib\unittest\case.py", line 329, in run
> testMethod()
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\tools.py",
> line 243, in wrapped
> f(obj)
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\commitlog_test.py",
> line 226, in small_segment_size_test
> self._commitlog_test(segment_size_in_mb, 62.5, 13, files_error=0.2)
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\commitlog_test.py",
> line 99, in _commitlog_test
> error=files_error)
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\assertions.py",
> line 62, in assert_almost_equal
> assert vmin > vmax * (1.0 - error) or vmin == vmax, "values not within
> %.2f%% of the max: %s" % (error * 100, args)
> 'values not within 20.00% of the max: (10, 13)\n >> begin
> captured logging << \ndtest: DEBUG: cluster ccm
> directory: d:\\temp\\dtest-qnguzs\n- >> end captured
> logging << -'
> {noformat}
> Failure history:
> [consistent|http://cassci.datastax.com/view/cassandra-3.0/job/cassandra-3.0_dtest_win32/17/testReport/junit/commitlog_test/TestCommitLog/small_segment_size_test/]
> env: Both ci and local
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701474#comment-14701474
]
Benedict commented on CASSANDRA-10109:
--
bq. Are we currently discussing having and maintaining 2 separate execution
paths for Linux and other OS' when it comes to the txn logs and sstable
management?
bq. Are we offering the same guarantees of consistency and safety to a user's
data on Windows (and OSX) as we are on Linux?
ish.
It's worth noting that we're here worrying about situations that can already
occur but we do not as yet try to deal with. i.e. performing listing of the
sstables on disk during a period that Cassandra is live and actively mutating
this state. So really, your worry is misplaced. It should be worrying about the
status quo, which makes no attempt of any guarantees of providing a consistent
snapshot.
We're also very likely to be able to offer this still, just a little more
unpleasantly on Windows and OSX, however we also are very likely to use the
same basic codepath for both, with the expectation that on Linux it will just
complete first time, whereas on OSX or Windows it may spin a few times trying
to find a consistent view, but probably will not here either, since hitting
this kind of race condition is unlikely. And even if we do, it probably does
not cause anything catastrophic since, as I say, we currently make no such
efforts and the world hasn't collapsed.
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10072) "Replica(s) failed to execute read" on simple select on stress-created table with >1 nodes
[
https://issues.apache.org/jira/browse/CASSANDRA-10072?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701484#comment-14701484
]
Sylvain Lebresne commented on CASSANDRA-10072:
--
I doubt most of the test listed above where due to the python driver. At least
for a number of them, they were due to CASSANDRA-10093 and that's what fixed
it. Not all of the tests you list in your previous comment are fixed though,
but I'm looking at them one at a time.
> "Replica(s) failed to execute read" on simple select on stress-created table
> with >1 nodes
> --
>
> Key: CASSANDRA-10072
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10072
> Project: Cassandra
> Issue Type: Bug
>Reporter: Jim Witschey
>Assignee: Marcus Eriksson
> Fix For: 3.0 beta 2
>
> Attachments: repro.sh
>
>
> This looks similar to CASSANDRA-9825, though that was marked as resolved.
> To reproduce, create a multi-node cluster and insert values using stress.
> Then, try to run a {{SELECT}} statement on the {{keyspace1.standard1}} table
> created by stress.
> Running the select through the Python driver, I consistently get the result
> {code}
> cassandra.ReadFailure: code=1300 [Replica(s) failed to execute read]
> message="Operation failed - received 0 responses and 1 failures"
> info={'failures': 1, 'received_responses': 0, 'required_responses': 1,
> 'consistency': 'ONE'}
> {code}
> I've attached a script that reproduces it via the {{ccm}} CLI. As the script
> also demonstrates, the error doesn't show under 2.2, or when there is only 1
> node in the cluster.
> In the repro script, I've also included a commented-out insertion via cqlsh.
> If you uncomment that and comment out the stress line, the script
> demonstrates that the error doesn't occur when the table was created/inserted
> into via cqlsh.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701488#comment-14701488
]
Benedict commented on CASSANDRA-10109:
--
I'll also note the verbiage was just arbitrary. We can actually make pretty
strong guarantees: if old files are missing, we should expect a COMMIT record,
otherwise we have an inconsistent view. If new files are missing, we should
expect an ABORT message; otherwise we have an inconsistent view. We should
really expect all of the files listed to be on disk, and in this case we know
which filter to apply without any ambiguity.
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10072) "Replica(s) failed to execute read" on simple select on stress-created table with >1 nodes
[
https://issues.apache.org/jira/browse/CASSANDRA-10072?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701492#comment-14701492
]
Jim Witschey commented on CASSANDRA-10072:
--
[~slebresne] True, not all of them are fixed, but they are failing differently
and shouldn't be part of this ticket anymore. Thanks for looking at them!
> "Replica(s) failed to execute read" on simple select on stress-created table
> with >1 nodes
> --
>
> Key: CASSANDRA-10072
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10072
> Project: Cassandra
> Issue Type: Bug
>Reporter: Jim Witschey
>Assignee: Marcus Eriksson
> Fix For: 3.0 beta 2
>
> Attachments: repro.sh
>
>
> This looks similar to CASSANDRA-9825, though that was marked as resolved.
> To reproduce, create a multi-node cluster and insert values using stress.
> Then, try to run a {{SELECT}} statement on the {{keyspace1.standard1}} table
> created by stress.
> Running the select through the Python driver, I consistently get the result
> {code}
> cassandra.ReadFailure: code=1300 [Replica(s) failed to execute read]
> message="Operation failed - received 0 responses and 1 failures"
> info={'failures': 1, 'received_responses': 0, 'required_responses': 1,
> 'consistency': 'ONE'}
> {code}
> I've attached a script that reproduces it via the {{ccm}} CLI. As the script
> also demonstrates, the error doesn't show under 2.2, or when there is only 1
> node in the cluster.
> In the repro script, I've also included a commented-out insertion via cqlsh.
> If you uncomment that and comment out the stress line, the script
> demonstrates that the error doesn't occur when the table was created/inserted
> into via cqlsh.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[2/3] cassandra git commit: downgrade SecureDirectoryStream message to warn
downgrade SecureDirectoryStream message to warn
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ff6e3e78
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ff6e3e78
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ff6e3e78
Branch: refs/heads/trunk
Commit: ff6e3e7842cbe869ec5134909ba17c69fbac2b8d
Parents: 5241160
Author: Benedict Elliott Smith
Authored: Tue Aug 18 17:07:11 2015 +0100
Committer: Benedict Elliott Smith
Committed: Tue Aug 18 17:07:11 2015 +0100
--
src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ff6e3e78/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
--
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
index a9e460c..4b18749 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
@@ -1066,7 +1066,7 @@ public class TransactionLog extends
Transactional.AbstractTransactional implemen
try (DirectoryStream in = Files.newDirectoryStream(folder))
{
if (!(in instanceof SecureDirectoryStream))
-noSpamLogger.error("This platform does not support atomic
directory streams (SecureDirectoryStream); " +
+noSpamLogger.warn("This platform does not support atomic
directory streams (SecureDirectoryStream); " +
"race conditions when loading sstable
files could occurr");
in.forEach(path ->
[3/3] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8b8fc6ba Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8b8fc6ba Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8b8fc6ba Branch: refs/heads/trunk Commit: 8b8fc6bad8a8d18da1a271de151184b93c13f4e0 Parents: be51176 ff6e3e7 Author: Benedict Elliott Smith Authored: Tue Aug 18 17:07:26 2015 +0100 Committer: Benedict Elliott Smith Committed: Tue Aug 18 17:07:26 2015 +0100 -- src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[1/3] cassandra git commit: downgrade SecureDirectoryStream message to warn
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 524116065 -> ff6e3e784
refs/heads/trunk be51176d4 -> 8b8fc6bad
downgrade SecureDirectoryStream message to warn
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ff6e3e78
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ff6e3e78
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ff6e3e78
Branch: refs/heads/cassandra-3.0
Commit: ff6e3e7842cbe869ec5134909ba17c69fbac2b8d
Parents: 5241160
Author: Benedict Elliott Smith
Authored: Tue Aug 18 17:07:11 2015 +0100
Committer: Benedict Elliott Smith
Committed: Tue Aug 18 17:07:11 2015 +0100
--
src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ff6e3e78/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
--
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
index a9e460c..4b18749 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
@@ -1066,7 +1066,7 @@ public class TransactionLog extends
Transactional.AbstractTransactional implemen
try (DirectoryStream in = Files.newDirectoryStream(folder))
{
if (!(in instanceof SecureDirectoryStream))
-noSpamLogger.error("This platform does not support atomic
directory streams (SecureDirectoryStream); " +
+noSpamLogger.warn("This platform does not support atomic
directory streams (SecureDirectoryStream); " +
"race conditions when loading sstable
files could occurr");
in.forEach(path ->
[jira] [Commented] (CASSANDRA-10109) Windows dtest 3.0: ttl_test.py failures
[
https://issues.apache.org/jira/browse/CASSANDRA-10109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701506#comment-14701506
]
Joshua McKenzie commented on CASSANDRA-10109:
-
bq. I'll also note the verbiage was just arbitrary
I expected that was the case and was looking to clarify. You've done so, and
the above sounds reasonable to me.
> Windows dtest 3.0: ttl_test.py failures
> ---
>
> Key: CASSANDRA-10109
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10109
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Stefania
> Labels: Windows
> Fix For: 3.0.x
>
>
> ttl_test.py:TestTTL.update_column_ttl_with_default_ttl_test2
> ttl_test.py:TestTTL.update_multiple_columns_ttl_test
> ttl_test.py:TestTTL.update_single_column_ttl_test
> Errors locally are different than CI from yesterday. Yesterday on CI we have
> timeouts and general node hangs. Today on all 3 tests when run locally I see:
> {noformat}
> Traceback (most recent call last):
> File "c:\src\cassandra-dtest\dtest.py", line 532, in tearDown
> raise AssertionError('Unexpected error in %s node log: %s' % (node.name,
> errors))
> AssertionError: Unexpected error in node1 node log: ['ERROR [main] 2015-08-17
> 16:53:43,120 NoSpamLogger.java:97 - This platform does not support atomic
> directory streams (SecureDirectoryStream); race conditions when loading
> sstable files could occurr']
> {noformat}
> This traces back to the commit for CASSANDRA-7066 today by [~Stefania] and
> [~benedict]. Stefania - care to take this ticket and also look further into
> whether or not we're going to have issues with 7066 on Windows? That error
> message certainly *sounds* like it's not a good thing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[ https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701515#comment-14701515 ] Ariel Weisberg commented on CASSANDRA-8630: --- I guess I mixed up compressed and uncompressed? JFR has an option to load settings from an XML file. You can export the xml file from JMC and then modify the fields you care about. There are some options in there for those thresholds that I don't know off hand. You can also set them in the U and then export. I ran without rate limiting (commented out the acquire) and the original workload. |Version|Run1|| |8630 uncompressed|202| |8630 compressed|197| |Trunk uncompressed|201| |Trunk compressed|203| Looking at the profile I think costs here are dominated by all the Java code for merging. I'll run with JMC and no rate limiting and see where that gets me. I suspect it's going to be very similar. It kind of looks like a bit more than 50% of time is spent materializing data and maybe 40% is spent rewriting it. The top 4 classes are 3 merge iterators and ConcurrentHashMapV8 in concurrent linked hashmap. I guess cache maintenance is part of the overhead of compaction. It looks like we are optimizing the wrong part of the profile? Have things changed since <2.2? I kind of think we are DOA if compaction within a single column family is single threaded anyways. I think I heard of such a thing existing, but no one was satisfied with it. > Faster sequential IO (on compaction, streaming, etc) > > > Key: CASSANDRA-8630 > URL: https://issues.apache.org/jira/browse/CASSANDRA-8630 > Project: Cassandra > Issue Type: Improvement > Components: Core, Tools >Reporter: Oleg Anastasyev >Assignee: Stefania > Labels: compaction, performance > Fix For: 3.x > > Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png, > flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz, > mmaped_uncomp_hotspot.png > > > When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot > of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int). > This is because default implementations of readShort,readLong, etc as well as > their matching write* are implemented with numerous calls of byte by byte > read and write. > This makes a lot of syscalls as well. > A quick microbench shows than just reimplementation of these methods in > either way gives 8x speed increase. > A patch attached implements RandomAccessReader.read and > SequencialWriter.write methods in more efficient way. > I also eliminated some extra byte copies in CompositeType.split and > ColumnNameHelper.maxComponents, which were on my profiler's hotspot method > list during tests. > A stress tests on my laptop show that this patch makes compaction 25-30% > faster on uncompressed sstables and 15% faster for compressed ones. > A deployment to production shows much less CPU load for compaction. > (I attached a cpu load graph from one of our production, orange is niced CPU > load - i.e. compaction; yellow is user - i.e. not compaction related tasks) -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Comment Edited] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[ https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701515#comment-14701515 ] Ariel Weisberg edited comment on CASSANDRA-8630 at 8/18/15 4:24 PM: I guess I mixed up compressed and uncompressed? JFR has an option to load settings from an XML file. You can export the xml file from JMC and then modify the fields you care about. There are some options in there for those thresholds that I don't know off hand. You can also set them in the U and then export. I ran without rate limiting (commented out the acquire) and the original workload. |Version|Run1|| |8630 uncompressed|202| |8630 compressed|197| |Trunk uncompressed|201| |Trunk compressed|203| Looking at the profile I think costs here are dominated by all the Java code for merging. I'll run with JMC and no rate limiting and see where that gets me. I suspect it's going to be very similar. It kind of looks like a bit more than 50% of time is spent materializing data and maybe 40% is spent rewriting it. The top 4 classes are 3 merge iterators and ConcurrentHashMapV8 in concurrent linked hashmap. I guess cache maintenance is part of the overhead of compaction. I am hesitant to prognosticate on where to go to make compaction faster because I know a lot of this stuff is already recorded or debunked. Optimizing single threaded stuff does not seem like the right starting point to me though. was (Author: aweisberg): I guess I mixed up compressed and uncompressed? JFR has an option to load settings from an XML file. You can export the xml file from JMC and then modify the fields you care about. There are some options in there for those thresholds that I don't know off hand. You can also set them in the U and then export. I ran without rate limiting (commented out the acquire) and the original workload. |Version|Run1|| |8630 uncompressed|202| |8630 compressed|197| |Trunk uncompressed|201| |Trunk compressed|203| Looking at the profile I think costs here are dominated by all the Java code for merging. I'll run with JMC and no rate limiting and see where that gets me. I suspect it's going to be very similar. It kind of looks like a bit more than 50% of time is spent materializing data and maybe 40% is spent rewriting it. The top 4 classes are 3 merge iterators and ConcurrentHashMapV8 in concurrent linked hashmap. I guess cache maintenance is part of the overhead of compaction. It looks like we are optimizing the wrong part of the profile? Have things changed since <2.2? I kind of think we are DOA if compaction within a single column family is single threaded anyways. I think I heard of such a thing existing, but no one was satisfied with it. > Faster sequential IO (on compaction, streaming, etc) > > > Key: CASSANDRA-8630 > URL: https://issues.apache.org/jira/browse/CASSANDRA-8630 > Project: Cassandra > Issue Type: Improvement > Components: Core, Tools >Reporter: Oleg Anastasyev >Assignee: Stefania > Labels: compaction, performance > Fix For: 3.x > > Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png, > flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz, > mmaped_uncomp_hotspot.png > > > When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot > of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int). > This is because default implementations of readShort,readLong, etc as well as > their matching write* are implemented with numerous calls of byte by byte > read and write. > This makes a lot of syscalls as well. > A quick microbench shows than just reimplementation of these methods in > either way gives 8x speed increase. > A patch attached implements RandomAccessReader.read and > SequencialWriter.write methods in more efficient way. > I also eliminated some extra byte copies in CompositeType.split and > ColumnNameHelper.maxComponents, which were on my profiler's hotspot method > list during tests. > A stress tests on my laptop show that this patch makes compaction 25-30% > faster on uncompressed sstables and 15% faster for compressed ones. > A deployment to production shows much less CPU load for compaction. > (I attached a cpu load graph from one of our production, orange is niced CPU > load - i.e. compaction; yellow is user - i.e. not compaction related tasks) -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (CASSANDRA-10097) Windows dtest 3.0: bootstrap_test.py:TestBootstrap.bootstrap_with_reset_bootstrap_state_test fails
[
https://issues.apache.org/jira/browse/CASSANDRA-10097?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701517#comment-14701517
]
Paulo Motta commented on CASSANDRA-10097:
-
The test interrupts bootstrap of a new node by killing one of the seed nodes,
and then tries to re-bootstrap the node with the flag
{{-Dcassandra.reset_bootstrap_progress=true}}.
The problem is, the boostrap is finished before the seed node is killed on
Windows , so the new node is already bootstrapped when it tries to
re-bootstrap, so the test fails. Probably it works on Linux due to performance
differences with ccm and python threads.
Basic fix is to limit the stream_throughput_outbound_megabits_per_sec on the
seed node, to allow time for killing before the boostrap streaming is complete.
For some reason, this was being done only on the joining node.
Adding [~philipthompson] as reviewer, since it's a simple cassandra-dtest fix
on this [PR|https://github.com/riptano/cassandra-dtest/pull/475].
> Windows dtest 3.0:
> bootstrap_test.py:TestBootstrap.bootstrap_with_reset_bootstrap_state_test
> fails
> --
>
> Key: CASSANDRA-10097
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10097
> Project: Cassandra
> Issue Type: Sub-task
>Reporter: Joshua McKenzie
>Assignee: Paulo Motta
> Labels: Windows
> Fix For: 3.0.x
>
>
> {noformat}
> File "D:\Python27\lib\unittest\case.py", line 329, in run
> testMethod()
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\tools.py",
> line 243, in wrapped
> f(obj)
> File
> "D:\jenkins\workspace\cassandra-3.0_dtest_win32\cassandra-dtest\bootstrap_test.py",
> line 184, in bootstrap_with_reset_bootstrap_state_test
> node3.watch_log_for("Resetting bootstrap progress to start fresh",
> from_mark=mark)
> File "build\bdist.win-amd64\egg\ccmlib\node.py", line 382, in watch_log_for
> raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + "
> [" + self.name + "] Missing: " + str([e.pattern for e in tofind]) + ":\n" +
> reads)
> {noformat}
> Failure history:
> [consistent|http://cassci.datastax.com/view/cassandra-3.0/job/cassandra-3.0_dtest_win32/17/testReport/junit/bootstrap_test/TestBootstrap/bootstrap_with_reset_bootstrap_state_test/history/]
> Env: both ci and locally
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (CASSANDRA-8630) Faster sequential IO (on compaction, streaming, etc)
[ https://issues.apache.org/jira/browse/CASSANDRA-8630?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701519#comment-14701519 ] Benedict commented on CASSANDRA-8630: - Bear in mind you're performing a 100 way compaction, which is almost unheard of. Typical compaction is 4-way, in which scenario this is likely to dominate a great deal more. This is also performed on every read, not just compaction, so optimising this is definitely helpful. Especially given it is a relatively LHF. bq. I kind of think we are DOA if compaction within a single column family is single threaded anyways A single compaction task is single threaded. There may be multiple such compactions happening in parallel. We may reintroduce parallel compaction at a later date, but it's not something we're likely to introduce in the near future. > Faster sequential IO (on compaction, streaming, etc) > > > Key: CASSANDRA-8630 > URL: https://issues.apache.org/jira/browse/CASSANDRA-8630 > Project: Cassandra > Issue Type: Improvement > Components: Core, Tools >Reporter: Oleg Anastasyev >Assignee: Stefania > Labels: compaction, performance > Fix For: 3.x > > Attachments: 8630-FasterSequencialReadsAndWrites.txt, cpu_load.png, > flight_recorder_001_files.tar.gz, flight_recorder_002_files.tar.gz, > mmaped_uncomp_hotspot.png > > > When node is doing a lot of sequencial IO (streaming, compacting, etc) a lot > of CPU is lost in calls to RAF's int read() and DataOutputStream's write(int). > This is because default implementations of readShort,readLong, etc as well as > their matching write* are implemented with numerous calls of byte by byte > read and write. > This makes a lot of syscalls as well. > A quick microbench shows than just reimplementation of these methods in > either way gives 8x speed increase. > A patch attached implements RandomAccessReader.read and > SequencialWriter.write methods in more efficient way. > I also eliminated some extra byte copies in CompositeType.split and > ColumnNameHelper.maxComponents, which were on my profiler's hotspot method > list during tests. > A stress tests on my laptop show that this patch makes compaction 25-30% > faster on uncompressed sstables and 15% faster for compressed ones. > A deployment to production shows much less CPU load for compaction. > (I attached a cpu load graph from one of our production, orange is niced CPU > load - i.e. compaction; yellow is user - i.e. not compaction related tasks) -- This message was sent by Atlassian JIRA (v6.3.4#6332)
