This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git


The following commit(s) were added to refs/heads/cep-7-sai by this push:
     new 05dd58783a Add basic text analysis to SAI, including "case_sensitive", 
"normalize", and "ascii" modes
05dd58783a is described below

commit 05dd58783a83c3bd2073f8f96924a99a73cc4f8a
Author: Mike Adamson <madam...@datastax.com>
AuthorDate: Tue May 9 12:29:01 2023 +0100

    Add basic text analysis to SAI, including "case_sensitive", "normalize", 
and "ascii" modes
    
    patch by Mike Adamson; reviewed by Caleb Rackliffe and Andres de la Peña 
for CASSANDRA-18479
---
 .build/cassandra-deps-template.xml                 |   8 +-
 .build/parent-pom-template.xml                     |  10 +-
 .../apache/cassandra/index/sai/IndexContext.java   |  31 ++--
 .../cassandra/index/sai/StorageAttachedIndex.java  |   6 +-
 .../index/sai/analyzer/AbstractAnalyzer.java       |  26 ++++
 .../index/sai/analyzer/NonTokenizingAnalyzer.java  | 147 +++++++++++++++++++
 .../index/sai/analyzer/NonTokenizingOptions.java   | 156 +++++++++++++++++++++
 .../index/sai/analyzer/filter/BasicFilters.java    |  82 +++++++++++
 .../index/sai/analyzer/filter/FilterPipeline.java  |  69 +++++++++
 .../analyzer/filter/FilterPipelineExecutor.java    |  40 ++++++
 .../index/sai/disk/v1/SSTableIndexWriter.java      |   2 +-
 .../index/sai/memory/TrieMemoryIndex.java          |   2 +-
 .../cassandra/index/sai/plan/Expression.java       |   2 +-
 .../apache/cassandra/index/sai/plan/Operation.java |   2 +-
 .../sai/plan/StorageAttachedIndexSearcher.java     |   2 +-
 .../index/sai/virtual/ColumnIndexesSystemView.java |   2 +-
 .../index/sasi/analyzer/filter/StemmerFactory.java |  55 +++++---
 .../test/sai/ReplicaFilteringProtectionTest.java   |  67 +++++++++
 .../sai/analyzer/NonTokenizingAnalyzerTest.java    |  75 ++++++++++
 .../sai/analyzer/filter/BasicFiltersTest.java      |  73 ++++++++++
 .../index/sai/cql/StorageAttachedIndexDDLTest.java | 106 ++++++++++++++
 .../index/sai/virtual/IndexesSystemViewTest.java   |   2 +-
 22 files changed, 904 insertions(+), 61 deletions(-)

diff --git a/.build/cassandra-deps-template.xml 
b/.build/cassandra-deps-template.xml
index c2194a0ab7..76e3bfdd4c 100644
--- a/.build/cassandra-deps-template.xml
+++ b/.build/cassandra-deps-template.xml
@@ -328,10 +328,6 @@
       <groupId>de.jflex</groupId>
       <artifactId>jflex</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.github.rholder</groupId>
-      <artifactId>snowball-stemmer</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.googlecode.concurrent-trees</groupId>
       <artifactId>concurrent-trees</artifactId>
@@ -380,5 +376,9 @@
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analysis-common</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
index 6256f1339a..d4e09aafaf 100644
--- a/.build/parent-pom-template.xml
+++ b/.build/parent-pom-template.xml
@@ -874,11 +874,6 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>com.github.rholder</groupId>
-        <artifactId>snowball-stemmer</artifactId>
-        <version>1.3.0.581.1</version>
-      </dependency>
       <dependency>
         <groupId>com.googlecode.concurrent-trees</groupId>
         <artifactId>concurrent-trees</artifactId>
@@ -1047,6 +1042,11 @@
         <artifactId>lucene-core</artifactId>
         <version>9.7.0</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.lucene</groupId>
+        <artifactId>lucene-analysis-common</artifactId>
+        <version>9.7.0</version>
+      </dependency>
       <dependency>
         <groupId>com.carrotsearch.randomizedtesting</groupId>
         <artifactId>randomizedtesting-runner</artifactId>
diff --git a/src/java/org/apache/cassandra/index/sai/IndexContext.java 
b/src/java/org/apache/cassandra/index/sai/IndexContext.java
index 3e18afe346..59e25777bc 100644
--- a/src/java/org/apache/cassandra/index/sai/IndexContext.java
+++ b/src/java/org/apache/cassandra/index/sai/IndexContext.java
@@ -90,8 +90,7 @@ public class IndexContext
     private final IndexViewManager viewManager;
     private final IndexMetrics indexMetrics;
     private final ColumnQueryMetrics columnQueryMetrics;
-    private final AbstractAnalyzer.AnalyzerFactory indexAnalyzerFactory;
-    private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
+    private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
     private final PrimaryKey.Factory primaryKeyFactory;
 
     public IndexContext(String keyspace,
@@ -119,9 +118,8 @@ public class IndexContext
         this.columnQueryMetrics = isLiteral() ? new 
ColumnQueryMetrics.TrieIndexMetrics(this)
                                               : new 
ColumnQueryMetrics.BalancedTreeIndexMetrics(this);
 
-        // We currently only support the NoOpAnalyzer
-        this.indexAnalyzerFactory = 
AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
-        this.queryAnalyzerFactory = 
AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
+        this.analyzerFactory = indexMetadata == null ? 
AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap())
+                                                     : 
AbstractAnalyzer.fromOptions(getValidator(), indexMetadata.options);
     }
 
     public AbstractType<?> keyValidator()
@@ -199,23 +197,12 @@ public class IndexContext
     }
 
     /**
-     * Returns an {@code AnalyzerFactory} for use by the write path to 
transform incoming literal
-     * during indexing. The analyzers can be tokenising or non-tokenising. 
Tokenising analyzers
-     * will split the incoming terms into multiple terms in the index while 
non-tokenising analyzers
-     * will not split the incoming term but will transform the term (e.g. 
case-insensitive)
+     * Returns an {@link AbstractAnalyzer.AnalyzerFactory} for use by write 
and query paths to transform
+     * literal values.
      */
-    public AbstractAnalyzer.AnalyzerFactory getIndexAnalyzerFactory()
+    public AbstractAnalyzer.AnalyzerFactory getAnalyzerFactory()
     {
-        return indexAnalyzerFactory;
-    }
-
-    /**
-     * Return an {@code AnalyzerFactory} for use by the query path to 
transform query terms before
-     * searching for them in the index. This can be the same as the 
indexAnalyzerFactory.
-     */
-    public AbstractAnalyzer.AnalyzerFactory getQueryAnalyzerFactory()
-    {
-        return queryAnalyzerFactory;
+        return analyzerFactory;
     }
 
     public View getView()
@@ -256,9 +243,7 @@ public class IndexContext
     public void invalidate()
     {
         viewManager.invalidate();
-        indexAnalyzerFactory.close();
-        if (queryAnalyzerFactory != indexAnalyzerFactory)
-            queryAnalyzerFactory.close();
+        analyzerFactory.close();
         if (memtableIndexManager != null)
             memtableIndexManager.invalidate();
         if (indexMetrics != null)
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java 
b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
index 543c14e8a2..03f006dda7 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
@@ -74,6 +74,7 @@ import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.index.TargetParser;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
 import org.apache.cassandra.index.sai.disk.SSTableIndex;
 import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
 import org.apache.cassandra.index.sai.disk.format.Version;
@@ -139,7 +140,10 @@ public class StorageAttachedIndex implements Index
     private static final StorageAttachedIndexBuildingSupport 
INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport();
 
     private static final Set<String> VALID_OPTIONS = 
ImmutableSet.of(IndexTarget.TARGET_OPTION_NAME,
-                                                                     
IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+                                                                     
IndexTarget.CUSTOM_INDEX_OPTION_NAME,
+                                                                     
NonTokenizingOptions.CASE_SENSITIVE,
+                                                                     
NonTokenizingOptions.NORMALIZE,
+                                                                     
NonTokenizingOptions.ASCII);
 
     public static final Set<CQL3Type> SUPPORTED_TYPES = 
ImmutableSet.of(CQL3Type.Native.ASCII, CQL3Type.Native.BIGINT, 
CQL3Type.Native.DATE,
                                                                         
CQL3Type.Native.DOUBLE, CQL3Type.Native.FLOAT, CQL3Type.Native.INT,
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java 
b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
index d1e7bae2fe..1f5e339657 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
@@ -24,10 +24,13 @@ import java.util.Map;
 import java.util.NoSuchElementException;
 
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.StringType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 
 public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
 {
     protected ByteBuffer next = null;
+    protected String nextLiteral = null;
 
     /**
      * @return true if index value is transformed, e.g. normalized or 
lower-cased or tokenized.
@@ -65,6 +68,7 @@ public abstract class AbstractAnalyzer implements 
Iterator<ByteBuffer>
     public void reset(ByteBuffer input)
     {
         this.next = null;
+        this.nextLiteral = null;
 
         resetInternal(input);
     }
@@ -80,6 +84,28 @@ public abstract class AbstractAnalyzer implements 
Iterator<ByteBuffer>
 
     public static AnalyzerFactory fromOptions(AbstractType<?> type, 
Map<String, String> options)
     {
+        if (hasNonTokenizingOptions(options))
+        {
+            if (type instanceof StringType)
+            {
+                // validate options
+                NonTokenizingOptions.fromMap(options);
+                return () -> new NonTokenizingAnalyzer(type, options);
+            }
+            else
+            {
+                throw new InvalidRequestException("CQL type " + 
type.asCQL3Type() + " cannot be analyzed.");
+            }
+        }
+
         return NoOpAnalyzer::new;
     }
+
+    private static boolean hasNonTokenizingOptions(Map<String, String> options)
+    {
+        return options.get(NonTokenizingOptions.ASCII) != null ||
+               options.containsKey(NonTokenizingOptions.CASE_SENSITIVE) ||
+               options.containsKey(NonTokenizingOptions.NORMALIZE);
+    }
+
 }
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java 
b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
new file mode 100644
index 0000000000..f58158a823
--- /dev/null
+++ 
b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.StringType;
+import org.apache.cassandra.index.sai.analyzer.filter.BasicFilters;
+import org.apache.cassandra.index.sai.analyzer.filter.FilterPipeline;
+import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineExecutor;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Analyzer that does *not* tokenize the input. Optionally will
+ * apply filters for the input based on {@link NonTokenizingOptions}.
+ */
+public class NonTokenizingAnalyzer extends AbstractAnalyzer
+{
+    private static final Logger logger = 
LoggerFactory.getLogger(NonTokenizingAnalyzer.class);
+
+    private final AbstractType<?> type;
+    private final NonTokenizingOptions options;
+    private final FilterPipeline filterPipeline;
+
+    private ByteBuffer input;
+    private boolean hasNext = false;
+
+    NonTokenizingAnalyzer(AbstractType<?> type, Map<String, String> options)
+    {
+        this(type, NonTokenizingOptions.fromMap(options));
+    }
+
+    NonTokenizingAnalyzer(AbstractType<?> type, NonTokenizingOptions 
tokenizerOptions)
+    {
+        this.type = type;
+        this.options = tokenizerOptions;
+        this.filterPipeline = getFilterPipeline();
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        // check that we know how to handle the input, otherwise bail
+        if (!(type instanceof StringType)) return false;
+
+        if (hasNext)
+        {
+            try
+            {
+                String input = type.getString(this.input);
+
+                if (input == null)
+                {
+                    throw new MarshalException(String.format("'null' 
deserialized value for %s with %s",
+                                                             
ByteBufferUtil.bytesToHex(this.input), type));
+                }
+
+                String result = FilterPipelineExecutor.execute(filterPipeline, 
input);
+                
+                if (result == null)
+                {
+                    nextLiteral = null;
+                    next = null;
+                    return false;
+                }
+
+                nextLiteral = result;
+                next = type.fromString(result);
+
+                return true;
+            }
+            catch (MarshalException e)
+            {
+                logger.error("Failed to deserialize value with " + type, e);
+                return false;
+            }
+            finally
+            {
+                hasNext = false;
+            }
+        }
+
+        return false;
+    }
+
+    @Override
+    public boolean transformValue()
+    {
+        return !options.isCaseSensitive() || options.isNormalized() || 
options.isAscii();
+    }
+
+    @Override
+    protected void resetInternal(ByteBuffer input)
+    {
+        this.input = input;
+        this.hasNext = true;
+    }
+
+    private FilterPipeline getFilterPipeline()
+    {
+        FilterPipeline builder = new FilterPipeline(new 
BasicFilters.NoOperation());
+        
+        if (!options.isCaseSensitive())
+            builder = builder.add("to_lower", new BasicFilters.LowerCase());
+        
+        if (options.isNormalized())
+            builder = builder.add("normalize", new BasicFilters.Normalize());
+
+        if (options.isAscii())
+            builder = builder.add("ascii", new BasicFilters.Ascii());
+        
+        return builder;
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("caseSensitive", options.isCaseSensitive())
+                          .add("normalized", options.isNormalized())
+                          .add("ascii", options.isAscii())
+                          .toString();
+    }
+}
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java 
b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java
new file mode 100644
index 0000000000..ab6485acf5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.base.Strings;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+public class NonTokenizingOptions
+{
+    public static final String NORMALIZE = "normalize";
+    public static final String CASE_SENSITIVE = "case_sensitive";
+    public static final String ASCII = "ascii";
+
+    private boolean caseSensitive;
+    private boolean normalized;
+    private boolean ascii;
+
+    boolean isCaseSensitive()
+    {
+        return caseSensitive;
+    }
+
+    void setCaseSensitive(boolean caseSensitive)
+    {
+        this.caseSensitive = caseSensitive;
+    }
+    
+    boolean isNormalized()
+    {
+        return this.normalized;
+    }
+
+    void setAscii(boolean ascii)
+    {
+        this.ascii = ascii;
+    }
+
+    boolean isAscii()
+    {
+        return this.ascii;
+    }
+    
+    void setNormalized(boolean normalized)
+    {
+        this.normalized = normalized;
+    }
+
+    public static class OptionsBuilder
+    {
+        private boolean caseSensitive = true;
+        private boolean normalized = false;
+        private boolean ascii = false;
+
+        OptionsBuilder() {}
+
+        OptionsBuilder caseSensitive(boolean caseSensitive)
+        {
+            this.caseSensitive = caseSensitive;
+            return this;
+        }
+
+        OptionsBuilder ascii(boolean ascii)
+        {
+            this.ascii = ascii;
+            return this;
+        }
+
+        OptionsBuilder normalized(boolean normalized)
+        {
+            this.normalized = normalized;
+            return this;
+        }
+
+        public NonTokenizingOptions build()
+        {
+            NonTokenizingOptions options = new NonTokenizingOptions();
+            options.setCaseSensitive(caseSensitive);
+            options.setNormalized(normalized);
+            options.setAscii(ascii);
+            return options;
+        }
+    }
+
+    public static NonTokenizingOptions getDefaultOptions()
+    {
+        return fromMap(new HashMap<>(1));
+    }
+
+    public static NonTokenizingOptions fromMap(Map<String, String> options)
+    {
+        OptionsBuilder builder = new OptionsBuilder();
+
+        for (Map.Entry<String, String> entry : options.entrySet())
+        {
+            switch (entry.getKey())
+            {
+                case CASE_SENSITIVE:
+                {
+                    boolean boolValue = validateBoolean(entry.getValue(), 
CASE_SENSITIVE);
+                    builder = builder.caseSensitive(boolValue);
+                    break;
+                }
+                
+                case NORMALIZE:
+                {
+                    boolean boolValue = validateBoolean(entry.getValue(), 
NORMALIZE);
+                    builder = builder.normalized(boolValue);
+                    break;
+                }
+
+                case ASCII:
+                {
+                    boolean boolValue = validateBoolean(entry.getValue(), 
ASCII);
+                    builder = builder.ascii(boolValue);
+                    break;
+                }
+            }
+        }
+        return builder.build();
+    }
+
+    private static boolean validateBoolean(String value, String option)
+    {
+        if (Strings.isNullOrEmpty(value))
+        {
+            throw new InvalidRequestException("Empty value for boolean option 
'" + option + '\'');
+        }
+
+        if (!value.equalsIgnoreCase(Boolean.TRUE.toString()) && 
!value.equalsIgnoreCase(Boolean.FALSE.toString()))
+        {
+            throw new InvalidRequestException("Illegal value for boolean 
option '" + option + "': " + value);
+        }
+
+        return Boolean.parseBoolean(value);
+    }
+}
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicFilters.java 
b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicFilters.java
new file mode 100644
index 0000000000..b70fbae9da
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicFilters.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import java.text.Normalizer;
+import java.util.Locale;
+
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+
+public class BasicFilters
+{
+    private static final Locale DEFAULT_LOCALE = Locale.getDefault();
+
+    public static class LowerCase extends FilterPipeline.Task
+    {
+        private final Locale locale;
+
+        public LowerCase()
+        {
+            this.locale = DEFAULT_LOCALE;
+        }
+
+        @Override
+        public String process(String input)
+        {
+            return input.toLowerCase(locale);
+        }
+    }
+
+    public static class Normalize extends FilterPipeline.Task
+    {
+        public Normalize() { }
+
+        @Override
+        public String process(String input)
+        {
+            if (input == null) return null;
+            return Normalizer.isNormalized(input, Normalizer.Form.NFC) ? input 
: Normalizer.normalize(input, Normalizer.Form.NFC);
+        }
+    }
+
+    public static class Ascii extends FilterPipeline.Task
+    {
+        public Ascii() { }
+
+        @Override
+        public String process(String input)
+        {
+            if (input == null) return null;
+            char[] inputChars = input.toCharArray();
+            // The output can (potentially) be 4 times the size of the input
+            char[] outputChars = new char[inputChars.length * 4];
+            int outputSize = ASCIIFoldingFilter.foldToASCII(inputChars, 0, 
outputChars, 0, inputChars.length);
+            return new String(outputChars, 0, outputSize);
+        }
+    }
+
+    public static class NoOperation extends FilterPipeline.Task
+    {
+        @Override
+        public String process(String input)
+        {
+            return input;
+        }
+    }
+}
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipeline.java 
b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipeline.java
new file mode 100644
index 0000000000..017168321b
--- /dev/null
+++ 
b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipeline.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A linked list of {@link Task} objects. Used to apply a sequence of 
filtering tasks
+ * to provided textual input in a guaranteed order.
+ */
+@NotThreadSafe
+public class FilterPipeline
+{
+    private final Task head;
+    private Task tail;
+
+    public FilterPipeline(Task first)
+    {
+        this(first, first);
+    }
+
+    private FilterPipeline(Task first, Task tail)
+    {
+        this.head = first;
+        this.tail = tail;
+    }
+
+    public FilterPipeline add(String name, Task task)
+    {
+        Preconditions.checkArgument(task != this.tail, "Provided last task [" 
+ task.name + "] cannot be set to itself");
+        
+        this.tail.next = task;
+        this.tail.name = name;
+        
+        this.tail = task;
+        return this;
+    }
+
+    public Task head()
+    {
+        return this.head;
+    }
+
+    public abstract static class Task
+    {
+        public String name;
+        public Task next;
+
+        public abstract String process(String input);
+    }
+}
diff --git 
a/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
 
b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
new file mode 100644
index 0000000000..c863f1e3cd
--- /dev/null
+++ 
b/src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+/**
+ * Executes all linked {@link FilterPipeline.Task}s serially on the provided 
input and returns a result
+ */
+public class FilterPipelineExecutor
+{
+    public static String execute(FilterPipeline pipeline, String initialInput)
+    {
+        FilterPipeline.Task currentTask = pipeline.head();
+        String result = initialInput;
+        
+        while (true)
+        {
+            result = currentTask.process(result);
+            currentTask = currentTask.next;
+            
+            if (currentTask == null)
+                return result;
+        }
+    }
+}
diff --git 
a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java 
b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
index ec1b6a338f..544aae5e66 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
@@ -79,7 +79,7 @@ public class SSTableIndexWriter implements 
PerColumnIndexWriter
     {
         this.indexDescriptor = indexDescriptor;
         this.indexContext = indexContext;
-        this.analyzer = indexContext.getIndexAnalyzerFactory().create();
+        this.analyzer = indexContext.getAnalyzerFactory().create();
         this.limiter = limiter;
         this.isIndexValid = isIndexValid;
         this.maxTermSize = indexContext.isFrozen() ? MAX_FROZEN_TERM_SIZE : 
MAX_STRING_TERM_SIZE;
diff --git 
a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java 
b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
index e84a3f9466..0df665d930 100644
--- a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
@@ -77,7 +77,7 @@ public class TrieMemoryIndex
         this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
         this.primaryKeysReducer = new PrimaryKeysReducer();
         // The use of the analyzer is within a synchronized block so can be 
considered thread-safe
-        this.analyzerFactory = indexContext.getIndexAnalyzerFactory();
+        this.analyzerFactory = indexContext.getAnalyzerFactory();
         this.validator = indexContext.getValidator();
         this.isLiteral = TypeUtil.isLiteral(validator);
     }
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Expression.java 
b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
index a36bf315d5..d33470d515 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/Expression.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
@@ -93,7 +93,7 @@ public class Expression
     public Expression(IndexContext indexContext)
     {
         this.context = indexContext;
-        this.analyzerFactory = indexContext.getQueryAnalyzerFactory();
+        this.analyzerFactory = indexContext.getAnalyzerFactory();
         this.validator = indexContext.getValidator();
     }
 
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Operation.java 
b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
index d911e09277..e9fba9760c 100644
--- a/src/java/org/apache/cassandra/index/sai/plan/Operation.java
+++ b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
@@ -76,7 +76,7 @@ public class Operation
             IndexContext indexContext = controller.getContext(e);
             List<Expression> perColumn = analyzed.get(e.column());
 
-            AbstractAnalyzer analyzer = 
indexContext.getQueryAnalyzerFactory().create();
+            AbstractAnalyzer analyzer = 
indexContext.getAnalyzerFactory().create();
             try
             {
                 analyzer.reset(e.getIndexValue().duplicate());
diff --git 
a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
 
b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
index 09acca2649..adcf36d157 100644
--- 
a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
+++ 
b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
@@ -86,7 +86,7 @@ public class StorageAttachedIndexSearcher implements 
Index.Searcher
     {
         for (RowFilter.Expression expression : 
queryController.filterOperation())
         {
-            AbstractAnalyzer analyzer = 
queryController.getContext(expression).getIndexAnalyzerFactory().create();
+            AbstractAnalyzer analyzer = 
queryController.getContext(expression).getAnalyzerFactory().create();
             try
             {
                 if (analyzer.transformValue())
diff --git 
a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java 
b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
index 2a9528db62..39eaa6d7ee 100644
--- 
a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
+++ 
b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
@@ -104,7 +104,7 @@ public class ColumnIndexesSystemView extends 
AbstractVirtualTable
                                .column(IS_QUERYABLE, 
manager.isIndexQueryable(index))
                                .column(IS_BUILDING, 
manager.isIndexBuilding(indexName))
                                .column(IS_STRING, context.isLiteral())
-                               .column(ANALYZER, 
context.getIndexAnalyzerFactory().toString());
+                               .column(ANALYZER, 
context.getAnalyzerFactory().toString());
                     }
                 }
             }
diff --git 
a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java 
b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
index 457876ab24..9786a86ae2 100644
--- 
a/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
+++ 
b/src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
@@ -22,16 +22,29 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
-import org.apache.cassandra.concurrent.ImmediateExecutor;
-import org.tartarus.snowball.SnowballStemmer;
-import org.tartarus.snowball.ext.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.github.benmanes.caffeine.cache.CacheLoader;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.ImmediateExecutor;
+import org.tartarus.snowball.SnowballStemmer;
+import org.tartarus.snowball.ext.DanishStemmer;
+import org.tartarus.snowball.ext.DutchStemmer;
+import org.tartarus.snowball.ext.EnglishStemmer;
+import org.tartarus.snowball.ext.FinnishStemmer;
+import org.tartarus.snowball.ext.FrenchStemmer;
+import org.tartarus.snowball.ext.GermanStemmer;
+import org.tartarus.snowball.ext.HungarianStemmer;
+import org.tartarus.snowball.ext.ItalianStemmer;
+import org.tartarus.snowball.ext.NorwegianStemmer;
+import org.tartarus.snowball.ext.PortugueseStemmer;
+import org.tartarus.snowball.ext.RomanianStemmer;
+import org.tartarus.snowball.ext.RussianStemmer;
+import org.tartarus.snowball.ext.SpanishStemmer;
+import org.tartarus.snowball.ext.SwedishStemmer;
+import org.tartarus.snowball.ext.TurkishStemmer;
 
 /**
  * Returns a SnowballStemmer instance appropriate for
@@ -63,21 +76,21 @@ public class StemmerFactory
     static
     {
         SUPPORTED_LANGUAGES = new HashMap<>();
-        SUPPORTED_LANGUAGES.put("de", germanStemmer.class);
-        SUPPORTED_LANGUAGES.put("da", danishStemmer.class);
-        SUPPORTED_LANGUAGES.put("es", spanishStemmer.class);
-        SUPPORTED_LANGUAGES.put("en", englishStemmer.class);
-        SUPPORTED_LANGUAGES.put("fl", finnishStemmer.class);
-        SUPPORTED_LANGUAGES.put("fr", frenchStemmer.class);
-        SUPPORTED_LANGUAGES.put("hu", hungarianStemmer.class);
-        SUPPORTED_LANGUAGES.put("it", italianStemmer.class);
-        SUPPORTED_LANGUAGES.put("nl", dutchStemmer.class);
-        SUPPORTED_LANGUAGES.put("no", norwegianStemmer.class);
-        SUPPORTED_LANGUAGES.put("pt", portugueseStemmer.class);
-        SUPPORTED_LANGUAGES.put("ro", romanianStemmer.class);
-        SUPPORTED_LANGUAGES.put("ru", russianStemmer.class);
-        SUPPORTED_LANGUAGES.put("sv", swedishStemmer.class);
-        SUPPORTED_LANGUAGES.put("tr", turkishStemmer.class);
+        SUPPORTED_LANGUAGES.put("de", GermanStemmer.class);
+        SUPPORTED_LANGUAGES.put("da", DanishStemmer.class);
+        SUPPORTED_LANGUAGES.put("es", SpanishStemmer.class);
+        SUPPORTED_LANGUAGES.put("en", EnglishStemmer.class);
+        SUPPORTED_LANGUAGES.put("fl", FinnishStemmer.class);
+        SUPPORTED_LANGUAGES.put("fr", FrenchStemmer.class);
+        SUPPORTED_LANGUAGES.put("hu", HungarianStemmer.class);
+        SUPPORTED_LANGUAGES.put("it", ItalianStemmer.class);
+        SUPPORTED_LANGUAGES.put("nl", DutchStemmer.class);
+        SUPPORTED_LANGUAGES.put("no", NorwegianStemmer.class);
+        SUPPORTED_LANGUAGES.put("pt", PortugueseStemmer.class);
+        SUPPORTED_LANGUAGES.put("ro", RomanianStemmer.class);
+        SUPPORTED_LANGUAGES.put("ru", RussianStemmer.class);
+        SUPPORTED_LANGUAGES.put("sv", SwedishStemmer.class);
+        SUPPORTED_LANGUAGES.put("tr", TurkishStemmer.class);
     }
 
     public static SnowballStemmer getStemmer(Locale locale)
diff --git 
a/test/distributed/org/apache/cassandra/distributed/test/sai/ReplicaFilteringProtectionTest.java
 
b/test/distributed/org/apache/cassandra/distributed/test/sai/ReplicaFilteringProtectionTest.java
new file mode 100644
index 0000000000..95ed4c7946
--- /dev/null
+++ 
b/test/distributed/org/apache/cassandra/distributed/test/sai/ReplicaFilteringProtectionTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class ReplicaFilteringProtectionTest extends TestBaseImpl
+{
+    private static final int REPLICAS = 2;
+
+    @Test
+    public void testRFPWithIndexTransformations() throws IOException
+    {
+        try (Cluster cluster = init(Cluster.build()
+                                           .withNodes(REPLICAS)
+                                           .withConfig(config -> 
config.set("hinted_handoff_enabled", false)
+                                                                       
.set("commitlog_sync", "batch")).start()))
+        {
+            String tableName = "sai_rfp";
+            String fullTableName = KEYSPACE + '.' + tableName;
+
+            cluster.schemaChange("CREATE TABLE " + fullTableName + " (k int 
PRIMARY KEY, v text)");
+            cluster.schemaChange("CREATE CUSTOM INDEX ON " + fullTableName + 
"(v) USING 'StorageAttachedIndex' " +
+                                 "WITH OPTIONS = { 'case_sensitive' : false}");
+
+            // both nodes have the old value
+            cluster.coordinator(1).execute("INSERT INTO " + fullTableName + 
"(k, v) VALUES (0, 'OLD')", ALL);
+
+            String select = "SELECT * FROM " + fullTableName + " WHERE v = 
'old'";
+            Object[][] initialRows = cluster.coordinator(1).execute(select, 
ALL);
+            assertRows(initialRows, row(0, "OLD"));
+
+            // only one node gets the new value
+            cluster.get(1).executeInternal("UPDATE " + fullTableName + " SET v 
= 'new' WHERE k = 0");
+
+            // querying by the old value shouldn't return the old surviving row
+            SimpleQueryResult oldResult = 
cluster.coordinator(1).executeWithResult(select, ALL);
+            assertRows(oldResult.toObjectArrays());
+        }
+    }
+}
diff --git 
a/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
 
b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
new file mode 100644
index 0000000000..fb73a98eae
--- /dev/null
+++ 
b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+
+public class NonTokenizingAnalyzerTest
+{
+    @Test
+    public void asciiAnalyzer() throws Exception
+    {
+        NonTokenizingOptions options = 
NonTokenizingOptions.getDefaultOptions();
+        options.setCaseSensitive(false);
+        options.setAscii(true);
+
+        assertEquals("eppinger", getAnalyzedString("Éppinger", options));
+    }
+
+    @Test
+    public void asciiAnalyzerFalse() throws Exception
+    {
+        NonTokenizingOptions options = 
NonTokenizingOptions.getDefaultOptions();
+        options.setCaseSensitive(true);
+        options.setAscii(false);
+
+        assertEquals("Éppinger", getAnalyzedString("Éppinger", options));
+    }
+
+    @Test
+    public void caseInsensitiveAnalyzer() throws Exception
+    {
+        NonTokenizingOptions options = 
NonTokenizingOptions.getDefaultOptions();
+        options.setCaseSensitive(false);
+
+        assertEquals("nip it in the bud", getAnalyzedString("Nip it in the 
bud", options));
+    }
+
+    @Test
+    public void caseSensitiveAnalyzer() throws Exception
+    {
+        NonTokenizingOptions options = 
NonTokenizingOptions.getDefaultOptions();
+
+        assertEquals("Nip it in the bud", getAnalyzedString("Nip it in the 
bud", options));
+    }
+
+    private String getAnalyzedString(String input, NonTokenizingOptions 
options) throws Exception
+    {
+        NonTokenizingAnalyzer analyzer = new 
NonTokenizingAnalyzer(UTF8Type.instance, options);
+        analyzer.reset(ByteBuffer.wrap(input.getBytes()));
+        return analyzer.hasNext() ? ByteBufferUtil.string(analyzer.next) : 
null;
+    }
+}
diff --git 
a/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicFiltersTest.java
 
b/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicFiltersTest.java
new file mode 100644
index 0000000000..01faf488c1
--- /dev/null
+++ 
b/test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicFiltersTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer.filter;
+
+import java.text.Normalizer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+
+import static org.junit.Assert.assertEquals;
+
+public class BasicFiltersTest
+{
+    @Test
+    public void testLowerCase()
+    {
+        BasicFilters.LowerCase lowerCase = new BasicFilters.LowerCase();
+        
+        for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 
1000); count++)
+        {
+            String actual = SAITester.getRandom().nextTextString(10, 50);
+            assertEquals(actual.toLowerCase(), lowerCase.process(actual));
+        }
+    }
+    
+    @Test
+    public void testNormalize()
+    {
+        BasicFilters.Normalize normalize = new BasicFilters.Normalize();
+
+        for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 
1000); count++)
+        {
+            String actual = SAITester.getRandom().nextTextString(10, 50);
+            assertEquals(Normalizer.normalize(actual, Normalizer.Form.NFC), 
normalize.process(actual));
+        }
+    }
+    
+    @Test
+    public void testAscii()
+    {
+        BasicFilters.Ascii ascii = new BasicFilters.Ascii();
+
+        for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 
1000); count++)
+        {
+            String actual = SAITester.getRandom().nextTextString(100, 5000);
+
+            char[] actualChars = actual.toCharArray();
+            char[] expectedChars = new char[actualChars.length * 4];
+            int expectedSize = ASCIIFoldingFilter.foldToASCII(actualChars, 0, 
expectedChars, 0, actualChars.length);
+            String expected = new String(expectedChars, 0, expectedSize);
+
+            assertEquals(expected, ascii.process(actual));
+        }
+    }
+}
diff --git 
a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java 
b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
index fab116101f..2b621fcfe7 100644
--- 
a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
+++ 
b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
@@ -264,6 +264,112 @@ public class StorageAttachedIndexDDLTest extends SAITester
         assertEquals(1, saiCreationCounter.get());
     }
 
+    @Test
+    public void shouldBeCaseSensitiveByDefault()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'Camel'").size());
+
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 
'camel'").size());
+    }
+
+    @Test
+    public void shouldEnableCaseSensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : true }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'Camel'").size());
+
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 
'camel'").size());
+    }
+
+    @Test
+    public void shouldEnableCaseInsensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : false }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'camel'").size());
+    }
+
+    @Test
+    public void shouldBeNonNormalizedByDefault()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'Cam\u00E1l'").size());
+
+        // Both \u00E1 and \u0061\u0301 are visible as the character á, but 
without NFC normalization, they won't match.
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 
'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNonNormalizedSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : false }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'Cam\u00E1l'").size());
+
+        // Both \u00E1 and \u0061\u0301 are visible as the character á, but 
without NFC normalization, they won't match.
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 
'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNormalizedSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true }");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableNormalizedCaseInsensitiveSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true, 'case_sensitive' : 
false}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldEnableAsciiSearch()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 
'StorageAttachedIndex' WITH OPTIONS = { 'ascii' : true, 'case_sensitive' : 
false}");
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Éppinger')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 
'eppinger'").size());
+    }
+
     @Test
     public void shouldCreateIndexOnReversedType() throws Throwable
     {
diff --git 
a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java 
b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
index a8f5bf9a58..19d4cbfdf1 100644
--- 
a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
+++ 
b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
@@ -129,6 +129,6 @@ public class IndexesSystemViewTest extends SAITester
                        isQueryable,
                        isBuilding,
                        isString,
-                       context.getIndexAnalyzerFactory().toString());
+                       context.getAnalyzerFactory().toString());
     }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org

Reply via email to