This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8368-d7ce3a6bff97a43a380ce71ff48f3bc016b261ae in repository https://gitbox.apache.org/repos/asf/texera.git
commit 4f36c15468317c18a72d9434d262f30c1e6c482c Author: Kary Zheng <[email protected]> AuthorDate: Sat Oct 3 07:42:13 2026 +0000 feat(operator): export the estimators and the Hugging Face models (#8368) ### What changes were proposed in this PR? The scikit-learn estimators are fitted on one port and score on the other, so the script holds both frames and narrows each by the same rule: a fit and a score taken on different columns would compare two different models. The four Hugging Face models declare the column types they take, and the iris regression keeps the row when a petal measurement is empty rather than ending the run on it. Sklearn Prediction and Sklearn Testing each consume a fitted model on an input port. The verification fits one in Python and hands the same model to both paths, so they are run both ways like the rest. Review also turned up three defects that this PR fixes rather than mirrors. `SklearnTesting` squeezed the feature frame before predicting, so a single feature column, or the single row a drop left behind, reached scikit-learn as one dimension and was rejected. That was the executor's own behaviour and the export had copied it, so both paths pass the frame itself now. Separately, the three HuggingFace text operators guarded a missing cell with `is None`. Arrow normalizes an empty cell to null, so that held for the executor as long as the column really was a string, but `attributeTypeRules` is a warning the editor prints rather than a filter, and the exported script reads a frame where a column holding nothing else comes back as float64. Both paths ask pandas now. Third, the iris regression's engine code checked a missing measurement with `is None`, so a NaN went on to the model and got a prediction the export leaves empty. It now asks `pd.isna` too, as the export does. ### Any related issues, documentation, discussions? Part of #8325, 20 of 27; that issue lists the set in order. It needs #8327 for the trait, so it does not compile until that lands, and the rows these operators add to the verification runner follow with the harness rather than as whole new files here. Closes #8316, closes #8056. Closes #8422, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. Once the harness lands, each is also run through the engine and through its generated script, on every configuration its schema offers, and the two answers compared. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]> --- .../HuggingFaceIrisLogisticRegressionOpDesc.scala | 81 +++++++++++++++++++++- .../HuggingFaceSentimentAnalysisOpDesc.scala | 74 +++++++++++++++++--- .../HuggingFaceSpamSMSDetectionOpDesc.scala | 61 +++++++++++++--- .../HuggingFaceTextSummarizationOpDesc.scala | 65 ++++++++++++++--- .../operator/sklearn/SklearnClassifierOpDesc.scala | 50 ++++++++++++- .../sklearn/SklearnLinearRegressionOpDesc.scala | 52 +++++++++++++- .../operator/sklearn/SklearnModelOpDesc.scala | 33 ++++++++- .../operator/sklearn/SklearnPredictionOpDesc.scala | 73 ++++++++++++++++++- .../sklearn/testing/SklearnTestingOpDesc.scala | 56 +++++++++++++-- .../sklearn/training/SklearnTrainingOpDesc.scala | 30 +++++++- ...ggingFaceIrisLogisticRegressionOpDescSpec.scala | 26 +++++++ .../HuggingFaceSentimentAnalysisOpDescSpec.scala | 33 ++++++++- .../HuggingFaceSpamSMSDetectionOpDescSpec.scala | 33 ++++++++- .../HuggingFaceTextSummarizationOpDescSpec.scala | 33 ++++++++- .../SklearnBernoulliNaiveBayesOpDescSpec.scala | 15 ++++ .../SklearnLinearRegressionOpDescSpec.scala | 27 ++++++++ .../operator/sklearn/SklearnModelOpDescSpec.scala | 28 ++++++++ .../sklearn/SklearnPredictionOpDescSpec.scala | 68 +++++++++++++++++- .../sklearn/testing/SklearnTestingOpDescSpec.scala | 21 ++++-- 19 files changed, 807 insertions(+), 52 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala index 9a9ac56325..3cac29dd53 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDesc.scala @@ -20,24 +20,40 @@ package org.apache.texera.amber.operator.huggingFace import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -class HuggingFaceIrisLogisticRegressionOpDesc extends PythonOperatorDescriptor { +// type constraint: both measurements are standardized against the training means +// and handed to the model as floats, so each column can only be numeric. +@JsonSchemaInject(json = """ +{ + "attributeTypeRules": { + "petalLengthCmAttribute": { "enum": ["integer", "long", "double"] }, + "petalWidthCmAttribute": { "enum": ["integer", "long", "double"] } + } +} +""") +class HuggingFaceIrisLogisticRegressionOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator { @JsonProperty(value = "petalLengthCmAttribute", required = true) @JsonPropertyDescription("attribute in your dataset corresponding to PetalLengthCm") @AutofillAttributeName + @SampleColumn("petal_length") var petalLengthCmAttribute: EncodableString = _ @JsonProperty(value = "petalWidthCmAttribute", required = true) @JsonPropertyDescription("attribute in your dataset corresponding to PetalWidthCm") @AutofillAttributeName + @SampleColumn("petal_width") var petalWidthCmAttribute: EncodableString = _ @JsonProperty( @@ -67,6 +83,7 @@ class HuggingFaceIrisLogisticRegressionOpDesc extends PythonOperatorDescriptor { override def generatePythonCode(): String = { pyb"""from pytexera import * |import numpy as np + |import pandas as pd |import torch |import torch.nn as nn |from huggingface_hub import PyTorchModelHubMixin @@ -91,6 +108,16 @@ class HuggingFaceIrisLogisticRegressionOpDesc extends PythonOperatorDescriptor { | training_features_stds = [1.72528903, 0.73788937] | length = tuple_[$petalLengthCmAttribute] | width = tuple_[$petalWidthCmAttribute] + | # An empty cell arrives as None, which numpy carries as an object the + | # standardization cannot subtract from, and a NaN is no measurement + | # either. Keep the row and leave the prediction empty rather than + | # ending the run or predicting from a measurement the model was + | # never given, as the exported script does. + | if pd.isna(length) or pd.isna(width): + | tuple_[$predictionClassName] = None + | tuple_[$predictionProbabilityName] = None + | yield tuple_ + | return | features = np.array([[length, width]]) | features = ((features - training_features_means) / training_features_stds) | features = torch.from_numpy(features).float() @@ -103,6 +130,54 @@ class HuggingFaceIrisLogisticRegressionOpDesc extends PythonOperatorDescriptor { | yield tuple_""".encode } + // Standalone mirror of generatePythonCode: rebuild+load the pretrained linear + // model once, then apply the same per-row standardize→sigmoid→threshold logic, + // adding the STRING predicted class and DOUBLE probability columns (in + // getOutputSchemas order) to produce out1df. + override def generateStandaloneCode(): String = { + val lengthLit = pyStringLiteral(petalLengthCmAttribute) + val widthLit = pyStringLiteral(petalWidthCmAttribute) + s"""import numpy as np + |import torch + |import torch.nn as nn + |from huggingface_hub import PyTorchModelHubMixin + | + |class LinearModel(nn.Module, PyTorchModelHubMixin): + | def __init__(self): + | super().__init__() + | self.fc = nn.Linear(2, 1) + | + | def forward(self, x): + | return self.fc(x) + | + |model = LinearModel.from_pretrained("sadhaklal/logistic-regression-iris") + |model.eval() + | + |training_features_means = [3.72666667, 1.17619048] + |training_features_stds = [1.72528903, 0.73788937] + |out1df = in1df.copy() + |_classes = [] + |_probs = [] + |for _length, _width in zip(out1df[$lengthLit], out1df[$widthLit]): + | # The operator's guard: pandas answers for the NaN a frame read from + | # JSON holds and for the None a Tuple hands over. + | if pd.isna(_length) or pd.isna(_width): + | _classes.append(None) + | _probs.append(None) + | continue + | features = np.array([[_length, _width]]) + | features = ((features - training_features_means) / training_features_stds) + | features = torch.from_numpy(features).float() + | with torch.no_grad(): + | logits = model(features) + | proba = torch.sigmoid(logits.squeeze()) + | preds = (proba > 0.5).long() + | _probs.append(float(proba)) + | _classes.append("Iris-setosa" if preds == 1 else "Not Iris-setosa") + |out1df[${pyStringLiteral(predictionClassName)}] = _classes + |out1df[${pyStringLiteral(predictionProbabilityName)}] = _probs""".stripMargin + } + override def operatorInfo: OperatorInfo = OperatorInfo( "Hugging Face Iris Logistic Regression", diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala index 0a8fb8a894..0467d1e0b3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala @@ -20,17 +20,33 @@ package org.apache.texera.amber.operator.huggingFace import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext -class HuggingFaceSentimentAnalysisOpDesc extends PythonOperatorDescriptor { +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} +// type constraint: the tokenizer scores text and refuses anything that is not a +// string, so the column can only be a string. +@JsonSchemaInject(json = """ +{ + "attributeTypeRules": { + "attribute": { "enum": ["string"] } + } +} +""") +class HuggingFaceSentimentAnalysisOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator { @JsonProperty(value = "attribute", required = true) @JsonPropertyDescription("column to perform sentiment analysis on") @AutofillAttributeName + @SampleColumn("short_text") var attribute: EncodableString = _ @JsonProperty( @@ -63,6 +79,7 @@ class HuggingFaceSentimentAnalysisOpDesc extends PythonOperatorDescriptor { |from transformers import AutoModelForSequenceClassification |from transformers import AutoTokenizer, AutoConfig |import numpy as np + |import pandas as pd |from scipy.special import softmax | |class ProcessTupleOperator(UDFOperatorV2): @@ -76,10 +93,11 @@ class HuggingFaceSentimentAnalysisOpDesc extends PythonOperatorDescriptor { | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: | text = tuple_[$attribute] - | # An empty cell arrives as None, which the tokenizer rejects. Keep the row - | # and leave the scores empty rather than ending the run over a value the - | # model has nothing to say about. - | if text is None or (isinstance(text, str) and not text.strip()): + | # An empty cell arrives as None, and a column the type rule was meant to + | # keep out can carry a NaN of its own. The tokenizer rejects both. Keep + | # the row and leave the scores empty rather than ending the run over a + | # value the model has nothing to say about. + | if pd.isna(text) or (isinstance(text, str) and not text.strip()): | for label in ($resultAttributePositive, $resultAttributeNeutral, $resultAttributeNegative): | tuple_[label] = None | yield tuple_ @@ -96,6 +114,46 @@ class HuggingFaceSentimentAnalysisOpDesc extends PythonOperatorDescriptor { | yield tuple_""".encode } + // Standalone mirror of generatePythonCode: load the model once, then apply the + // same per-row softmax-over-3-labels logic to in1df, adding the three DOUBLE + // result columns (in the same order as getOutputSchemas) to produce out1df. + override def generateStandaloneCode(): String = { + val positiveLit = pyStringLiteral(resultAttributePositive) + val neutralLit = pyStringLiteral(resultAttributeNeutral) + val negativeLit = pyStringLiteral(resultAttributeNegative) + s"""from transformers import AutoModelForSequenceClassification + |from transformers import AutoTokenizer, AutoConfig + |import numpy as np + |import pandas as pd + |from scipy.special import softmax + | + |model_name = "cardiffnlp/twitter-roberta-base-sentiment-latest" + |tokenizer = AutoTokenizer.from_pretrained(model_name) + |config = AutoConfig.from_pretrained(model_name) + |model = AutoModelForSequenceClassification.from_pretrained(model_name) + | + |out1df = in1df.copy() + |labels = {"positive": $positiveLit, "neutral": $neutralLit, "negative": $negativeLit} + |for _col in ($positiveLit, $neutralLit, $negativeLit): + | out1df[_col] = 0.0 + |for _idx, _text in out1df[${pyStringLiteral(attribute)}].items(): + | # An empty cell reaches the frame as None, or as NaN once a column holds + | # nothing else, and the tokenizer rejects both. Keep the row and leave the + | # scores empty rather than ending the run over a value the model has + | # nothing to say about. + | if pd.isna(_text) or (isinstance(_text, str) and not _text.strip()): + | for _col in ($positiveLit, $neutralLit, $negativeLit): + | out1df.at[_idx, _col] = None + | continue + | encoded_input = tokenizer(_text, return_tensors='pt') + | output = model(**encoded_input) + | scores = softmax(output[0][0].detach().numpy()) + | ranking = np.argsort(scores)[::-1] + | for i in range(scores.shape[0]): + | label = labels[config.id2label[ranking[i]]] + | out1df.at[_idx, label] = np.round(float(scores[ranking[i]]), 4)""".stripMargin + } + override def operatorInfo: OperatorInfo = OperatorInfo( "Hugging Face Sentiment Analysis", diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala index 5ac3c03858..e305a354cd 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDesc.scala @@ -20,17 +20,33 @@ package org.apache.texera.amber.operator.huggingFace import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext -class HuggingFaceSpamSMSDetectionOpDesc extends PythonOperatorDescriptor { +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} +// type constraint: the classification pipeline reads text and refuses anything +// that is not a string, so the column can only be a string. +@JsonSchemaInject(json = """ +{ + "attributeTypeRules": { + "attribute": { "enum": ["string"] } + } +} +""") +class HuggingFaceSpamSMSDetectionOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator { @JsonProperty(value = "attribute", required = true) @JsonPropertyDescription("column to perform spam detection on") @AutofillAttributeName + @SampleColumn("short_text") var attribute: EncodableString = _ @JsonProperty( @@ -51,6 +67,7 @@ class HuggingFaceSpamSMSDetectionOpDesc extends PythonOperatorDescriptor { override def generatePythonCode(): String = { pyb"""from transformers import pipeline + |import pandas as pd |from pytexera import * | |class ProcessTupleOperator(UDFOperatorV2): @@ -61,10 +78,11 @@ class HuggingFaceSpamSMSDetectionOpDesc extends PythonOperatorDescriptor { | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: | text = tuple_[$attribute] - | # An empty cell arrives as None, which the pipeline rejects. Keep the row - | # and leave the results empty rather than ending the run over a value the - | # model has nothing to say about. - | if text is None or (isinstance(text, str) and not text.strip()): + | # An empty cell arrives as None, and a column the type rule was meant to + | # keep out can carry a NaN of its own. The pipeline rejects both. Keep + | # the row and leave the results empty rather than ending the run over a + | # value the model has nothing to say about. + | if pd.isna(text) or (isinstance(text, str) and not text.strip()): | tuple_[$resultAttributeSpam] = None | tuple_[$resultAttributeProbability] = None | yield tuple_ @@ -75,6 +93,33 @@ class HuggingFaceSpamSMSDetectionOpDesc extends PythonOperatorDescriptor { | yield tuple_""".encode } + // Standalone mirror of generatePythonCode: build the text-classification + // pipeline once, run it per row, and add the BOOLEAN spam flag (LABEL_1) and + // the DOUBLE score columns (in getOutputSchemas order) to produce out1df. + override def generateStandaloneCode(): String = { + val attributeLit = pyStringLiteral(attribute) + val spamLit = pyStringLiteral(resultAttributeSpam) + val probabilityLit = pyStringLiteral(resultAttributeProbability) + s"""from transformers import pipeline + |import pandas as pd + | + |_pipeline = pipeline("text-classification", model="mrm8488/bert-tiny-finetuned-sms-spam-detection") + |out1df = in1df.copy() + | + |def _classify(_t): + | # An empty cell reaches the frame as None, or as NaN once a column holds + | # nothing else, and the pipeline rejects both. Keep the row and leave the + | # results empty rather than ending the run over a value the model has + | # nothing to say about. + | if pd.isna(_t) or (isinstance(_t, str) and not _t.strip()): + | return None + | return _pipeline(_t)[0] + | + |_results = [_classify(_t) for _t in out1df[$attributeLit]] + |out1df[$spamLit] = [None if _r is None else _r["label"] == "LABEL_1" for _r in _results] + |out1df[$probabilityLit] = [None if _r is None else _r["score"] for _r in _results]""".stripMargin + } + override def operatorInfo: OperatorInfo = OperatorInfo( "Hugging Face Spam Detection", diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala index 541bcf46ac..50589907e2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDesc.scala @@ -20,17 +20,33 @@ package org.apache.texera.amber.operator.huggingFace import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext -class HuggingFaceTextSummarizationOpDesc extends PythonOperatorDescriptor { +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} +// type constraint: the tokenizer summarizes text and refuses anything that is not +// a string, so the column can only be a string. +@JsonSchemaInject(json = """ +{ + "attributeTypeRules": { + "attribute": { "enum": ["string"] } + } +} +""") +class HuggingFaceTextSummarizationOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator { @JsonProperty(value = "attribute", required = true) @JsonPropertyDescription("attribute to perform text summarization on") @AutofillAttributeName + @SampleColumn("long_text") var attribute: EncodableString = _ @JsonProperty( @@ -44,6 +60,7 @@ class HuggingFaceTextSummarizationOpDesc extends PythonOperatorDescriptor { override def generatePythonCode(): String = { pyb""" |from transformers import BertTokenizerFast, EncoderDecoderModel + |import pandas as pd |import torch |from pytexera import * | @@ -58,10 +75,11 @@ class HuggingFaceTextSummarizationOpDesc extends PythonOperatorDescriptor { | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: | text = tuple_[$attribute] - | # An empty cell arrives as None, which the tokenizer rejects. Keep the row - | # and leave the summary empty rather than ending the run over a value the - | # model has nothing to say about. - | if text is None or (isinstance(text, str) and not text.strip()): + | # An empty cell arrives as None, and a column the type rule was meant to + | # keep out can carry a NaN of its own. The tokenizer rejects both. Keep + | # the row and leave the summary empty rather than ending the run over a + | # value the model has nothing to say about. + | if pd.isna(text) or (isinstance(text, str) and not text.strip()): | tuple_[$resultAttribute] = None | yield tuple_ | return @@ -76,6 +94,37 @@ class HuggingFaceTextSummarizationOpDesc extends PythonOperatorDescriptor { | yield tuple_""".encode } + // Standalone mirror of generatePythonCode: load the encoder-decoder model + // once, generate a summary per row, and add the STRING result column to + // produce out1df. + override def generateStandaloneCode(): String = { + s"""from transformers import BertTokenizerFast, EncoderDecoderModel + |import pandas as pd + |import torch + | + |model_name = "mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization" + |device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + |tokenizer = BertTokenizerFast.from_pretrained(model_name) + |model = EncoderDecoderModel.from_pretrained(model_name).to(device) + | + |out1df = in1df.copy() + |_summaries = [] + |for _text in out1df[${pyStringLiteral(attribute)}]: + | # An empty cell reaches the frame as None, or as NaN once a column holds + | # nothing else, and the tokenizer rejects both. Keep the row and leave the + | # summary empty rather than ending the run over a value the model has + | # nothing to say about. + | if pd.isna(_text) or (isinstance(_text, str) and not _text.strip()): + | _summaries.append(None) + | continue + | inputs = tokenizer([_text], padding="max_length", truncation=True, max_length=512, return_tensors="pt") + | input_ids = inputs.input_ids.to(device) + | attention_mask = inputs.attention_mask.to(device) + | output = model.generate(input_ids, attention_mask=attention_mask) + | _summaries.append(tokenizer.decode(output[0], skip_special_tokens=True)) + |out1df[${pyStringLiteral(resultAttribute)}] = _summaries""".stripMargin + } + override def operatorInfo: OperatorInfo = OperatorInfo( "Hugging Face Text Summarization", diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala index a2323e5d19..00657c905f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala @@ -19,11 +19,13 @@ package org.apache.texera.amber.operator.sklearn +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc { +abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc with StandaloneCodeGenerator { override def getImportStatements = "" @@ -76,4 +78,50 @@ $reportMissingKept ), outputPorts = List(OutputPort(blocking = true)) ) + + override def generateStandaloneCode(): String = { + val estimator = getImportStatements.split(" ").last + val tfidfPart = if (tfidfTransformer) "TfidfTransformer()," else "" + val targetLit = pyStringLiteral(target) + val modelNameLit = pyStringLiteral(getUserFriendlyModelName) + val narrowTrain = dropNonFeatureColumns("X_train", "") + val narrowTest = dropNonFeatureColumns("X_test", "") + + s"""${getImportStatements} + |from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score + |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer + |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer + |import numpy as np + |import pandas as pd + | + |# The same rows the operator drops, and on both frames: the model is fitted + |# on one and scored on the other, so narrowing only one side would fit and + |# score on different data. A local name rather than a reassignment, since + |# the input variable belongs to whichever operator produced it. + |_train = ${dropMissingRowsStandalone("in1df")} + |if len(_train) < len(in1df): + | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") + |Y_train = _train[$targetLit] + |X_train = _train.drop($targetLit, axis=1) + |$narrowTrain + |model = make_pipeline(${vectorizerStage(c => pyStringLiteral(c))}$tfidfPart$estimator()).fit(X_train, Y_train) + | + |_test = ${dropMissingRowsStandalone("in2df")} + |# The operator runs once per port, so it says this for the table it scores + |# as well as for the one it fits. + |if len(_test) < len(in2df): + | print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values") + |Y_test = _test[$targetLit] + |X_test = _test.drop($targetLit, axis=1) + |$narrowTest + |predictions = model.predict(X_test) + |print("Overall Accuracy:", round(accuracy_score(Y_test, predictions), 4)) + |f1s = f1_score(Y_test, predictions, average=None) + |precisions = precision_score(Y_test, predictions, average=None) + |recalls = recall_score(Y_test, predictions, average=None) + |for i, class_name in enumerate(np.unique(Y_test)): + | print("Class", repr(class_name), " - F1:", round(f1s[i], 4), ", Precision:", round(precisions[i], 4), ", Recall:", round(recalls[i], 4)) + |out1df = pd.DataFrame([{"model_name": $modelNameLit, "model": model}])""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala index 9ea793338e..948b2b3061 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDesc.scala @@ -25,16 +25,23 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -class SklearnLinearRegressionOpDesc extends PythonOperatorDescriptor with SklearnFittableColumns { +class SklearnLinearRegressionOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator + with SklearnFittableColumns { @JsonSchemaTitle("Target Attribute") @JsonPropertyDescription("Attribute in your dataset corresponding to target.") @JsonProperty(required = true) @AutofillAttributeName + // The label the estimator fits against. Test-only steering: without it the + // first column wins, which on a feature/label table is a feature. + @SampleColumn("a\"b\\c_species") var target: EncodableString = _ @JsonSchemaTitle("Degree") @@ -95,4 +102,43 @@ class SklearnLinearRegressionOpDesc extends PythonOperatorDescriptor with Sklear ) } + override def generateStandaloneCode(): String = { + val targetLit = pyStringLiteral(target) + s"""from sklearn.metrics import mean_absolute_error, r2_score + |from sklearn.pipeline import make_pipeline + |from sklearn.linear_model import LinearRegression + |from sklearn.preprocessing import PolynomialFeatures + |import pandas as pd + | + |# The same rows the operator drops, and on both frames: the model is fitted + |# on one and scored on the other, so narrowing only one side would fit and + |# score on different data. Local names rather than reassignments, since the + |# input variables belong to whichever operators produced them. + |_train = in1df.dropna() + |if len(_train) < len(in1df): + | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") + |Y_train = _train[$targetLit] + |X_train = _train.drop($targetLit, axis=1) + |${narrowToFittableColumns("X_train", "")} + |pipeline = make_pipeline( + | PolynomialFeatures(degree=$degree), + | LinearRegression() + |) + |model = pipeline.fit(X_train, Y_train) + | + |_test = in2df.dropna() + |# The operator runs once per port, so it says this for the table it scores + |# as well as for the one it fits. + |if len(_test) < len(in2df): + | print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values") + |Y_test = _test[$targetLit] + |X_test = _test.drop($targetLit, axis=1) + |${narrowToFittableColumns("X_test", "")} + |predictions = model.predict(X_test) + |mae = round(mean_absolute_error(Y_test, predictions), 4) + |r2 = round(r2_score(Y_test, predictions), 4) + |print("MAE:", mae, ", R2:", r2) + |out1df = pd.DataFrame([{"model_name": "LinearRegression", "model": model}])""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala index c771d31aff..c7cd0e4355 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala @@ -34,12 +34,14 @@ import com.kjetland.jackson.jsonSchema.annotations.{ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, CommonOpDescAnnotation, - HideAnnotation + HideAnnotation, + SampleColumn } // `text` names the columns Count Vectorizer tokenizes, so they are string columns @@ -72,6 +74,9 @@ import org.apache.texera.amber.operator.metadata.annotations.{ """) abstract class SklearnModelOpDesc extends PythonOperatorDescriptor with SklearnFittableColumns { + // The label the estimator fits against. Test-only steering: without it the + // first column wins, which on a feature/label table is a feature. + @SampleColumn("a\"b\\c_species") @JsonSchemaTitle("Target Attribute") @JsonPropertyDescription("Attribute in your dataset corresponding to target.") @JsonProperty(required = true) @@ -182,6 +187,22 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor with SklearnF else if (handlesMissingValues) pyb"table.dropna(subset=[$target])".toString else "table.dropna()" + /** [[dropMissingRows]] for the standalone path, which names its own frame and + * has no `self` to decode a column name through. Stated separately rather than + * shared through a renderer: `pyb` encodes by the STATIC type of what it + * interpolates, so a column handed through a `String => String` loses the + * annotation that makes it encode and reaches the script raw. + */ + @JsonIgnore + protected def dropMissingRowsStandalone(frame: String): String = { + val subset = + if (countVectorizer) (text :+ target).map(c => pyStringLiteral(c).toString) + else if (handlesMissingValues) Seq(pyStringLiteral(target).toString) + else Seq.empty + if (subset.isEmpty) s"$frame.dropna()" + else subset.mkString(s"$frame.dropna(subset=[", ", ", "])") + } + // Rows the estimator keeps are still rows the user did not know were incomplete, // so say how many reached the fit. Empty for the estimators that dropped them all. @JsonIgnore @@ -202,6 +223,16 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor with SklearnF s" produces. Turn Count Vectorizer off, or use $alternatives." ) } + // The generated code drops the target before the text pipeline reads its + // columns, so naming it here asks the pipeline for a column that is no + // longer there. Refused rather than vectorized: the label is the answer, + // and a model given it as a feature reads that answer off its own input. + if (text.contains(target)) { + throw new RuntimeException( + s""""$target" is the Target Attribute, so it cannot also be a Text Attribute.""" + + " Remove it from Text Attribute, or fit against a different column." + ) + } } Map( operatorInfo.outputPorts.head.id -> Schema() diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala index 5d8b1b1e7d..c1d022974c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala @@ -24,14 +24,15 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameOnPort1 } import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -class SklearnPredictionOpDesc extends PythonOperatorDescriptor { +class SklearnPredictionOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonProperty(value = "Model Attribute", required = true, defaultValue = "model") @JsonPropertyDescription("attribute corresponding to ML model") @AutofillAttributeName @@ -99,4 +100,72 @@ class SklearnPredictionOpDesc extends PythonOperatorDescriptor { .add(resultAttribute, resultType) ) } + + /** Python that narrows `X` to the columns the model was fitted on. + * + * The fitting side leaves out the columns an estimator cannot fit, so this + * side has to leave out the same ones or scikit-learn refuses the frame for + * naming features it never saw. Read off the model rather than re-derived: + * what it was fitted on is a fact it carries, and asking it cannot drift from + * whatever rule the fitting operator applied. + */ + private val narrowToFittedFeatures: String = + """_fitted = getattr(model, "feature_names_in_", None) + |if _fitted is not None: + | X = X[list(_fitted)]""".stripMargin + + /** Python that predicts on the rows whose features are all present. + * + * The executor takes a row at a time and leaves the result empty where a + * feature is missing, so the row keeps its place. One call over the whole + * frame cannot do that: scikit-learn ends the run on the first missing value + * rather than leaving a gap. So predict on the complete rows and write those + * answers back where they came from. A frame with nothing missing is + * assigned in one piece, which keeps the type `predict` returned. + */ + private def predictLeavingIncompleteRowsEmpty(resultLit: String, asText: Boolean): String = { + val predicted = + if (asText) "[str(_p) for _p in model.predict(X[_complete])]" + else "model.predict(X[_complete]).tolist()" + s"""_complete = X.notna().all(axis=1) + |_predicted = $predicted if _complete.any() else [] + |if _complete.all(): + | out1df[$resultLit] = _predicted + |else: + | out1df[$resultLit] = None + | out1df.loc[_complete, $resultLit] = _predicted""".stripMargin + } + + /** Python that takes the model the executor predicts with. + * + * The executor keeps the model of every row the model port hands it, each one + * overwriting the last, so the model it predicts with is the one on the final + * row. Reading the first row instead would answer with a different model + * whenever that port carries more than one. With no data rows the executor + * never touches the model, so neither does this: both ports may be empty. + */ + private def takeTheModelTheExecutorKeeps(modelLit: String): String = + s"model = in1df[$modelLit].iloc[-1] if len(in2df) else None" + + override def generateStandaloneCode(): String = { + val modelLit = pyStringLiteral(model) + val resultLit = pyStringLiteral(resultAttribute) + if (groundTruthAttribute.nonEmpty) { + s"""from sklearn.pipeline import Pipeline + | + |${takeTheModelTheExecutorKeeps(modelLit)} + |out1df = in2df.copy() + |X = in2df.drop(${pyStringLiteral(groundTruthAttribute)}, axis=1) + |$narrowToFittedFeatures + |${predictLeavingIncompleteRowsEmpty(resultLit, asText = false)}""".stripMargin + } else { + s"""from sklearn.pipeline import Pipeline + | + |${takeTheModelTheExecutorKeeps(modelLit)} + |out1df = in2df.copy() + |X = in2df + |$narrowToFittedFeatures + |${predictLeavingIncompleteRowsEmpty(resultLit, asText = true)}""".stripMargin + } + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala index 94319f0936..5786f041d9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.sklearn.SklearnFittableColumns import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, @@ -31,9 +31,15 @@ import org.apache.texera.amber.operator.metadata.annotations.{ } import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} -class SklearnTestingOpDesc extends PythonOperatorDescriptor with SklearnFittableColumns { +class SklearnTestingOpDesc + extends PythonOperatorDescriptor + with StandaloneCodeGenerator + with SklearnFittableColumns { @JsonProperty(required = true, defaultValue = "false") @JsonSchemaTitle("Regression") @JsonPropertyDescription( @@ -76,7 +82,10 @@ class SklearnTestingOpDesc extends PythonOperatorDescriptor with SklearnFittable | Y = table[$target] | X = table.drop($target, axis=1) |${narrowToFittableColumns("X", " " * 12)} - | predictions = model.predict(X.squeeze()) + | #predict wants (n_samples, n_features). Squeezing the frame first + | #collapsed a single feature, or a single surviving row, to one + | #dimension, which scikit-learn rejects. + | predictions = model.predict(X) | if $isRegressionStr: | tuple_["R2"] = r2_score(Y, predictions) | tuple_["RMSE"] = root_mean_squared_error(Y, predictions) @@ -118,4 +127,43 @@ class SklearnTestingOpDesc extends PythonOperatorDescriptor with SklearnFittable _.add(_, AttributeType.DOUBLE) ) ) + + override def generateStandaloneCode(): String = { + val isRegressionStr = if (isRegression) "True" else "False" + val modelLit = pyStringLiteral(model) + val targetLit = pyStringLiteral(target) + s"""from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, root_mean_squared_error, mean_absolute_error, r2_score + | + |out1df = in1df.copy() + |# The same drop the executor makes before it scores: the model arrives + |# already fitted, so this operator cannot ask which estimator it holds and + |# drops on every column to be safe. Scoring the rows it skips would answer + |# with a number the run never reported, or fail inside scikit-learn. + |rows_read = len(in2df) + |scored_df = in2df.dropna() + |if len(scored_df) < rows_read: + | print("Skipped", rows_read - len(scored_df), "of", rows_read, "rows with missing values") + |Y = scored_df[$targetLit] + |X = scored_df.drop($targetLit, axis=1) + |${narrowToFittableColumns("X", "")} + |# predict wants (n_samples, n_features). Squeezing the frame first collapsed + |# a single feature, or a single surviving row, to one dimension, which + |# scikit-learn rejects. + |# + |# The model port carries one model per row and the executor scores each of + |# them on its own, so a row has to be given the score of the model it holds. + |# One score written down the column would report the first model's answer + |# for all of them. + |predictions = [_model.predict(X) for _model in in1df[$modelLit]] + |if $isRegressionStr: + | out1df["R2"] = [r2_score(Y, _p) for _p in predictions] + | out1df["RMSE"] = [root_mean_squared_error(Y, _p) for _p in predictions] + | out1df["MAE"] = [mean_absolute_error(Y, _p) for _p in predictions] + |else: + | out1df["accuracy"] = [round(accuracy_score(Y, _p), 4) for _p in predictions] + | out1df["f1"] = [f1_score(Y, _p, average="weighted") for _p in predictions] + | out1df["precision"] = [precision_score(Y, _p, average="weighted") for _p in predictions] + | out1df["recall"] = [recall_score(Y, _p, average="weighted") for _p in predictions]""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala index c51f322a62..9dab7c27b5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala @@ -19,12 +19,14 @@ package org.apache.texera.amber.operator.sklearn.training +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.sklearn.SklearnModelOpDesc -class SklearnTrainingOpDesc extends SklearnModelOpDesc { +class SklearnTrainingOpDesc extends SklearnModelOpDesc with StandaloneCodeGenerator { override def getImportStatements = "" @@ -64,4 +66,30 @@ $reportMissingKept inputPorts = List(InputPort(PortIdentity(), "training")), outputPorts = List(OutputPort(blocking = true)) ) + + override def generateStandaloneCode(): String = { + val estimator = getImportStatements.split(" ").last + val tfidfPart = if (tfidfTransformer) "TfidfTransformer()," else "" + val targetLit = pyStringLiteral(target) + val modelNameLit = pyStringLiteral(getUserFriendlyModelName) + val narrowX = dropNonFeatureColumns("X", "") + + s"""${getImportStatements} + |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer + |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer + |import pandas as pd + | + |# The same rows the operator drops. A local name rather than a + |# reassignment, since the input variable belongs to whichever operator + |# produced it. + |_train = ${dropMissingRowsStandalone("in1df")} + |if len(_train) < len(in1df): + | print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values") + |Y = _train[$targetLit] + |X = _train.drop($targetLit, axis=1) + |$narrowX + |model = make_pipeline(${vectorizerStage(c => pyStringLiteral(c))}$tfidfPart$estimator()).fit(X, Y) + |out1df = pd.DataFrame([{"model_name": $modelNameLit, "model": model}])""".stripMargin + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDescSpec.scala index 1c3701b4d6..ef567c669a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceIrisLogisticRegressionOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.huggingFace +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -105,6 +106,17 @@ class HuggingFaceIrisLogisticRegressionOpDescSpec extends AnyFlatSpec with Match carries(code, "species") shouldBe true } + // An empty cell reaches the standardization as a None, which numpy cannot + // subtract from, and a NaN is no measurement either, so the row is answered + // rather than ending the run or predicting from it, as the export does. + it should "leave the prediction empty when a measurement is missing, NaN included" in { + val d = configured() + val code = d.generatePythonCode() + code should include("if pd.isna(length) or pd.isna(width):") + code should include("yield tuple_") + code should include("return") + } + "HuggingFaceIrisLogisticRegressionOpDesc.getPhysicalOp" should "wire an OpExecWithCode python executor carrying the operator's ports" in { val d = configured() @@ -128,4 +140,18 @@ class HuggingFaceIrisLogisticRegressionOpDescSpec extends AnyFlatSpec with Match h.predictionClassName shouldBe "species" h.predictionProbabilityName shouldBe "probability" } + + "HuggingFaceIrisLogisticRegressionOpDesc (class-level)" should + "carry @JsonSchemaInject restricting both petal columns to numeric attributes" in { + val ann = + classOf[HuggingFaceIrisLogisticRegressionOpDesc].getAnnotation(classOf[JsonSchemaInject]) + ann should not be null + val payload = ann.json + payload should include("attributeTypeRules") + payload should include("petalLengthCmAttribute") + payload should include("petalWidthCmAttribute") + payload should include("integer") + payload should include("long") + payload should include("double") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDescSpec.scala index 65401d5832..40aca6d5d2 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.huggingFace +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -170,11 +171,15 @@ class HuggingFaceSentimentAnalysisOpDescSpec extends AnyFlatSpec with Matchers { // An empty cell arrives as None, and the tokenizer answers it with // `ValueError: You need to specify either text or text_target`, ending the run. + // pandas is asked rather than None compared, because the type rule naming the + // column string is a warning the editor prints, not a filter, and a numeric + // column reaches the executor with its own NaN. val guard = code.linesIterator - .find(_.contains("text is None")) + .find(_.contains("pd.isna(text)")) .getOrElse(fail("generated code no longer guards an empty text cell")) guard should include("strip()") - code.indexOf("text is None") should be < code.indexOf("self.tokenizer(") + code should include("import pandas as pd") + code.indexOf("pd.isna(text)") should be < code.indexOf("self.tokenizer(") // getOutputSchemas advertises all three score columns on the output port, so the // row this path yields has to carry all three keys -- a fill loop that blanks only @@ -189,6 +194,20 @@ class HuggingFaceSentimentAnalysisOpDescSpec extends AnyFlatSpec with Matchers { } } + // The script reads a frame, where a column holding nothing else comes back as + // float64 and its cells as NaN, which `is None` does not catch. + it should "guard a missing text cell in the exported script, NaN included" in { + val d = configured() + val code = d.generateStandaloneCode() + + val guard = code.linesIterator + .find(_.contains("pd.isna(_text)")) + .getOrElse(fail("exported script no longer guards a missing text cell")) + guard should include("strip()") + code should include("import pandas as pd") + code.indexOf("pd.isna(_text)") should be < code.indexOf("tokenizer(_text") + } + "HuggingFaceSentimentAnalysisOpDesc.getPhysicalOp" should "wire an OpExecWithCode python executor carrying the operator's ports" in { val d = configured() @@ -212,4 +231,14 @@ class HuggingFaceSentimentAnalysisOpDescSpec extends AnyFlatSpec with Matchers { h.resultAttributeNeutral shouldBe "neu" h.resultAttributeNegative shouldBe "neg" } + + "HuggingFaceSentimentAnalysisOpDesc (class-level)" should + "carry @JsonSchemaInject restricting `attribute` to STRING columns" in { + val ann = classOf[HuggingFaceSentimentAnalysisOpDesc].getAnnotation(classOf[JsonSchemaInject]) + ann should not be null + val payload = ann.json + payload should include("attributeTypeRules") + payload should include("attribute") + payload should include("string") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDescSpec.scala index e028a986ac..3f1cb83b57 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSpamSMSDetectionOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.huggingFace +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -131,11 +132,29 @@ class HuggingFaceSpamSMSDetectionOpDescSpec extends AnyFlatSpec with Matchers { // An empty cell arrives as None, and the pipeline answers it with // `ValueError: You need to specify either text or text_target`, ending the run. + // pandas is asked rather than None compared, because the type rule naming the + // column string is a warning the editor prints, not a filter, and a numeric + // column reaches the executor with its own NaN. val guard = code.linesIterator - .find(_.contains("text is None")) + .find(_.contains("pd.isna(text)")) .getOrElse(fail("generated code no longer guards an empty text cell")) guard should include("strip()") - code.indexOf("text is None") should be < code.indexOf("self.pipeline(") + code should include("import pandas as pd") + code.indexOf("pd.isna(text)") should be < code.indexOf("self.pipeline(") + } + + // The script reads a frame, where a column holding nothing else comes back as + // float64 and its cells as NaN, which `is None` does not catch. + it should "guard a missing text cell in the exported script, NaN included" in { + val d = configured() + val code = d.generateStandaloneCode() + + val guard = code.linesIterator + .find(_.contains("pd.isna(_t)")) + .getOrElse(fail("exported script no longer guards a missing text cell")) + guard should include("strip()") + code should include("import pandas as pd") + code.indexOf("pd.isna(_t)") should be < code.indexOf("_pipeline(_t)") } "HuggingFaceSpamSMSDetectionOpDesc.getPhysicalOp" should @@ -160,4 +179,14 @@ class HuggingFaceSpamSMSDetectionOpDescSpec extends AnyFlatSpec with Matchers { h.resultAttributeSpam shouldBe "is_spam" h.resultAttributeProbability shouldBe "score" } + + "HuggingFaceSpamSMSDetectionOpDesc (class-level)" should + "carry @JsonSchemaInject restricting `attribute` to STRING columns" in { + val ann = classOf[HuggingFaceSpamSMSDetectionOpDesc].getAnnotation(classOf[JsonSchemaInject]) + ann should not be null + val payload = ann.json + payload should include("attributeTypeRules") + payload should include("attribute") + payload should include("string") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDescSpec.scala index 1c4f3e0d8b..1606e8d6f7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceTextSummarizationOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.huggingFace +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -107,11 +108,29 @@ class HuggingFaceTextSummarizationOpDescSpec extends AnyFlatSpec with Matchers { // An empty cell arrives as None, and the tokenizer answers it with // `ValueError: text input must be of type str ...`, ending the run. + // pandas is asked rather than None compared, because the type rule naming the + // column string is a warning the editor prints, not a filter, and a numeric + // column reaches the executor with its own NaN. val guard = code.linesIterator - .find(_.contains("text is None")) + .find(_.contains("pd.isna(text)")) .getOrElse(fail("generated code no longer guards an empty text cell")) guard should include("strip()") - code.indexOf("text is None") should be < code.indexOf("self.tokenizer([text]") + code should include("import pandas as pd") + code.indexOf("pd.isna(text)") should be < code.indexOf("self.tokenizer([text]") + } + + // The script reads a frame, where a column holding nothing else comes back as + // float64 and its cells as NaN, which `is None` does not catch. + it should "guard a missing text cell in the exported script, NaN included" in { + val d = configured() + val code = d.generateStandaloneCode() + + val guard = code.linesIterator + .find(_.contains("pd.isna(_text)")) + .getOrElse(fail("exported script no longer guards a missing text cell")) + guard should include("strip()") + code should include("import pandas as pd") + code.indexOf("pd.isna(_text)") should be < code.indexOf("tokenizer([_text]") } "HuggingFaceTextSummarizationOpDesc.getPhysicalOp" should @@ -135,4 +154,14 @@ class HuggingFaceTextSummarizationOpDescSpec extends AnyFlatSpec with Matchers { h.attribute shouldBe "text" h.resultAttribute shouldBe "summary" } + + "HuggingFaceTextSummarizationOpDesc (class-level)" should + "carry @JsonSchemaInject restricting `attribute` to STRING columns" in { + val ann = classOf[HuggingFaceTextSummarizationOpDesc].getAnnotation(classOf[JsonSchemaInject]) + ann should not be null + val payload = ann.json + payload should include("attributeTypeRules") + payload should include("attribute") + payload should include("string") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala index f51e72b26c..96ac3eca76 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala @@ -83,6 +83,21 @@ class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { code should include("\"Skipped\"") } + // The operator runs once per port and counts the rows it dropped each time, so + // the script says it for the table it scores as well as the one it fits. + "SklearnBernoulliNaiveBayesOpDesc.generateStandaloneCode" should + "count the rows it dropped on both tables" in { + val d = new SklearnBernoulliNaiveBayesOpDesc + d.target = "y" + val code = d.generateStandaloneCode() + code should include( + """print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values")""" + ) + code should include( + """print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values")""" + ) + } + "SklearnBernoulliNaiveBayesOpDesc" should "round-trip its config fields through the polymorphic base" in { val d = new SklearnBernoulliNaiveBayesOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDescSpec.scala index d0364e1998..e3f90c7bf1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearRegressionOpDescSpec.scala @@ -95,6 +95,33 @@ class SklearnLinearRegressionOpDescSpec extends AnyFlatSpec with Matchers { code should include("No column left to fit on") } + // Both frames, since the model is fitted on one and scored on the other: a + // narrowing on only one side hands `predict` a different feature set. + "SklearnLinearRegressionOpDesc.generateStandaloneCode" should + "narrow both the training and the testing features" in { + val d = new SklearnLinearRegressionOpDesc + d.target = "y" + val code = d.generateStandaloneCode() + code should include("""_fittable = X_train.select_dtypes(include=["number", "bool"])""") + code should include("X_train = _fittable") + code should include("""_fittable = X_test.select_dtypes(include=["number", "bool"])""") + code should include("X_test = _fittable") + } + + // The operator runs once per port and counts the rows it dropped each time, so + // the script says it for the table it scores as well as the one it fits. + it should "count the rows it dropped on both tables" in { + val d = new SklearnLinearRegressionOpDesc + d.target = "y" + val code = d.generateStandaloneCode() + code should include( + """print("Skipped", len(in1df) - len(_train), "of", len(in1df), "rows with missing values")""" + ) + code should include( + """print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values")""" + ) + } + "SklearnLinearRegressionOpDesc" should "round-trip its target through the polymorphic base" in { val d = new SklearnLinearRegressionOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala index 127ddabaec..1ab28a78fb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala @@ -142,4 +142,32 @@ class SklearnModelOpDescSpec extends AnyFlatSpec with Matchers { } d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) } + + it should "reject the target named as a text column" in { + val d = new TestSklearnModelOpDesc + d.countVectorizer = true + d.target = "species" + d.text = List("note", "species") + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("species") + thrown.getMessage should include("Target Attribute") + thrown.getMessage should include("Text Attribute") + } + + it should "let the text columns through while none of them is the target" in { + val d = new TestSklearnModelOpDesc + d.countVectorizer = true + d.target = "species" + d.text = List("note") + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } + + it should "leave a stale text column alone while Count Vectorizer is off" in { + // Nothing reads `text` with the switch off, and the panel hides it, so a value + // left behind by an earlier configuration must not report the operator invalid. + val d = new TestSklearnModelOpDesc + d.target = "species" + d.text = List("species") + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDescSpec.scala index 16aec5da13..b640e4888d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDescSpec.scala @@ -148,10 +148,72 @@ class SklearnPredictionOpDescSpec extends AnyFlatSpec with Matchers { d.model = "model" d.resultAttribute = "prediction" d.groundTruthAttribute = "y" - val code = d.generatePythonCode() + Seq(d.generatePythonCode(), d.generateStandaloneCode()).foreach { code => + code should include(""""feature_names_in_", None)""") + code should include("if _fitted is not None:") + } + // The two paths narrow with different expressions: this one holds a Tuple, + // the standalone one a frame. + d.generatePythonCode() should include("input_features.get_partial_tuple(list(_fitted))") + } + + // The branch that names no ground truth predicts on the whole frame, so it needs + // the same narrowing as the one that drops a column first. + it should "narrow the features with no ground-truth column configured too" in { + val d = new SklearnPredictionOpDesc + d.model = "model" + d.resultAttribute = "prediction" + d.groundTruthAttribute = "" + val code = d.generateStandaloneCode() code should include(""""feature_names_in_", None)""") - code should include("if _fitted is not None:") - code should include("input_features.get_partial_tuple(list(_fitted))") + code should include("X = X[list(_fitted)]") + } + + // The executor leaves a row's result empty where a feature is missing and keeps + // the row. Predicting the frame in one call cannot do that: scikit-learn ends the + // run on the first missing value, so the exported script would lose every row's + // prediction over one blank cell. Both branches predict on the complete rows. + it should "predict on the complete rows and leave the rest empty" in { + val d = new SklearnPredictionOpDesc + d.model = "model" + d.resultAttribute = "prediction" + Seq("y", "").foreach { groundTruth => + d.groundTruthAttribute = groundTruth + val code = d.generateStandaloneCode() + code should include("_complete = X.notna().all(axis=1)") + code should include("model.predict(X[_complete])") + code should include("out1df[\"prediction\"] = None") + code should include("out1df.loc[_complete, \"prediction\"] = _predicted") + code should not include "model.predict(X)" + } + } + + // The executor keeps the model of every row the model port hands it, each one + // overwriting the last, so it predicts with the model on the final row. A model + // port carrying more than one row is where reading the first row instead would + // answer with a different model than the run did. + it should "predict with the last model the model port carries, as the executor does" in { + val d = new SklearnPredictionOpDesc + d.model = "model" + d.resultAttribute = "prediction" + // The executor's own line, which holds one model rather than collecting them. + d.generatePythonCode() should include("self.model = tuple_[") + Seq("y", "").foreach { groundTruth => + d.groundTruthAttribute = groundTruth + d.generateStandaloneCode() should include("model = in1df[\"model\"].iloc[-1]") + } + } + + // The executor predicts once per data row, so with none it never reads the model + // and an empty model port is not an error. + it should "not read the model when there are no data rows" in { + val d = new SklearnPredictionOpDesc + d.model = "model" + d.resultAttribute = "prediction" + Seq("y", "").foreach { groundTruth => + d.groundTruthAttribute = groundTruth + d.generateStandaloneCode() should include("iloc[-1] if len(in2df) else None") + } } "SklearnPredictionOpDesc" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDescSpec.scala index 8401c6e20b..ad5b3b8021 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDescSpec.scala @@ -80,6 +80,18 @@ class SklearnTestingOpDescSpec extends AnyFlatSpec with Matchers { code should include(".predict(") } + // Both paths hand predict the frame itself. Asserted as text because the + // executor's own block is only ever run inside the engine. + it should "keep the feature frame two dimensional in both paths" in { + val d = new SklearnTestingOpDesc + d.model = "model" + d.target = "y" + d.generatePythonCode() should include("model.predict(X)") + d.generatePythonCode() should not include "squeeze" + d.generateStandaloneCode() should include("model.predict(X)") + d.generateStandaloneCode() should not include "squeeze" + } + // The scores are computed over the rows the model can be applied to, the way // COUNT and MIN are computed over the rows that have a value. it should "drop rows with missing values before scoring" in { @@ -96,10 +108,11 @@ class SklearnTestingOpDescSpec extends AnyFlatSpec with Matchers { val d = new SklearnTestingOpDesc d.model = "model" d.target = "y" - val code = d.generatePythonCode() - code should include("""_fittable = X.select_dtypes(include=["number", "bool"])""") - code should include("""print("Ignoring columns an estimator cannot fit:", _ignored)""") - code should include("X = _fittable") + Seq(d.generatePythonCode(), d.generateStandaloneCode()).foreach { code => + code should include("""_fittable = X.select_dtypes(include=["number", "bool"])""") + code should include("""print("Ignoring columns an estimator cannot fit:", _ignored)""") + code should include("X = _fittable") + } } "SklearnTestingOpDesc" should
