carloea2 commented on code in PR #8368:
URL: https://github.com/apache/texera/pull/8368#discussion_r3973444700
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala:
##########
@@ -99,4 +100,39 @@ class SklearnPredictionOpDesc extends
PythonOperatorDescriptor {
.add(resultAttribute, resultType)
)
}
+
+ /** Python that narrows `X` to the columns the model was fitted on.
+ *
+ * The fitting side leaves out the columns an estimator cannot fit, so this
+ * side has to leave out the same ones or scikit-learn refuses the frame for
+ * naming features it never saw. Read off the model rather than re-derived:
+ * what it was fitted on is a fact it carries, and asking it cannot drift
from
+ * whatever rule the fitting operator applied.
+ */
+ private val narrowToFittedFeatures: String =
+ """_fitted = getattr(model, "feature_names_in_", None)
+ |if _fitted is not None:
+ | X = X[list(_fitted)]""".stripMargin
+
+ override def generateStandaloneCode(): String = {
+ val modelLit = pyStringLiteral(model)
+ val resultLit = pyStringLiteral(resultAttribute)
+ if (groundTruthAttribute.nonEmpty) {
+ s"""from sklearn.pipeline import Pipeline
+ |
+ |model = in1df[$modelLit].iloc[0]
+ |out1df = in2df.copy()
+ |X = in2df.drop(${pyStringLiteral(groundTruthAttribute)}, axis=1)
+ |$narrowToFittedFeatures
+ |out1df[$resultLit] = model.predict(X)""".stripMargin
Review Comment:
This bulk prediction loses the native missing feature behavior. With a
LogisticRegression model and input features [0, NaN, 3], the engine keeps all
three rows and leaves the middle prediction null; the export raises ValueError
for the entire batch. Predict only complete feature rows and place the results
back without dropping the incomplete rows. Both ground truth branches need this.
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala:
##########
@@ -96,6 +112,44 @@ class HuggingFaceSentimentAnalysisOpDesc extends
PythonOperatorDescriptor {
| yield tuple_""".encode
}
+ // Standalone mirror of generatePythonCode: load the model once, then apply
the
+ // same per-row softmax-over-3-labels logic to in1df, adding the three DOUBLE
+ // result columns (in the same order as getOutputSchemas) to produce out1df.
+ override def generateStandaloneCode(): String = {
+ val positiveLit = pyStringLiteral(resultAttributePositive)
+ val neutralLit = pyStringLiteral(resultAttributeNeutral)
+ val negativeLit = pyStringLiteral(resultAttributeNegative)
+ s"""from transformers import AutoModelForSequenceClassification
+ |from transformers import AutoTokenizer, AutoConfig
+ |import numpy as np
+ |from scipy.special import softmax
+ |
+ |model_name = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+ |tokenizer = AutoTokenizer.from_pretrained(model_name)
+ |config = AutoConfig.from_pretrained(model_name)
+ |model = AutoModelForSequenceClassification.from_pretrained(model_name)
+ |
+ |out1df = in1df.copy()
+ |labels = {"positive": $positiveLit, "neutral": $neutralLit,
"negative": $negativeLit}
+ |for _col in ($positiveLit, $neutralLit, $negativeLit):
+ | out1df[_col] = 0.0
+ |for _idx, _text in out1df[${pyStringLiteral(attribute)}].items():
+ | # An empty cell arrives as None, which the tokenizer rejects. Keep
the row
+ | # and leave the scores empty rather than ending the run over a
value the
+ | # model has nothing to say about.
+ | if _text is None or (isinstance(_text, str) and not _text.strip()):
+ | for _col in ($positiveLit, $neutralLit, $negativeLit):
+ | out1df.at[_idx, _col] = None
+ | continue
+ | encoded_input = tokenizer(_text, return_tensors='pt')
+ | output = model(**encoded_input)
+ | scores = softmax(output[0][0].detach().numpy())
+ | ranking = np.argsort(scores)[::-1]
+ | for i in range(scores.shape[0]):
+ | label = labels[config.id2label[ranking[i]]]
+ | out1df.at[_idx, label] = np.round(float(scores[ranking[i]]),
4)""".stripMargin
Review Comment:
The incoming index need not be unique: Union uses pd.concat without
resetting it. If two input rows share index 0, each `.at[0, label]` write
updates both rows, so they end with the last text’s scores. Reset the local
output index or assign scores by position, and cover Union feeding this
operator.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
