carloea2 commented on code in PR #8368:
URL: https://github.com/apache/texera/pull/8368#discussion_r3973444703
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceSentimentAnalysisOpDesc.scala:
##########
@@ -96,6 +112,44 @@ class HuggingFaceSentimentAnalysisOpDesc extends
PythonOperatorDescriptor {
| yield tuple_""".encode
}
+ // Standalone mirror of generatePythonCode: load the model once, then apply
the
+ // same per-row softmax-over-3-labels logic to in1df, adding the three DOUBLE
+ // result columns (in the same order as getOutputSchemas) to produce out1df.
+ override def generateStandaloneCode(): String = {
+ val positiveLit = pyStringLiteral(resultAttributePositive)
+ val neutralLit = pyStringLiteral(resultAttributeNeutral)
+ val negativeLit = pyStringLiteral(resultAttributeNegative)
+ s"""from transformers import AutoModelForSequenceClassification
+ |from transformers import AutoTokenizer, AutoConfig
+ |import numpy as np
+ |from scipy.special import softmax
+ |
+ |model_name = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+ |tokenizer = AutoTokenizer.from_pretrained(model_name)
+ |config = AutoConfig.from_pretrained(model_name)
+ |model = AutoModelForSequenceClassification.from_pretrained(model_name)
+ |
+ |out1df = in1df.copy()
+ |labels = {"positive": $positiveLit, "neutral": $neutralLit,
"negative": $negativeLit}
+ |for _col in ($positiveLit, $neutralLit, $negativeLit):
+ | out1df[_col] = 0.0
+ |for _idx, _text in out1df[${pyStringLiteral(attribute)}].items():
+ | # An empty cell arrives as None, which the tokenizer rejects. Keep
the row
+ | # and leave the scores empty rather than ending the run over a
value the
+ | # model has nothing to say about.
+ | if _text is None or (isinstance(_text, str) and not _text.strip()):
+ | for _col in ($positiveLit, $neutralLit, $negativeLit):
+ | out1df.at[_idx, _col] = None
+ | continue
+ | encoded_input = tokenizer(_text, return_tensors='pt')
+ | output = model(**encoded_input)
+ | scores = softmax(output[0][0].detach().numpy())
+ | ranking = np.argsort(scores)[::-1]
+ | for i in range(scores.shape[0]):
+ | label = labels[config.id2label[ranking[i]]]
+ | out1df.at[_idx, label] = np.round(float(scores[ranking[i]]),
4)""".stripMargin
Review Comment:
Withdrawn: Union already uses ignore_index=True, so the example I gave does
not occur in the current export. I missed that argument. No change is needed
for this comment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
