This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-7577-d61214a689be3b83eaa27499efe7daa13de71671 in repository https://gitbox.apache.org/repos/asf/texera.git
commit e46564f178e86feab51255d785d67fe3d8a49df0 Author: Eugene Gu <[email protected]> AuthorDate: Thu Aug 13 03:59:24 2026 +0000 test(workflow-operator): cover the vectorizer branches of Sklearn code generation (#7577) ### What changes were proposed in this PR? This PR adds unit test coverage for the vectorizer branches of the Python code generation in the two shared Sklearn base descriptors: - `SklearnTrainingOpDesc` (base of the 26 Sklearn training operators) - `SklearnClassifierOpDesc` (base of the 25 Sklearn classifier operators) Both templates branch on the `countVectorizer` and `tfidfTransformer` properties to select the text column and prepend `CountVectorizer()` / `TfidfTransformer()` stages to the generated `make_pipeline` call, but no test in the repository generated code with either flag set: every existing `generatePythonCode()` assertion runs with both flags default-false, and the specs that do set `countVectorizer = true` are Jackson round-trip tests that never invoke code generation. Two new specs exercise each base through a representative concrete subclass (`SklearnTrainingKNNOpDesc` / `SklearnKNNOpDesc`), matching how the operators use the bases: - `SklearnTrainingOpDescCodegenSpec` (4 tests) - `SklearnClassifierOpDescCodegenSpec` (4 tests) Each spec covers all four flag combinations the templates distinguish, with positive and negative assertions: the both-false baseline (whole-feature path, no vectorizer stages), `countVectorizer` alone (text-column selection plus `CountVectorizer()` stage), both flags (stage order asserted via the full `make_pipeline(CountVectorizer(), TfidfTransformer(), ...)` call), and `tfidfTransformer` alone (a reachable codegen branch even though the UI hides the field when `countVectorizer` is off). Attribute names are `EncodableString`s, so the expected values are built with the production `PythonTemplateBuilder.wrapWithPythonDecoderExpr`, pinning the real base64 decode expressions in the generated code. No production code is changed. ### Any related issues, documentation, discussions? Closes #7574 ### How was this PR tested? This PR is itself test-only. The new specs were run with: ``` sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDescCodegenSpec org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDescCodegenSpec" ``` All 8 tests pass. The suite was additionally mutation-checked: six manual template mutations (swapping the `CountVectorizer`/`TfidfTransformer` stage order, gating the text-column selection on the wrong flag, and ignoring the `tfidfTransformer` flag, in each base) each caused test failures, and the sources were restored afterwards. `scalafmtCheck` passes. ### Was this PR authored or co-authored using generative AI tooling? Co-authored by: Claude Code (Claude Fable 5) --- .../SklearnClassifierOpDescCodegenSpec.scala | 111 +++++++++++++++++++++ .../SklearnTrainingOpDescCodegenSpec.scala | 108 ++++++++++++++++++++ 2 files changed, 219 insertions(+) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala new file mode 100644 index 0000000000..9e732002eb --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.sklearn + +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +/** + * Covers the vectorizer branches of the shared train-and-evaluate codegen + * template in SklearnClassifierOpDesc. The abstract base is exercised through + * a representative concrete subclass (SklearnKNNOpDesc), matching how the 25 + * classifier operators use it. + * + * The template distinguishes all four flag combinations: countVectorizer + * gates the text-column selection and the CountVectorizer() stage, while + * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI + * hides it when countVectorizer is off, but codegen still honors it). + */ +class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { + + // UI-provided attribute names are EncodableStrings: the encoded template + // renders them as a runtime base64-decode expression, not a Python literal. + private def decodeExpr(value: String): String = + PythonTemplateBuilder.wrapWithPythonDecoderExpr( + Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8)) + ) + + // Collapse space runs so assertions on the make_pipeline call are not + // coupled to the exact spacing the empty branches leave behind. + private def normalized(code: String): String = code.replaceAll(" +", " ") + + private def descriptor( + countVectorizer: Boolean = false, + tfidfTransformer: Boolean = false + ): SklearnKNNOpDesc = { + val d = new SklearnKNNOpDesc + d.target = "label" + d.text = "docs" + d.countVectorizer = countVectorizer + d.tfidfTransformer = tfidfTransformer + d + } + + "SklearnClassifierOpDesc.generatePythonCode" should + "generate a plain feature pipeline when both vectorizer flags are off" in { + val code = descriptor().generatePythonCode() + code should include("from sklearn.neighbors import KNeighborsClassifier") + code should include(s"Y = table[${decodeExpr("label")}]") + code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") + // Feature-column path: X is kept whole, the text attribute is never read. + code should include("X = X\n") + code should not include decodeExpr("docs") + normalized(code) should include( + "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)" + ) + code should include("predictions = self.model.predict(X)") + code should not include "CountVectorizer()" + code should not include "TfidfTransformer()" + } + + it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { + val code = descriptor(countVectorizer = true).generatePythonCode() + code should include(s"X = X[${decodeExpr("docs")}]") + code should not include "X = X\n" + normalized(code) should include( + "self.model = make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + ) + code should not include "TfidfTransformer()" + } + + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { + val code = + descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() + code should include(s"X = X[${decodeExpr("docs")}]") + normalized(code) should include( + "self.model = make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + ) + } + + it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { + val code = descriptor(tfidfTransformer = true).generatePythonCode() + // Without countVectorizer there is no text-column selection. + code should include("X = X\n") + code should not include decodeExpr("docs") + normalized(code) should include( + "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + ) + code should not include "CountVectorizer()" + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala new file mode 100644 index 0000000000..0c00ca271b --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.sklearn.training + +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +/** + * Covers the vectorizer branches of the shared training codegen template in + * SklearnTrainingOpDesc. The template is exercised through a representative + * concrete subclass (SklearnTrainingKNNOpDesc) so the generated pipeline ends + * in a real estimator, matching how the 26 training operators use the base. + * + * The template distinguishes all four flag combinations: countVectorizer + * gates the text-column selection and the CountVectorizer() stage, while + * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI + * hides it when countVectorizer is off, but codegen still honors it). + */ +class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { + + // UI-provided attribute names are EncodableStrings: the encoded template + // renders them as a runtime base64-decode expression, not a Python literal. + private def decodeExpr(value: String): String = + PythonTemplateBuilder.wrapWithPythonDecoderExpr( + Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8)) + ) + + // Collapse space runs so assertions on the make_pipeline call are not + // coupled to the exact spacing the empty branches leave behind. + private def normalized(code: String): String = code.replaceAll(" +", " ") + + private def descriptor( + countVectorizer: Boolean = false, + tfidfTransformer: Boolean = false + ): SklearnTrainingKNNOpDesc = { + val d = new SklearnTrainingKNNOpDesc + d.target = "label" + d.text = "docs" + d.countVectorizer = countVectorizer + d.tfidfTransformer = tfidfTransformer + d + } + + "SklearnTrainingOpDesc.generatePythonCode" should + "generate a plain feature pipeline when both vectorizer flags are off" in { + val code = descriptor().generatePythonCode() + code should include("from sklearn.neighbors import KNeighborsClassifier") + code should include(s"Y = table[${decodeExpr("label")}]") + code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") + // Feature-column path: X is kept whole, the text attribute is never read. + code should include("X = X\n") + code should not include decodeExpr("docs") + normalized(code) should include("make_pipeline( KNeighborsClassifier()).fit(X, Y)") + code should not include "CountVectorizer()" + code should not include "TfidfTransformer()" + } + + it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { + val code = descriptor(countVectorizer = true).generatePythonCode() + code should include(s"X = X[${decodeExpr("docs")}]") + code should not include "X = X\n" + normalized(code) should include( + "make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + ) + code should not include "TfidfTransformer()" + } + + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { + val code = + descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() + code should include(s"X = X[${decodeExpr("docs")}]") + normalized(code) should include( + "make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + ) + } + + it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { + val code = descriptor(tfidfTransformer = true).generatePythonCode() + // Without countVectorizer there is no text-column selection. + code should include("X = X\n") + code should not include decodeExpr("docs") + normalized(code) should include( + "make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + ) + code should not include "CountVectorizer()" + } +}
