This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/main by this push:
new 490e57f2cc feat(visualization): drop rows with missing values before
laying out a Dumbbell Plot (#7567)
490e57f2cc is described below
commit 490e57f2cc384581401f56aa9eafb5cd7af03d7d
Author: Kary Zheng <[email protected]>
AuthorDate: Fri Aug 14 09:41:00 2026 +0000
feat(visualization): drop rows with missing values before laying out a
Dumbbell Plot (#7567)
### What changes were proposed in this PR?
Dumbbell Plot took the distinct values of the compared column and sorted
them. An empty cell reaches the operator as `None`, so `unique()`
returned a list with a `None` in it and `sorted` compared `None` against
a string, ending the run with a TypeError.
An empty value is ordinary input here. A blank CSV cell arrives as null,
since univocity returns null for an empty field and
`AttributeTypeUtils.parseField` passes it through by design.
It now drops rows missing any of the three columns it reads, which is
what the other visualization operators do: twenty-four of them open
their generated Python with `dropna(subset=[...]) #remove missing
values`, and Dumbbell Plot was the one that did not. A table left empty
by the drop renders the operator's own error rather than an exception,
matching how it already reports an empty input.
### Any related issues, documentation, discussions?
Closes #7562
### How was this PR tested?
`DumbbellPlotOpDescSpec` gains a case asserting the generated Python
drops on all three configured columns before it sorts. It fails on the
previous behavior, 7 passed / 1 failed before the change and 8 / 0
after.
### Was this PR authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
Co-authored-by: Xinyuan Lin <[email protected]>
---
.../dumbbellPlot/DumbbellPlotOpDesc.scala | 4 ++++
.../dumbbellPlot/DumbbellPlotOpDescSpec.scala | 27 ++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git
a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala
b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala
index 9d62310e85..e855507e56 100644
---
a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala
+++
b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala
@@ -183,6 +183,10 @@ class DumbbellPlotOpDesc extends PythonOperatorDescriptor {
| if table.empty:
| yield {'html-content': self.render_error("input table is
empty.")}
| return
+ | table = table.dropna(subset=[$comparedColumnName,
$categoryColumnName, $measurementColumnName]) #remove missing values
+ | if table.empty:
+ | yield {'html-content': self.render_error("input table has
no rows with all of the configured columns filled in.")}
+ | return
| ${createPlotlyDumbbellLineFigure()}
| ${addPlotlyDots()}
| # convert fig to html content
diff --git
a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala
b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala
index 5f32abb409..b9841984eb 100644
---
a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala
+++
b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala
@@ -26,8 +26,14 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
class DumbbellPlotOpDescSpec extends AnyFlatSpec with Matchers {
+ private def b64(s: String): String =
+ Base64.getEncoder.encodeToString(s.getBytes(StandardCharsets.UTF_8))
+
"DumbbellPlotOpDesc.operatorInfo" should
"advertise the name and Basic visualization group" in {
val info = (new DumbbellPlotOpDesc).operatorInfo
@@ -67,6 +73,27 @@ class DumbbellPlotOpDescSpec extends AnyFlatSpec with
Matchers {
code should include("go.Scatter(")
}
+ it should "drop rows missing any configured column before laying the plot
out" in {
+ val d = new DumbbellPlotOpDesc
+ d.categoryColumnName = "entity"
+ d.measurementColumnName = "metric"
+ d.comparedColumnName = "phase"
+ d.dumbbellStartValue = "before"
+ d.dumbbellEndValue = "after"
+ val code = d.generatePythonCode()
+
+ // Without this, an empty cell in the compared column reaches `sorted` as
None
+ // and raises a TypeError against the other, string, entity names.
+ val dropna = code.linesIterator
+ .find(_.contains("dropna"))
+ .getOrElse(fail("generated code no longer drops rows with missing
values"))
+ // The pyb macro base64-encodes interpolated column names, so the plain
text
+ // never appears in the template.
+ Seq("phase", "entity", "metric").foreach(col => dropna should
include(b64(col)))
+
+ code.indexOf("dropna") should be < code.indexOf("sorted(")
+ }
+
"DumbbellPlotOpDesc.createPlotlyDumbbellLineFigure" should
"select the showlegend flag from showLegends" in {
val on = new DumbbellPlotOpDesc
