kz930 commented on code in PR #8597:
URL: https://github.com/apache/texera/pull/8597#discussion_r4255202396
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDesc.scala:
##########
@@ -117,23 +117,27 @@ class TypeCastingOpDesc extends MapOpDesc with
StandaloneCodeGenerator {
// declared type still says so: the values reach the helper as
floats.
val wrap =
!declared.get(unit.attribute).contains(AttributeType.DOUBLE)
val wrapArg = if (wrap) "True" else "False"
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_int32(x, $wrapArg)).astype("Int64")"""
+ // The column's mask decides the holes, so a stored NaN is cast like
+ // any other double; see `_texera_cast_cells`.
+ s"""_texera_cast_cells(out1df[$colLit], lambda x:
_texera_cast_int32(x, $wrapArg), pd.NA).astype("Int64")"""
case AttributeType.LONG =>
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_integral(x)).astype("Int64")"""
+ s"""_texera_cast_cells(out1df[$colLit], _texera_cast_integral,
pd.NA).astype("Int64")"""
case AttributeType.DOUBLE =>
// NaN rather than pd.NA: float64 is how a double column is held
here,
// and it carries its hole as NaN. pd.NA would not survive the
astype.
s"""out1df[$colLit].apply(lambda x: float("nan") if pd.isna(x) else
_texera_cast_double(x)).astype("float64")"""
Review Comment:
Fixed in 5d465257c. The DOUBLE cast now builds a nullable Float64 from the
column's own mask, and the chained DOUBLE then STRING case is in the spec.
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDesc.scala:
##########
@@ -117,23 +117,27 @@ class TypeCastingOpDesc extends MapOpDesc with
StandaloneCodeGenerator {
// declared type still says so: the values reach the helper as
floats.
val wrap =
!declared.get(unit.attribute).contains(AttributeType.DOUBLE)
val wrapArg = if (wrap) "True" else "False"
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_int32(x, $wrapArg)).astype("Int64")"""
+ // The column's mask decides the holes, so a stored NaN is cast like
+ // any other double; see `_texera_cast_cells`.
+ s"""_texera_cast_cells(out1df[$colLit], lambda x:
_texera_cast_int32(x, $wrapArg), pd.NA).astype("Int64")"""
case AttributeType.LONG =>
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_integral(x)).astype("Int64")"""
+ s"""_texera_cast_cells(out1df[$colLit], _texera_cast_integral,
pd.NA).astype("Int64")"""
case AttributeType.DOUBLE =>
// NaN rather than pd.NA: float64 is how a double column is held
here,
// and it carries its hole as NaN. pd.NA would not survive the
astype.
s"""out1df[$colLit].apply(lambda x: float("nan") if pd.isna(x) else
_texera_cast_double(x)).astype("float64")"""
case AttributeType.BOOLEAN =>
// Nullable "boolean" for the same reason, and because
`.astype(bool)`
// reads NaN as True: NaN is a non-zero float.
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_boolean(x)).astype("boolean")"""
+ s"""_texera_cast_cells(out1df[$colLit], _texera_cast_boolean,
pd.NA).astype("boolean")"""
case AttributeType.TIMESTAMP =>
- // A number is an instant in milliseconds and needs its own reading;
- // see the helper. Text keeps the parser it already had.
+ // A number is an instant in milliseconds and needs its own reading.
+ // Text is read a cell at a time, and over the years the engine
+ // reaches rather than the ones pandas parses into by default; see
+ // the helpers.
if (declared.get(unit.attribute).contains(AttributeType.LONG))
s"""_texera_epoch_millis_to_timestamp(out1df[$colLit])"""
Review Comment:
Fixed in 5d465257c. When localtime refuses an instant, the cell takes the
zone's standard offset, as java.util.Date does before 1900. The spec covers
1600, 2024, 2500 and null with localtime made to refuse negative seconds.
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDesc.scala:
##########
@@ -117,23 +117,27 @@ class TypeCastingOpDesc extends MapOpDesc with
StandaloneCodeGenerator {
// declared type still says so: the values reach the helper as
floats.
val wrap =
!declared.get(unit.attribute).contains(AttributeType.DOUBLE)
val wrapArg = if (wrap) "True" else "False"
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_int32(x, $wrapArg)).astype("Int64")"""
+ // The column's mask decides the holes, so a stored NaN is cast like
+ // any other double; see `_texera_cast_cells`.
+ s"""_texera_cast_cells(out1df[$colLit], lambda x:
_texera_cast_int32(x, $wrapArg), pd.NA).astype("Int64")"""
case AttributeType.LONG =>
- s"""out1df[$colLit].apply(lambda x: pd.NA if pd.isna(x) else
_texera_cast_integral(x)).astype("Int64")"""
+ s"""_texera_cast_cells(out1df[$colLit], _texera_cast_integral,
pd.NA).astype("Int64")"""
case AttributeType.DOUBLE =>
// NaN rather than pd.NA: float64 is how a double column is held
here,
// and it carries its hole as NaN. pd.NA would not survive the
astype.
s"""out1df[$colLit].apply(lambda x: float("nan") if pd.isna(x) else
_texera_cast_double(x)).astype("float64")"""
Review Comment:
Fixed in 5d465257c. The double cast now builds a nullable Float64 from the
column's own mask, so a NaN read from text stays apart from null. The chained
case `["NaN", null, "1.25", "Infinity"]` to DOUBLE then STRING is in the spec.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
