kz930 commented on code in PR #8597:
URL: https://github.com/apache/texera/pull/8597#discussion_r4275764632
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala:
##########
@@ -115,27 +128,111 @@ object StandaloneHelpers {
| elif isinstance(x, bool):
| value = 1 if x else 0
| else:
- | value = int(x)
+ | value = x
| if wrap:
- | return ((value + 2147483648) % 4294967296) - 2147483648
- | # int() first: Double.intValue truncates toward zero before it
clamps.
- | return max(-2147483648, min(2147483647, int(value)))
+ | return ((int(value) + 2147483648) % 4294967296) - 2147483648
+ | return _texera_saturate(value, -2147483648, 2147483647)
+ |
+ |
+ |def _texera_text_to_timestamp(s):
+ | # Read cell by cell, the way the engine reads a column:
DateParserUtils
+ | # is handed one field at a time, so a row states its own format and
the
+ | # rest of the column has no say in it.
+ | #
+ | # What is read here is held at microsecond resolution and not the
+ | # nanoseconds pandas parses into by default, which reach 1677 to
2262:
+ | # the engine holds a java.sql.Timestamp, where the year 2500 is an
+ | # ordinary moment and emptying it would answer for a row the run
itself
+ | # had no trouble with. Text is cut to the millisecond, because
+ | # DateParserUtils reads it into a java.util.Date, which counts no
finer.
+ | # A column that is already a moment is handed back
+ | # at the resolution it arrived in instead, because parseField
returns a
+ | # java.sql.Timestamp untouched and that class counts nanoseconds.
+ | #
+ | # Still coerced, which the strict cast is not: the engine accepts a
set
+ | # of formats no single pandas call states, so text neither can read
is
+ | # answered with an empty cell rather than by ending the run.
+ | #
+ | # A reading that states an offset is moved to the zone the machine
is
+ | # set to and then holds that wall clock, which is what the engine
does
+ | # with one: DateParserUtils reads the offset and java.sql.Timestamp
+ | # keeps no zone of its own. Left alone, the offset travels as far as
+ | # the conversion below and stops the cast on a value the run reads.
+ | from dateutil.parser import parse as _parse_date
+ |
+ | if pd.api.types.is_datetime64_any_dtype(s):
+ | if getattr(s.dtype, "tz", None) is not None:
+ | return _texera_local_wall_clocks(s)
+ | return s
+ |
+ | def _one(x):
+ | if pd.isna(x):
+ | return None
+ | try:
+ | parsed = _parse_date(str(x).strip())
+ | except (ValueError, OverflowError):
+ | return None
+ | if parsed.tzinfo is not None:
+ | parsed = _texera_local_wall_clock(parsed)
+ | return parsed.replace(microsecond=parsed.microsecond // 1000 *
1000)
+ |
+ | return s.map(_one).astype("datetime64[us]")
|
|
|def _texera_epoch_millis_to_timestamp(s):
| # `new Timestamp(long)` reads MILLISECONDS where pd.to_datetime
defaults
| # to nanoseconds, and renders in the JVM's default zone, so leaving
the
| # result in UTC would put it a whole offset away. Text needs
neither: a
| # parsed wall clock is already the wall clock.
- | # tzlocal() and not the current offset: the zone carries its
daylight
- | # rules, and each instant needs the offset in force when it
happened.
+ | return _texera_local_wall_clocks(pd.to_datetime(s, unit="ms",
errors="coerce", utc=True))
+ |
+ |
+ |def _texera_local_wall_clock(moment):
+ | # The wall clock the machine's zone shows at an instant that states
its
+ | # offset. tzlocal() and not the current offset: the zone carries its
+ | # daylight rules, and each instant needs the offset in force when it
+ | # happened.
+ | #
+ | # Windows' localtime refuses an instant before 1970, and tzlocal()
asks
+ | # it. Such an instant takes the zone's standard offset instead,
which is
+ | # what java.util.Date answers before its zone history starts in
1900.
+ | import datetime
+ | import time
+ |
| from dateutil.tz import tzlocal
|
- | return (
- | pd.to_datetime(s, unit="ms", errors="coerce", utc=True)
- | .dt.tz_convert(tzlocal())
- | .dt.tz_localize(None)
- | )
+ | try:
+ | return moment.astimezone(tzlocal()).replace(tzinfo=None)
+ | except (OSError, OverflowError):
+ | utc =
moment.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+ | return utc + datetime.timedelta(seconds=-time.timezone)
Review Comment:
Thanks, reproduced: dateutil's `tzlocal()` only knows today's two offsets,
so this was off on every platform (2021-07 in Mexico City was an hour off on
macOS too). Fixed in 7a3203d72 by reading the zone by IANA name, with your case
plus 2021-07, 1921 and pre-1900 as a regression test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
