Github user BryanCutler commented on a diff in the pull request: https://github.com/apache/spark/pull/18664#discussion_r144167906 --- Diff: python/pyspark/sql/types.py --- @@ -1619,11 +1619,47 @@ def to_arrow_type(dt): arrow_type = pa.decimal(dt.precision, dt.scale) elif type(dt) == StringType: arrow_type = pa.string() + elif type(dt) == DateType: + arrow_type = pa.date32() + elif type(dt) == TimestampType: + arrow_type = pa.timestamp('us', tz='UTC') else: raise TypeError("Unsupported type in conversion to Arrow: " + str(dt)) return arrow_type +def _check_localize_series_timestamps(s): + from pandas.types.common import is_datetime64_dtype + # TODO: handle nested timestamps? + if is_datetime64_dtype(s.dtype): + # TODO: pyarrow.Column.to_pandas keeps data in UTC but removes timezone + return s.dt.tz_localize('UTC').dt.tz_convert('tzlocal()').dt.tz_localize(None) + else: + return s + + +def _check_localize_dataframe_timestamps(df): + from pandas.types.common import is_datetime64tz_dtype + for column, series in df.iteritems(): + # TODO: handle nested timestamps? + if is_datetime64tz_dtype(series.dtype): + df[column] = series.dt.tz_convert('tzlocal()').dt.tz_localize(None) + return df + + +def _utc_normalize_series_timestamps(s): + """ Convert a tz-naive timestamp in local tz to UTC normalized + """ + # TODO: this should be system local tz or SESSION_LOCAL_TIMEZONE? + return s.dt.tz_localize('tzlocal()').dt.tz_convert('UTC').values.astype('datetime64[us]') --- End diff -- using `values.astype('datetime64[us]')` was the only way I could the right unit to send to Spark. Even though the pyarrow type specifies the unit, if there is a timezone also, it will ignore it. Not sure if this is a bug.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org