This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 31ff8be118 [#8274] Update Python Client to support time/timestamp
precision (#8462)
31ff8be118 is described below
commit 31ff8be118b42017f48e0b02e0d03a781f022b7d
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Sep 8 13:04:46 2025 +0800
[#8274] Update Python Client to support time/timestamp precision (#8462)
### Why are the changes needed?
Fix: #8274
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
UT already added.
Signed-off-by: zacsun <[email protected]>
Co-authored-by: predator4ann <[email protected]>
---
.../api/types/json_serdes/_helper/serdes_utils.py | 16 +++
clients/client-python/gravitino/api/types/type.py | 12 +-
clients/client-python/gravitino/api/types/types.py | 151 ++++++++++++++++++---
clients/client-python/gravitino/utils/serdes.py | 5 +
.../unittests/json_serdes/test_type_serdes.py | 71 ++++++++++
.../tests/unittests/rel/test_types.py | 61 +++++++++
6 files changed, 293 insertions(+), 23 deletions(-)
diff --git
a/clients/client-python/gravitino/api/types/json_serdes/_helper/serdes_utils.py
b/clients/client-python/gravitino/api/types/json_serdes/_helper/serdes_utils.py
index 6aaa12a45e..cab0ebbacc 100644
---
a/clients/client-python/gravitino/api/types/json_serdes/_helper/serdes_utils.py
+++
b/clients/client-python/gravitino/api/types/json_serdes/_helper/serdes_utils.py
@@ -201,6 +201,22 @@ class SerdesUtils(SerdesUtilsBase):
if varchar_matched:
return Types.VarCharType.of(length=int(varchar_matched.group(1)))
+ time_matched = cls.TIME_PATTERN.match(type_string)
+ if time_matched:
+ return Types.TimeType.of(precision=int(time_matched.group(1)))
+
+ timestamp_matched = cls.TIMESTAMP_PATTERN.match(type_string)
+ if timestamp_matched:
+ return Types.TimestampType.without_time_zone(
+ precision=int(timestamp_matched.group(1))
+ )
+
+ timestamp_tz_matched = cls.TIMESTAMP_TZ_PATTERN.match(type_string)
+ if timestamp_tz_matched:
+ return Types.TimestampType.with_time_zone(
+ precision=int(timestamp_tz_matched.group(1))
+ )
+
return Types.UnparsedType.of(type_string)
@classmethod
diff --git a/clients/client-python/gravitino/api/types/type.py
b/clients/client-python/gravitino/api/types/type.py
index 9b089ea187..bbc467a05e 100644
--- a/clients/client-python/gravitino/api/types/type.py
+++ b/clients/client-python/gravitino/api/types/type.py
@@ -146,7 +146,17 @@ class NumericType(PrimitiveType, ABC):
class DateTimeType(PrimitiveType, ABC):
"""Base class for all date/time types."""
- pass
+ # Indicates that precision for the date/time type was not explicitly set
by the user.
+ # The value should be converted to the catalog's default precision.
+ DATE_TIME_PRECISION_NOT_SET = -1
+
+ # Represents the minimum precision range for timestamp, time and other
date/time types.
+ # The minimum precision is 0, which means second-level precision.
+ MIN_ALLOWED_PRECISION = 0
+
+ # Represents the maximum precision allowed for timestamp, time and other
date/time types.
+ # The maximum precision is 12, which means picosecond-level precision.
+ MAX_ALLOWED_PRECISION = 12
class IntervalType(PrimitiveType, ABC):
diff --git a/clients/client-python/gravitino/api/types/types.py
b/clients/client-python/gravitino/api/types/types.py
index 6e82725713..9549c1ec5e 100644
--- a/clients/client-python/gravitino/api/types/types.py
+++ b/clients/client-python/gravitino/api/types/types.py
@@ -322,60 +322,167 @@ class Types:
class TimeType(DateTimeType):
_instance: Types.TimeType = None
+ _precision: int
- def __new__(cls):
- if cls._instance is None:
- cls._instance = super(Types.TimeType, cls).__new__(cls)
- cls._instance.__init__()
- return cls._instance
+ def __new__(cls, precision: int = None):
+ if precision is None:
+ # Forward compatibility: Use singleton when there is no
precision parameter
+ if cls._instance is None:
+ cls._instance = super(Types.TimeType, cls).__new__(cls)
+ cls._instance.__init__()
+ return cls._instance
+ # Create a new instance when the precision parameter is present
+ return super(Types.TimeType, cls).__new__(cls)
+
+ def __init__(self, precision: int = None):
+ if hasattr(self, "_initialized"):
+ return
+ super().__init__()
+ self._precision = (
+ precision if precision is not None else
self.DATE_TIME_PRECISION_NOT_SET
+ )
+ self._initialized = True
@classmethod
def get(cls) -> Types.TimeType:
+ """Returns the default TimeType instance (without precision)"""
return cls()
+ @classmethod
+ def of(cls, precision: int) -> Types.TimeType:
+ """Create a TimeType instance with the specified precision"""
+ if not (
+ cls.MIN_ALLOWED_PRECISION <= precision <=
cls.MAX_ALLOWED_PRECISION
+ ):
+ raise ValueError(
+ f"precision must be in range "
+ f"[{cls.MIN_ALLOWED_PRECISION},
{cls.MAX_ALLOWED_PRECISION}]: "
+ f"precision: {precision}"
+ )
+ return cls(precision)
+
+ def precision(self) -> int:
+ """Returns the precision of the time type"""
+ return self._precision
+
+ def has_precision_set(self) -> bool:
+ """Returns whether the time type has precision set"""
+ return self._precision != self.DATE_TIME_PRECISION_NOT_SET
+
def name(self) -> Name:
return Name.TIME
def simple_string(self) -> str:
- return "time"
+ return f"time({self._precision})" if self.has_precision_set() else
"time"
+
+ def __eq__(self, other):
+ if not isinstance(other, Types.TimeType):
+ return False
+ return self._precision == other._precision
+
+ def __hash__(self):
+ return hash(self._precision)
class TimestampType(DateTimeType):
_instance_with_tz: Types.TimestampType = None
_instance_without_tz: Types.TimestampType = None
_with_time_zone: bool
+ _precision: int
- def __new__(cls, with_time_zone: bool):
- if with_time_zone:
- if cls._instance_with_tz is None:
- cls._instance_with_tz = super(Types.TimestampType,
cls).__new__(cls)
- cls._instance_with_tz.__init__(with_time_zone)
- return cls._instance_with_tz
- if cls._instance_without_tz is None:
- cls._instance_without_tz = super(Types.TimestampType,
cls).__new__(cls)
- cls._instance_without_tz.__init__(with_time_zone)
- return cls._instance_without_tz
+ def __new__(cls, with_time_zone: bool, precision: int = None):
+ if precision is None:
+ # Use singleton when there is no precision parameter
+ if with_time_zone:
+ if cls._instance_with_tz is None:
+ cls._instance_with_tz = super(Types.TimestampType,
cls).__new__(
+ cls
+ )
+ cls._instance_with_tz.__init__(with_time_zone)
+ return cls._instance_with_tz
+ if cls._instance_without_tz is None:
+ cls._instance_without_tz = super(Types.TimestampType,
cls).__new__(
+ cls
+ )
+ cls._instance_without_tz.__init__(with_time_zone)
+ return cls._instance_without_tz
+ # Create a new instance when the precision parameter is present
+ return super(Types.TimestampType, cls).__new__(cls)
+
+ def __init__(self, with_time_zone: bool, precision: int = None):
+ if hasattr(self, "_initialized"):
+ return
+ super().__init__()
+ self._with_time_zone = with_time_zone
+ self._precision = (
+ precision if precision is not None else
self.DATE_TIME_PRECISION_NOT_SET
+ )
+ self._initialized = True
@classmethod
- def with_time_zone(cls) -> Types.TimestampType:
+ def with_time_zone(cls, precision: int = None) -> Types.TimestampType:
+ """Create TimestampType with Timezone"""
+ if precision is not None:
+ if not (
+ cls.MIN_ALLOWED_PRECISION <= precision <=
cls.MAX_ALLOWED_PRECISION
+ ):
+ raise ValueError(
+ f"precision must be in range "
+ f"[{cls.MIN_ALLOWED_PRECISION},
{cls.MAX_ALLOWED_PRECISION}]: "
+ f"precision: {precision}"
+ )
+ return cls(True, precision)
return cls(True)
@classmethod
- def without_time_zone(cls) -> Types.TimestampType:
+ def without_time_zone(cls, precision: int = None) ->
Types.TimestampType:
+ """Create TimestampType without Timezone"""
+ if precision is not None:
+ if not (
+ cls.MIN_ALLOWED_PRECISION <= precision <=
cls.MAX_ALLOWED_PRECISION
+ ):
+ raise ValueError(
+ f"precision must be in range "
+ f"[{cls.MIN_ALLOWED_PRECISION},
{cls.MAX_ALLOWED_PRECISION}]: "
+ f"precision: {precision}"
+ )
+ return cls(False, precision)
return cls(False)
- def __init__(self, with_time_zone: bool):
- self._with_time_zone = with_time_zone
- super().__init__()
-
def has_time_zone(self) -> bool:
+ """Returns whether the timestamp type has a timezone"""
return self._with_time_zone
+ def precision(self) -> int:
+ """Returns the precision of the timestamp type"""
+ return self._precision
+
+ def has_precision_set(self) -> bool:
+ """Returns whether the precision is set for the timestamp type"""
+ return self._precision != self.DATE_TIME_PRECISION_NOT_SET
+
def name(self) -> Name:
return Name.TIMESTAMP
def simple_string(self) -> str:
+ if self.has_precision_set():
+ return (
+ f"timestamp_tz({self._precision})"
+ if self._with_time_zone
+ else f"timestamp({self._precision})"
+ )
return "timestamp_tz" if self._with_time_zone else "timestamp"
+ def __eq__(self, other):
+ if not isinstance(other, Types.TimestampType):
+ return False
+ return (
+ self._with_time_zone == other._with_time_zone
+ and self._precision == other._precision
+ )
+
+ def __hash__(self):
+ return hash((self._with_time_zone, self._precision))
+
class IntervalYearType(IntervalType):
"""The interval year type in Gravitino."""
diff --git a/clients/client-python/gravitino/utils/serdes.py
b/clients/client-python/gravitino/utils/serdes.py
index 8554e65cc4..b862ba9259 100644
--- a/clients/client-python/gravitino/utils/serdes.py
+++ b/clients/client-python/gravitino/utils/serdes.py
@@ -87,6 +87,11 @@ class SerdesUtilsBase:
FIXED_PATTERN: Final[Pattern[str]] = re.compile(r"fixed\(\s*(\d+)\s*\)")
FIXEDCHAR_PATTERN: Final[Pattern[str]] = re.compile(r"char\(\s*(\d+)\s*\)")
VARCHAR_PATTERN: Final[Pattern[str]] =
re.compile(r"varchar\(\s*(\d+)\s*\)")
+ TIME_PATTERN: Final[Pattern[str]] = re.compile(r"time\(\s*(\d+)\s*\)")
+ TIMESTAMP_PATTERN: Final[Pattern[str]] =
re.compile(r"timestamp\(\s*(\d+)\s*\)")
+ TIMESTAMP_TZ_PATTERN: Final[Pattern[str]] = re.compile(
+ r"timestamp_tz\(\s*(\d+)\s*\)"
+ )
TYPES: Final[Mapping] = MappingProxyType(
{
type_instance.simple_string(): type_instance
diff --git
a/clients/client-python/tests/unittests/json_serdes/test_type_serdes.py
b/clients/client-python/tests/unittests/json_serdes/test_type_serdes.py
index 8d71ab9d45..7ef195b649 100644
--- a/clients/client-python/tests/unittests/json_serdes/test_type_serdes.py
+++ b/clients/client-python/tests/unittests/json_serdes/test_type_serdes.py
@@ -46,6 +46,12 @@ class TestTypeSerdes(unittest.TestCase):
"fixed(10)": Types.FixedType.of(10),
"char(10)": Types.FixedCharType.of(10),
"varchar(10)": Types.VarCharType.of(10),
+ "time(6)": Types.TimeType.of(6),
+ "time(3)": Types.TimeType.of(3),
+ "timestamp(6)": Types.TimestampType.without_time_zone(6),
+ "timestamp(0)": Types.TimestampType.without_time_zone(0),
+ "timestamp_tz(6)": Types.TimestampType.with_time_zone(6),
+ "timestamp_tz(3)": Types.TimestampType.with_time_zone(3),
},
}
@@ -323,3 +329,68 @@ class TestTypeSerdes(unittest.TestCase):
TypeSerdes.deserialize,
data=invalid_data,
)
+
+ def test_time_type_precision_serialization(self):
+ """Test the serialization and deserialization of time type precision"""
+ # Test TimeType with precision
+ time_with_precision = Types.TimeType.of(6)
+ serialized = TypeSerdes.serialize(time_with_precision)
+ self.assertEqual(serialized, "time(6)")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, time_with_precision)
+ self.assertTrue(deserialized.has_precision_set())
+ self.assertEqual(deserialized.precision(), 6)
+
+ # Test TimestampType with precision (without timezone)
+ timestamp_with_precision = Types.TimestampType.without_time_zone(3)
+ serialized = TypeSerdes.serialize(timestamp_with_precision)
+ self.assertEqual(serialized, "timestamp(3)")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, timestamp_with_precision)
+ self.assertTrue(deserialized.has_precision_set())
+ self.assertEqual(deserialized.precision(), 3)
+ self.assertFalse(deserialized.has_time_zone())
+
+ # Test TimestampType with precision (with timezone)
+ timestamp_tz_with_precision = Types.TimestampType.with_time_zone(9)
+ serialized = TypeSerdes.serialize(timestamp_tz_with_precision)
+ self.assertEqual(serialized, "timestamp_tz(9)")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, timestamp_tz_with_precision)
+ self.assertTrue(deserialized.has_precision_set())
+ self.assertEqual(deserialized.precision(), 9)
+ self.assertTrue(deserialized.has_time_zone())
+
+ def test_backward_compatibility(self):
+ """Test forward compatibility - Time types without precision should
work properly"""
+ # Test TimeType without precision
+ time_without_precision = Types.TimeType.get()
+ serialized = TypeSerdes.serialize(time_without_precision)
+ self.assertEqual(serialized, "time")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, time_without_precision)
+ self.assertFalse(deserialized.has_precision_set())
+
+ # Test TimestampType without precision (without timezone)
+ timestamp_without_precision = Types.TimestampType.without_time_zone()
+ serialized = TypeSerdes.serialize(timestamp_without_precision)
+ self.assertEqual(serialized, "timestamp")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, timestamp_without_precision)
+ self.assertFalse(deserialized.has_precision_set())
+ self.assertFalse(deserialized.has_time_zone())
+
+ # Test TimestampType without precision (with timezone)
+ timestamp_tz_without_precision = Types.TimestampType.with_time_zone()
+ serialized = TypeSerdes.serialize(timestamp_tz_without_precision)
+ self.assertEqual(serialized, "timestamp_tz")
+
+ deserialized = TypeSerdes.deserialize(serialized)
+ self.assertEqual(deserialized, timestamp_tz_without_precision)
+ self.assertFalse(deserialized.has_precision_set())
+ self.assertTrue(deserialized.has_time_zone())
diff --git a/clients/client-python/tests/unittests/rel/test_types.py
b/clients/client-python/tests/unittests/rel/test_types.py
index 2acdbcf4ea..c742281e7e 100644
--- a/clients/client-python/tests/unittests/rel/test_types.py
+++ b/clients/client-python/tests/unittests/rel/test_types.py
@@ -91,17 +91,78 @@ class TestTypes(unittest.TestCase):
self.assertEqual(instance.simple_string(), "date")
def test_time_type(self):
+ # Test TimeType without precision
instance: Types.TimeType = Types.TimeType.get()
self.assertEqual(instance.name(), Name.TIME)
self.assertEqual(instance.simple_string(), "time")
+ self.assertFalse(instance.has_precision_set())
+ self.assertEqual(
+ instance.precision(), Types.TimeType.DATE_TIME_PRECISION_NOT_SET
+ )
+
+ # Test TimeType with precision
+ time_with_precision = Types.TimeType.of(6)
+ self.assertEqual(time_with_precision.name(), Name.TIME)
+ self.assertEqual(time_with_precision.simple_string(), "time(6)")
+ self.assertTrue(time_with_precision.has_precision_set())
+ self.assertEqual(time_with_precision.precision(), 6)
+
+ # Test Precision Range Verification
+ with self.assertRaises(ValueError):
+ Types.TimeType.of(-1)
+ with self.assertRaises(ValueError):
+ Types.TimeType.of(13)
+
+ time_with_precision_2 = Types.TimeType.of(6)
+ self.assertEqual(time_with_precision, time_with_precision_2)
+ self.assertEqual(hash(time_with_precision),
hash(time_with_precision_2))
+
+ time_different_precision = Types.TimeType.of(3)
+ self.assertNotEqual(time_with_precision, time_different_precision)
def test_timestamp_type(self):
+ # Test TimestampType without precision
instance_with_tz = Types.TimestampType.with_time_zone()
instance_without_tz = Types.TimestampType.without_time_zone()
self.assertTrue(instance_with_tz.has_time_zone())
self.assertFalse(instance_without_tz.has_time_zone())
self.assertEqual(instance_with_tz.simple_string(), "timestamp_tz")
self.assertEqual(instance_without_tz.simple_string(), "timestamp")
+ self.assertFalse(instance_with_tz.has_precision_set())
+ self.assertFalse(instance_without_tz.has_precision_set())
+
+ # Test TimestampType with precision (with timezone)
+ timestamp_tz_with_precision = Types.TimestampType.with_time_zone(6)
+ self.assertTrue(timestamp_tz_with_precision.has_time_zone())
+ self.assertTrue(timestamp_tz_with_precision.has_precision_set())
+ self.assertEqual(timestamp_tz_with_precision.precision(), 6)
+ self.assertEqual(timestamp_tz_with_precision.simple_string(),
"timestamp_tz(6)")
+
+ # Test TimestampType with precision (without timezone)
+ timestamp_with_precision = Types.TimestampType.without_time_zone(3)
+ self.assertFalse(timestamp_with_precision.has_time_zone())
+ self.assertTrue(timestamp_with_precision.has_precision_set())
+ self.assertEqual(timestamp_with_precision.precision(), 3)
+ self.assertEqual(timestamp_with_precision.simple_string(),
"timestamp(3)")
+
+ # Test Precision Range Verification
+ with self.assertRaises(ValueError):
+ Types.TimestampType.with_time_zone(-1)
+ with self.assertRaises(ValueError):
+ Types.TimestampType.without_time_zone(13)
+
+ timestamp_tz_with_precision_2 = Types.TimestampType.with_time_zone(6)
+ self.assertEqual(timestamp_tz_with_precision,
timestamp_tz_with_precision_2)
+ self.assertEqual(
+ hash(timestamp_tz_with_precision),
hash(timestamp_tz_with_precision_2)
+ )
+
+ # Different precision or timezone should not be equal
+ timestamp_different_precision = Types.TimestampType.with_time_zone(3)
+ self.assertNotEqual(timestamp_tz_with_precision,
timestamp_different_precision)
+
+ timestamp_different_tz = Types.TimestampType.without_time_zone(6)
+ self.assertNotEqual(timestamp_tz_with_precision,
timestamp_different_tz)
def test_interval_types(self):
year_instance: Types.IntervalYearType = Types.IntervalYearType.get()
