jun-he commented on a change in pull request #2910:
URL: https://github.com/apache/iceberg/pull/2910#discussion_r685327072
##########
File path: python/iceberg/api/transforms/projection_util.py
##########
@@ -18,49 +18,138 @@
import decimal
-from iceberg.api.expressions import Expressions, Operation
-
-
-class ProjectionUtil(object):
- @staticmethod
- def truncate_integer(name, pred, transform):
- boundary = pred.lit.value
- if pred.op == Operation.LT:
- return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary - 1))
- elif pred.op == Operation.LT_EQ:
- return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.GT:
- return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary + 1))
- elif pred.op == Operation.GT_EQ:
- return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.EQ:
- return Expressions.predicate(pred.op, name,
transform.apply(boundary))
-
- def truncate_long(name, pred, transform):
- return ProjectionUtil.truncate_integer(name, pred, transform)
-
- def truncate_decimal(name, pred, transform):
- boundary = pred.lit.value
-
- if pred.op == Operation.LT:
- minus_one = boundary - decimal.Decimal(1)
- return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(minus_one))
- elif pred.op == Operation.LT_EQ:
- return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.GT:
- plus_one = boundary + decimal.Decimal(1)
- return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(plus_one))
- elif pred.op == Operation.GT_EQ:
- return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.EQ:
- return Expressions.predicate(pred.op, name,
transform.apply(boundary))
-
- def truncate_array(name, pred, transform):
- boundary = pred.lit.value
-
- if pred.op == Operation.LT or pred.op == Operation.LT_EQ:
- return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.GT or pred.op == Operation.GT_EQ:
- return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
- elif pred.op == Operation.EQ:
- return Expressions.predicate(pred.op, name,
transform.apply(boundary))
+from ..expressions import Expressions, Operation
+from ..expressions.predicate import UnboundPredicate
+
+
+def truncate_integer(name, pred, transform):
+ boundary = pred.lit.value
+ if pred.op == Operation.LT:
+ return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary - 1))
+ elif pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT:
+ return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary + 1))
+ elif pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.EQ:
+ return Expressions.predicate(pred.op, name, transform.apply(boundary))
+
+
+def truncate_integer_strict(name, pred, transform):
+ boundary = pred.lit.value
+ if pred.op == Operation.LT:
+ return Expressions.predicate(Operation.LT, name,
transform.apply(boundary))
+ elif pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT, name,
transform.apply(boundary + 1))
+ elif pred.op == Operation.GT:
+ return Expressions.predicate(Operation.GT, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT, name,
transform.apply(boundary - 1))
+ elif pred.op == Operation.NOT_EQ:
+ return Expressions.predicate(Operation.NOT_EQ, name,
transform.apply(boundary))
+
+
+def truncate_long(name, pred, transform):
+ return truncate_integer(name, pred, transform)
+
+
+def truncate_long_strict(name, pred, transform):
+ return truncate_integer_strict(name, pred, transform)
+
+
+def truncate_decimal(name, pred, transform):
+ boundary = pred.lit.value
+
+ if pred.op == Operation.LT:
+ minus_one = boundary - decimal.Decimal(1)
+ return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(minus_one))
+ elif pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT:
+ plus_one = boundary + decimal.Decimal(1)
+ return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(plus_one))
+ elif pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.EQ:
+ return Expressions.predicate(pred.op, name, transform.apply(boundary))
+
+
+def truncate_decimal_strict(name, pred, transform):
+ boundary = pred.lit.value
+ minus_one = boundary - decimal.Decimal(1)
+ plus_one = boundary + decimal.Decimal(1)
+
+ if pred.op == Operation.LT:
+ return Expressions.predicate(Operation.LT, name,
transform.apply(boundary))
+ elif pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT, name,
transform.apply(plus_one))
+ elif pred.op == Operation.GT:
+ return Expressions.predicate(Operation.GT, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT, name,
transform.apply(minus_one))
+ elif pred.op == Operation.NOT_EQ:
+ return Expressions.predicate(Operation.NOT_EQ, name,
transform.apply(boundary))
+
+
+def truncate_array(name, pred, transform):
+ boundary = pred.lit.value
+
+ if pred.op == Operation.LT or pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT or pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT_EQ, name,
transform.apply(boundary))
+ elif pred.op == Operation.EQ:
+ return Expressions.predicate(pred.op, name, transform.apply(boundary))
+ # todo elif pred.op == Operation.STARTS_WITH:
+
+
+def truncate_array_strict(name, pred, transform):
+ boundary = pred.lit.value
+
+ if pred.op == Operation.LT or pred.op == Operation.LT_EQ:
+ return Expressions.predicate(Operation.LT, name,
transform.apply(boundary))
+ elif pred.op == Operation.GT or pred.op == Operation.GT_EQ:
+ return Expressions.predicate(Operation.GT, name,
transform.apply(boundary))
+ elif pred.op == Operation.NOT_EQ:
+ return Expressions.predicate(Operation.NOT_EQ, name,
transform.apply(boundary))
+
+
+def project_transform_predicate(transform, partiton_name, pred):
+ raise NotImplementedError
+
+
+def fix_inclusive_time_projection(projected: UnboundPredicate):
+ if projected is None:
+ return projected
+
+ if projected.op == Operation.LT and projected.lit.value < 0:
+ return Expressions.less_than(projected.term, projected.lit.value + 1)
+ elif projected.op == Operation.LT_EQ and projected.lit.value < 0:
+ return Expressions.less_than_or_equal(projected.term,
projected.lit.value + 1)
+ elif projected.op == Operation.EQ and projected.lit.value < 0:
+ raise NotImplementedError # todo implement in
Review comment:
This method is used to fix a
[bug](https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java#L262-L268)
to account for incorrectly transformed values. So here, it needs IN to match
two possible values, either the incorrect value (projectedValue + 1) or the
correct value (projectedValue). Here is the related java code:
https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java#L296-L300
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
