This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-orbit.git
The following commit(s) were added to refs/heads/main by this push:
new ec305c7c Convert date bounds correctly: space separator, and whole-day
rounding (#74)
ec305c7c is described below
commit ec305c7ca8959c56ebfebfc6485f434521bcafe3
Author: Serhiy Bzhezytskyy <[email protected]>
AuthorDate: Thu Sep 17 22:54:54 2026 +0300
Convert date bounds correctly: space separator, and whole-day rounding (#74)
---
solrorbit/conversion/query.py | 89 +++++++++++++++++++-----------
tests/unit/solr/test_workload_converter.py | 52 +++++++++++++++++
2 files changed, 109 insertions(+), 32 deletions(-)
diff --git a/solrorbit/conversion/query.py b/solrorbit/conversion/query.py
index 432b22b0..41f26786 100644
--- a/solrorbit/conversion/query.py
+++ b/solrorbit/conversion/query.py
@@ -26,7 +26,7 @@ Native Solr workloads should not go through this translation
layer.
"""
import logging
-from datetime import datetime
+from datetime import datetime, timedelta
from .field import normalize_field_name
@@ -183,21 +183,25 @@ def _translate_query_node(node: dict, fq_list: list =
None) -> str:
field = normalize_field_name(field)
# gt/lt are exclusive; Solr spells that with a curly bracket.
if "gte" in bounds:
- lo, lo_bracket = bounds["gte"], "["
+ lo, lo_bracket, lo_key = bounds["gte"], "[", "gte"
elif "gt" in bounds:
- lo, lo_bracket = bounds["gt"], "{"
+ lo, lo_bracket, lo_key = bounds["gt"], "{", "gt"
else:
- lo, lo_bracket = "*", "["
+ lo, lo_bracket, lo_key = "*", "[", None
if "lte" in bounds:
- hi, hi_bracket = bounds["lte"], "]"
+ hi, hi_bracket, hi_key = bounds["lte"], "]", "lte"
elif "lt" in bounds:
- hi, hi_bracket = bounds["lt"], "}"
+ hi, hi_bracket, hi_key = bounds["lt"], "}", "lt"
else:
- hi, hi_bracket = "*", "]"
+ hi, hi_bracket, hi_key = "*", "]", None
# Convert dates if format is specified (common for date fields)
os_format = bounds.get("format")
- lo = _convert_date_to_solr_format(lo, os_format)
- hi = _convert_date_to_solr_format(hi, os_format)
+ lo, lo_is_date_only = _convert_date_to_solr_format(lo, os_format)
+ hi, hi_is_date_only = _convert_date_to_solr_format(hi, os_format)
+ if hi_is_date_only and hi_key == "lte":
+ hi, hi_bracket = _round_date_only_bound(hi), "}"
+ if lo_is_date_only and lo_key == "gt":
+ lo, lo_bracket = _round_date_only_bound(lo), "["
return f"{field}:{lo_bracket}{lo} TO {hi}{hi_bracket}"
if "exists" in node:
@@ -529,7 +533,18 @@ def _calendar_interval_to_solr_gap(interval: str) -> str:
return mapping.get(str(interval).lower(), "+1MONTH")
-def _convert_date_to_solr_format(date_str, os_format=None) -> str:
+OS_TO_PYTHON_FORMAT = {
+ "dd/MM/yyyy": ("%d/%m/%Y", False),
+ "MM/dd/yyyy": ("%m/%d/%Y", False),
+ "yyyy-MM-dd": ("%Y-%m-%d", False),
+ "yyyy/MM/dd": ("%Y/%m/%d", False),
+ "dd-MM-yyyy": ("%d-%m-%Y", False),
+ "MM-dd-yyyy": ("%m-%d-%Y", False),
+ "yyyy-MM-dd HH:mm:ss": ("%Y-%m-%d %H:%M:%S", True),
+}
+
+
+def _convert_date_to_solr_format(date_str, os_format=None) -> tuple:
"""
Convert an OpenSearch date string to Solr ISO 8601 format.
@@ -538,50 +553,60 @@ def _convert_date_to_solr_format(date_str,
os_format=None) -> str:
os_format: Optional OpenSearch date format pattern (e.g., "dd/MM/yyyy")
Returns:
- ISO 8601 date string for Solr (e.g., "2015-01-01T00:00:00Z")
+ (value, is_date_only) — the ISO 8601 date string for Solr
+ (e.g., "2015-01-01T00:00:00Z"), and whether the source named a whole
+ day rather than an instant.
If the date is already in ISO format or conversion fails, returns the
original string unchanged.
"""
if not isinstance(date_str, str) or date_str in ("*", "now"):
- return date_str
-
- # Map OpenSearch date format patterns to Python strptime format
- OS_TO_PYTHON_FORMAT = {
- "dd/MM/yyyy": "%d/%m/%Y",
- "MM/dd/yyyy": "%m/%d/%Y",
- "yyyy-MM-dd": "%Y-%m-%d",
- "yyyy/MM/dd": "%Y/%m/%d",
- "dd-MM-yyyy": "%d-%m-%Y",
- "MM-dd-yyyy": "%m-%d-%Y",
- # Add more as needed
- }
+ return date_str, False
# If format is provided, use it to parse the date
if os_format:
- python_fmt = OS_TO_PYTHON_FORMAT.get(os_format)
- if python_fmt:
+ pattern = OS_TO_PYTHON_FORMAT.get(os_format)
+ if pattern:
+ python_fmt, has_time = pattern
try:
dt = datetime.strptime(date_str, python_fmt)
- return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+ return dt.strftime("%Y-%m-%dT%H:%M:%SZ"), not has_time
except ValueError:
logger.warning(f"Failed to parse date '{date_str}' with format
'{os_format}'")
- return date_str
+ return date_str, False
else:
logger.warning(f"Unknown OpenSearch date format: '{os_format}'")
# Try common patterns if no format specified
- for python_fmt in OS_TO_PYTHON_FORMAT.values():
+ for python_fmt, has_time in OS_TO_PYTHON_FORMAT.values():
try:
dt = datetime.strptime(date_str, python_fmt)
- return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+ return dt.strftime("%Y-%m-%dT%H:%M:%SZ"), not has_time
except ValueError:
continue
# If it's already in ISO-like format, return as-is
# (handles cases like "2015-01-01T00:00:00Z" or partial ISO)
- if "T" in date_str or len(date_str) == 10: # YYYY-MM-DD
- return date_str
+ if "T" in date_str:
+ return date_str, False
logger.warning(f"Could not parse date '{date_str}', using as-is")
- return date_str
+ return date_str, False
+
+
+def _round_date_only_bound(value: str) -> str:
+ """
+ Advance a whole-day bound to the start of the following day.
+
+ A date without a time names a day, and OpenSearch rounds it to the edge of
+ that day: `lte` and `gt` go to its LAST millisecond, `gte` and `lt` to its
+ first. Solr rounds nothing, so only the two that move to the end of the day
+ need translating, and naming the next day's first instant says that without
+ depending on how fine Solr's date precision happens to be.
+ """
+ try:
+ dt = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+ except ValueError:
+ logger.warning(f"Could not round whole-day bound '{value}', using
as-is")
+ return value
+ return (dt + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
diff --git a/tests/unit/solr/test_workload_converter.py
b/tests/unit/solr/test_workload_converter.py
index 423dc9c7..fd1566bb 100644
--- a/tests/unit/solr/test_workload_converter.py
+++ b/tests/unit/solr/test_workload_converter.py
@@ -228,6 +228,58 @@ class TestTranslateToSolrJsonDsl(unittest.TestCase):
self.assertEqual("fare_amount:[* TO 100}", self._range({"lt": 100}))
self.assertEqual("fare_amount:[* TO 100]", self._range({"lte": 100}))
+ def _date_range(self, bounds):
+ return translate_to_solr_json_dsl(
+ {"query": {"range": {"dropoff_datetime": bounds}}})["query"]
+
+ def test_a_datetime_with_a_space_separator_is_converted(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-01T00:00:00Z TO 2016-01-01T00:00:00Z}",
+ self._date_range({"gte": "2015-01-01 00:00:00", "lt": "2016-01-01
00:00:00"}))
+
+ def test_a_whole_day_lte_covers_that_day(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-01T00:00:00Z TO 2015-01-22T00:00:00Z}",
+ self._date_range({"gte": "01/01/2015", "lte": "21/01/2015",
+ "format": "dd/MM/yyyy"}))
+
+ def test_a_whole_day_gt_excludes_that_day(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-02T00:00:00Z TO *]",
+ self._date_range({"gt": "01/01/2015", "format": "dd/MM/yyyy"}))
+
+ def test_the_bounds_that_round_down_do_not_move(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-01T00:00:00Z TO *]",
+ self._date_range({"gte": "01/01/2015", "format": "dd/MM/yyyy"}))
+ self.assertEqual(
+ "dropoff_datetime:[* TO 2015-01-01T00:00:00Z}",
+ self._date_range({"lt": "01/01/2015", "format": "dd/MM/yyyy"}))
+
+ def test_only_the_bound_that_is_used_is_rounded(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-01T00:00:00Z TO 2015-01-22T00:00:00Z}",
+ self._date_range({"gte": "01/01/2015", "gt": "05/01/2015",
+ "lte": "21/01/2015", "lt": "10/01/2015",
+ "format": "dd/MM/yyyy"}))
+
+ def test_a_bound_that_is_not_a_date_keeps_its_bracket(self):
+ for value in ("0000000010", "not-a-date", "2015-02-30"):
+ self.assertEqual(
+ f"serial_no:[0000000001 TO {value}]",
+ translate_to_solr_json_dsl(
+ {"query": {"range": {"serial_no": {"gte": "0000000001",
+ "lte":
value}}}})["query"])
+
+ def test_an_instant_is_not_rounded(self):
+ self.assertEqual(
+ "dropoff_datetime:[2015-01-01T00:00:00Z TO 2016-01-01T00:00:00Z]",
+ self._date_range({"gte": "2015-01-01T00:00:00Z",
+ "lte": "2016-01-01T00:00:00Z"}))
+ self.assertEqual(
+ "dropoff_datetime:[* TO 2016-01-01T00:00:00Z]",
+ self._date_range({"lte": "2016-01-01 00:00:00"}))
+
def test_bool_with_filter_goes_to_fq(self):
body = {
"query": {
