Copilot commented on code in PR #159:
URL: https://github.com/apache/solr-orbit/pull/159#discussion_r4656338878
##########
solrorbit/conversion/query.py:
##########
@@ -382,14 +363,135 @@ def translate_to_solr_json_dsl(body: dict) -> dict:
aggs = body.get("aggs") or body.get("aggregations")
if aggs and isinstance(aggs, dict):
- facets = _convert_aggregations_to_facets(aggs)
+ facets = _convert_aggregations_to_facets(aggs, body.get("query"))
if facets:
result["facet"] = facets
+ zone = _time_zone_of(aggs)
+ if zone and facets:
+ result["params"] = {"TZ": zone}
Review Comment:
OpenSearch's `time_zone` is per aggregation, but this selects the first zone
and applies it to the entire Solr request. Two calendar histograms using
`America/New_York` and `Europe/Paris` therefore both round in New York; even an
unzoned sibling's date gaps can inherit that zone. Please generate
independently zoned absolute bucket edges, or explicitly reject incompatible
aggregation zones instead of silently using the first one.
##########
solrorbit/conversion/query.py:
##########
@@ -382,14 +363,135 @@ def translate_to_solr_json_dsl(body: dict) -> dict:
aggs = body.get("aggs") or body.get("aggregations")
if aggs and isinstance(aggs, dict):
- facets = _convert_aggregations_to_facets(aggs)
+ facets = _convert_aggregations_to_facets(aggs, body.get("query"))
if facets:
result["facet"] = facets
+ zone = _time_zone_of(aggs)
+ if zone and facets:
+ result["params"] = {"TZ": zone}
return result
-def _convert_aggregations_to_facets(aggs: dict) -> dict:
+def _range_bounds(bounds: dict) -> tuple:
+ """The (low, low bracket, high, high bracket) Solr spells for an
OpenSearch range clause."""
+ # gt/lt are exclusive; Solr spells that with a curly bracket.
+ if "gte" in bounds:
+ lo, lo_bracket, lo_key = bounds["gte"], "[", "gte"
+ elif "gt" in bounds:
+ lo, lo_bracket, lo_key = bounds["gt"], "{", "gt"
+ else:
+ lo, lo_bracket, lo_key = "*", "[", None
+ if "lte" in bounds:
+ hi, hi_bracket, hi_key = bounds["lte"], "]", "lte"
+ elif "lt" in bounds:
+ hi, hi_bracket, hi_key = bounds["lt"], "}", "lt"
+ else:
+ hi, hi_bracket, hi_key = "*", "]", None
+ # Convert dates if format is specified (common for date fields)
+ os_format = bounds.get("format")
+ lo, lo_is_date_only = _convert_date_to_solr_format(lo, os_format)
+ hi, hi_is_date_only = _convert_date_to_solr_format(hi, os_format)
+ if hi_is_date_only and hi_key == "lte":
+ hi, hi_bracket = _round_date_only_bound(hi), "}"
+ if lo_is_date_only and lo_key == "gt":
+ lo, lo_bracket = _round_date_only_bound(lo), "["
+ return lo, lo_bracket, hi, hi_bracket
+
+
+def _date_bounds_for_field(query, field: str):
+ """
+ The (start, end) of the range the query puts on ``field``, or None.
+
+ OpenSearch derives a date_histogram's span from the data; a Solr range
facet needs it
+ spelled out. The query's own range on the aggregated field is the span the
buckets cover.
+ """
+ if not isinstance(query, dict):
+ return None
+ if "range" in query and isinstance(query["range"], dict):
+ for name, bounds in query["range"].items():
+ if normalize_field_name(name) == field and isinstance(bounds,
dict):
+ lo, _, hi, _ = _range_bounds(bounds)
+ if _SOLR_INSTANT.fullmatch(str(lo)) and
_SOLR_INSTANT.fullmatch(str(hi)):
+ return lo, hi
Review Comment:
Discarding the upper bracket loses the distinction between `lt` and an
instant-valued `lte`. For example, `gte: 2015-01-01T00:00:00Z`, `lte:
2015-01-02T00:00:00Z` with `fixed_interval: 1d` produces only the January 1
range: documents exactly at January 2 match the query but have no histogram
bucket. Please preserve upper-bound inclusivity and generate the additional
bucket when an inclusive upper bound lies on a bucket boundary, including in
explicit zoned ranges.
##########
solrorbit/conversion/query.py:
##########
@@ -382,14 +363,135 @@ def translate_to_solr_json_dsl(body: dict) -> dict:
aggs = body.get("aggs") or body.get("aggregations")
if aggs and isinstance(aggs, dict):
- facets = _convert_aggregations_to_facets(aggs)
+ facets = _convert_aggregations_to_facets(aggs, body.get("query"))
if facets:
result["facet"] = facets
+ zone = _time_zone_of(aggs)
+ if zone and facets:
+ result["params"] = {"TZ": zone}
return result
-def _convert_aggregations_to_facets(aggs: dict) -> dict:
+def _range_bounds(bounds: dict) -> tuple:
+ """The (low, low bracket, high, high bracket) Solr spells for an
OpenSearch range clause."""
+ # gt/lt are exclusive; Solr spells that with a curly bracket.
+ if "gte" in bounds:
+ lo, lo_bracket, lo_key = bounds["gte"], "[", "gte"
+ elif "gt" in bounds:
+ lo, lo_bracket, lo_key = bounds["gt"], "{", "gt"
+ else:
+ lo, lo_bracket, lo_key = "*", "[", None
+ if "lte" in bounds:
+ hi, hi_bracket, hi_key = bounds["lte"], "]", "lte"
+ elif "lt" in bounds:
+ hi, hi_bracket, hi_key = bounds["lt"], "}", "lt"
+ else:
+ hi, hi_bracket, hi_key = "*", "]", None
+ # Convert dates if format is specified (common for date fields)
+ os_format = bounds.get("format")
+ lo, lo_is_date_only = _convert_date_to_solr_format(lo, os_format)
+ hi, hi_is_date_only = _convert_date_to_solr_format(hi, os_format)
+ if hi_is_date_only and hi_key == "lte":
+ hi, hi_bracket = _round_date_only_bound(hi), "}"
+ if lo_is_date_only and lo_key == "gt":
+ lo, lo_bracket = _round_date_only_bound(lo), "["
+ return lo, lo_bracket, hi, hi_bracket
+
+
+def _date_bounds_for_field(query, field: str):
+ """
+ The (start, end) of the range the query puts on ``field``, or None.
+
+ OpenSearch derives a date_histogram's span from the data; a Solr range
facet needs it
+ spelled out. The query's own range on the aggregated field is the span the
buckets cover.
+ """
+ if not isinstance(query, dict):
+ return None
+ if "range" in query and isinstance(query["range"], dict):
+ for name, bounds in query["range"].items():
+ if normalize_field_name(name) == field and isinstance(bounds,
dict):
+ lo, _, hi, _ = _range_bounds(bounds)
+ if _SOLR_INSTANT.fullmatch(str(lo)) and
_SOLR_INSTANT.fullmatch(str(hi)):
+ return lo, hi
+ return None
+ bool_q = query.get("bool")
+ if isinstance(bool_q, dict):
+ for key in ("must", "filter"):
+ clauses = bool_q.get(key) or []
+ for clause in [clauses] if isinstance(clauses, dict) else clauses:
+ found = _date_bounds_for_field(clause, field)
+ if found:
+ return found
+ return None
+
+
+def _align_to_epoch(start: str, interval: str):
+ """
+ ``start`` moved down to a multiple of a fixed interval counted from the
epoch.
+
+ A fixed_interval's buckets are aligned to 1970-01-01, not to the start of
the range, so
+ ``60d`` over 2015 begins at 2014-11-06. A Solr range facet starts where it
is told to.
+ """
+ match = re.fullmatch(r"(\d*)(ms|[smhd])", str(interval).strip())
+ if not match:
+ return None
+ seconds = {"ms": 0.001, "s": 1, "m": 60, "h": 3600, "d":
86400}[match.group(2)]
+ step = int(match.group(1) or 1) * seconds
+ epoch = datetime(1970, 1, 1)
+ elapsed = (datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ") -
epoch).total_seconds()
+ return (epoch + timedelta(seconds=(elapsed // step) *
step)).strftime("%Y-%m-%dT%H:%M:%SZ")
Review Comment:
Millisecond intervals lose their alignment here: floating-point floor
division can move an exact boundary backward, and the seconds-only formatter
then truncates fractional milliseconds. For `2015-01-01T00:00:00Z`, `1ms`
returns `2014-12-31T23:59:59Z`; the already-supported `2699999ms` returns
`23:51:14Z` instead of `23:51:14.048Z`. Please align using integer milliseconds
and retain fractional precision in the emitted start.
##########
solrorbit/conversion/query.py:
##########
@@ -454,17 +564,45 @@ def _convert_single_agg(agg_name: str, agg_def: dict):
logger.warning("date_histogram agg '%s' has no field — skipping",
agg_name)
return None
gap = _date_histogram_gap(dh_conf, agg_name)
- facet_def = {
- "type": "range",
- "field": field,
- "gap": gap,
- "mincount": 1,
- "start": "NOW/YEAR-10YEAR",
- "end": "NOW/YEAR+1YEAR",
- }
+ bounds = _date_bounds_for_field(query, field)
+ if bounds:
+ start, end = bounds
+ if "fixed_interval" in dh_conf:
+ start = _align_to_epoch(start, dh_conf["fixed_interval"]) or
start
+ else:
+ logger.warning(
+ "date_histogram agg '%s' has no date range on '%s' in its
query: start and end "
+ "are placeholders that must be set to the data's range before
the operation runs.",
+ agg_name, field)
+ start, end = "REPLACE_WITH_START", "REPLACE_WITH_END"
+ zone = dh_conf.get("time_zone")
+ ranges, in_zone = None, False
+ if zone and bounds:
Review Comment:
A monthly histogram without `time_zone` starting at `2015-01-15T12:00:00Z`
produces buckets anchored on the 15th at noon, changing bucket keys and counts.
Week and quarter intervals also bypass rounding because they are absent from
`_CALENDAR_ROUNDING`. Align every supported calendar interval to its calendar
boundary, using UTC when no zone is specified.
##########
solrorbit/conversion/query.py:
##########
@@ -647,6 +785,9 @@ def _convert_date_to_solr_format(date_str, os_format=None)
-> tuple:
return date_str, False
+_SOLR_INSTANT = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z")
Review Comment:
This pattern rejects valid ISO instants with fractional seconds, such as
`2015-01-01T00:00:00.000Z`, even though `_convert_date_to_solr_format`
preserves them and the resulting range query is usable by Solr. A histogram
over such a fully bounded query receives
`REPLACE_WITH_START`/`REPLACE_WITH_END` and fails when run. Please accept
fractional-second instants and update the alignment and zoned-range parsers to
handle the same precision.
##########
solrorbit/conversion/query.py:
##########
@@ -419,14 +522,21 @@ def _convert_aggregations_to_facets(aggs: dict) -> dict:
for agg_name, agg_def in aggs.items():
if not isinstance(agg_def, dict):
continue
- entry = _convert_single_agg(agg_name, agg_def)
+ if "stats" in agg_def:
+ # `stats` is four numbers in OpenSearch and four function facets
in Solr: <name>_min ... <name>_avg.
+ field = normalize_field_name(agg_def["stats"].get("field", ""))
+ if field:
+ for stat in ("min", "max", "sum", "avg"):
+ result[f"{agg_name}_{stat}"] = f"{stat}({field})"
Review Comment:
The generated metric names can collide with existing sibling aggregation
names. For example, a `fare` stats aggregation alongside a `fare_avg` metric
writes the same result key twice, silently dropping one requested metric; the
winner depends on aggregation order. Detect conflicts with sibling and
generated names and report a conversion error rather than overwriting a facet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]