This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new fd524f4b217 fix(sqllab): preserve exact decimals in results and
exports (#44739)
fd524f4b217 is described below
commit fd524f4b217f2dfd5d5f75ca9109413edf9515d7
Author: Amin Ghadersohi <[email protected]>
AuthorDate: Thu Oct 1 12:48:43 2026 +1000
fix(sqllab): preserve exact decimals in results and exports (#44739)
---
UPDATING.md | 20 ++
docs/docs/using-superset/number-formatting.mdx | 16 ++
.../FilterableTable/exactDecimals.test.tsx | 92 ++++++++
.../src/components/FilterableTable/index.tsx | 41 +---
.../components/FilterableTable/sortResults.test.ts | 238 +++++++++++++++++++++
.../src/components/FilterableTable/sortResults.ts | 180 ++++++++++++++++
superset/commands/sql_lab/export.py | 14 ++
superset/commands/streaming_export/base.py | 11 +-
superset/common/query_context_processor.py | 1 +
superset/dataframe.py | 38 +++-
superset/result_set.py | 6 +
superset/sql/execution/celery_task.py | 2 +-
superset/sql_lab.py | 2 +-
superset/utils/csv.py | 10 +
superset/views/utils.py | 2 +-
tests/unit_tests/commands/sql_lab/export_test.py | 113 ++++++++++
.../sql_lab/streaming_export_command_test.py | 74 +++++++
.../common/test_query_context_processor.py | 12 ++
tests/unit_tests/dataframe_test.py | 83 ++++++-
tests/unit_tests/result_set_test.py | 71 ++++++
tests/unit_tests/sql/execution/test_celery_task.py | 2 +-
tests/unit_tests/sql_lab_test.py | 2 +-
22 files changed, 979 insertions(+), 51 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index da66b164f96..54c33402ad3 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -126,6 +126,26 @@ get the same handling. With MariaDB Connector/Python
ssl_verify_cert=True. Other MySQL-compatible engines such as OceanBase and
StarRocks keep their existing SSL handling.
+### SQL Lab decimal results use exact strings
+
+SQL Lab represents database `DECIMAL`/`NUMERIC` values as JSON strings instead
+of JSON numbers, preserving precision and trailing zeros in the results grid
+and exports. The strings use fixed-point notation, and non-finite `NaN` and
+`Infinity` decimals are returned as `null`. Numeric sorting still compares
+their exact values. API consumers performing arithmetic should parse these
+strings with a decimal library, not JavaScript `Number`. This includes SQL Lab
+extensions: the `data` rows passed to `sqlLab.onDidQuerySuccess` listeners
+carry strings for `numeric` columns (for example PostgreSQL `SUM(bigint)`,
+`AVG` or `ROUND` results), so adding them with `+` concatenates instead of
+summing. The `sqleditor.extension.resultTable` override, which replaces
+`FilterableTable` in both SQL Lab results and table previews, also receives
+string rows but not the built-in exact-decimal comparator; extensions must
+implement their own decimal sorting. Chart and dataset queries also use
fixed-point strings for Decimal
+columns that fall back to string conversion (for example, Decimals mixed with
+floats), and non-finite Decimals in those columns become null. Other chart
+result serialization and chart number formatting are unchanged. Re-run queries
+whose cached JSON results were produced before upgrading all workers.
+
### Version history retention setting
Use `VERSION_HISTORY_RETENTION_DAYS` for both the application setting and
diff --git a/docs/docs/using-superset/number-formatting.mdx
b/docs/docs/using-superset/number-formatting.mdx
index a3be77b05d5..d6e05634eeb 100644
--- a/docs/docs/using-superset/number-formatting.mdx
+++ b/docs/docs/using-superset/number-formatting.mdx
@@ -90,3 +90,19 @@ Use these when a metric's underlying values are stored in
meters or centimeters
## Currency
Some chart types also expose currency-specific formatting, including a dynamic
mode that reads the currency from a column value. See [Dynamic Currency
Formatting](./creating-your-first-dashboard#dynamic-currency-formatting) for
details.
+
+## Exact decimals in SQL Lab
+
+SQL Lab displays database `DECIMAL` and `NUMERIC` results as exact fixed-point
+text, including trailing zeros and never in scientific notation. Its JSON
+responses use strings so a browser does not round values such as
+`12345678901234567890.123456789012345678`. Sorting remains numeric and
+compares all digits; CSV exports, including streaming exports, retain those
+digits and use fixed-point notation for finite decimals too. Extensions
overriding
+`sqleditor.extension.resultTable` replace the built-in table in SQL Lab results
+and table previews. They receive decimal strings but not the built-in
comparator,
+so they must implement their own exact-decimal sorting. Chart and dataset
+queries share the result-set loader: Decimal columns that fall back to strings
+(for example, Decimals mixed with floats) also use fixed-point notation, with
+non-finite Decimals converted to null. Other chart result serialization and
+chart number formatting are unchanged.
diff --git
a/superset-frontend/src/components/FilterableTable/exactDecimals.test.tsx
b/superset-frontend/src/components/FilterableTable/exactDecimals.test.tsx
new file mode 100644
index 00000000000..1a091f9c7a6
--- /dev/null
+++ b/superset-frontend/src/components/FilterableTable/exactDecimals.test.tsx
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import {
+ render,
+ screen,
+ userEvent,
+ within,
+} from 'spec/helpers/testing-library';
+import { setupAGGridModules } from
'@superset-ui/core/components/ThemedAgGridReact';
+import { FilterableTable } from '.';
+import { getCachedSortKey } from './sortResults';
+
+test('displays and sorts exact decimal strings without rounding', async () => {
+ setupAGGridModules();
+ const smaller = '12345678901234567890.123456789012345678';
+ const larger = '12345678901234567890.123456789012345679';
+ const ascending = ['-10.50', '0.00', '2.00', '10.50', smaller, larger];
+ render(
+ <FilterableTable
+ orderedColumnKeys={['amount']}
+ data={[larger, smaller, '10.50', '2.00', '0.00', '-10.50'].map(
+ amount => ({ amount }),
+ )}
+ height={500}
+ />,
+ );
+ for (const value of ascending) {
+ expect(screen.getByText(value)).toBeInTheDocument();
+ }
+ const header = within(screen.getByRole('grid'))
+ .getByText('amount')
+ .closest('[role=button]');
+ expect(header).not.toBeNull();
+ const values = () =>
+ Array.from(
+ document.querySelectorAll('[role="gridcell"][col-id="amount"]'),
+ ).map(cell => cell.textContent);
+ await userEvent.click(header!);
+ expect(values()).toEqual(ascending);
+ await userEvent.click(header!);
+ expect(values()).toEqual([...ascending].reverse());
+});
+
+test('uses the new rows array for sort-key caching when data changes', async
() => {
+ setupAGGridModules();
+ const orderedColumnKeys = ['amount'];
+ const data = [{ amount: '10.50' }, { amount: '2.00' }];
+ const { rerender } = render(
+ <FilterableTable
+ orderedColumnKeys={orderedColumnKeys}
+ data={data}
+ height={500}
+ />,
+ );
+ const header = within(screen.getByRole('grid'))
+ .getByText('amount')
+ .closest('[role=button]');
+ expect(header).not.toBeNull();
+ await userEvent.click(header!);
+ expect(getCachedSortKey('10.50', data)).toBeDefined();
+
+ const nextData = [{ amount: '20.50' }, { amount: '3.00' }];
+ rerender(
+ <FilterableTable
+ orderedColumnKeys={orderedColumnKeys}
+ data={nextData}
+ height={500}
+ />,
+ );
+ expect(
+ Array.from(
+ document.querySelectorAll('[role="gridcell"][col-id="amount"]'),
+ ).map(cell => cell.textContent),
+ ).toEqual(['3.00', '20.50']);
+ expect(getCachedSortKey('20.50', nextData)).toBeDefined();
+});
diff --git a/superset-frontend/src/components/FilterableTable/index.tsx
b/superset-frontend/src/components/FilterableTable/index.tsx
index 492f42e4f40..a2c1bcff5ca 100644
--- a/superset-frontend/src/components/FilterableTable/index.tsx
+++ b/superset-frontend/src/components/FilterableTable/index.tsx
@@ -25,37 +25,7 @@ import { renderResultCell } from './utils';
import type { FilterableTableProps, Datum, CellDataType } from './types';
-// This regex handles all possible number formats in javascript, including
ints, floats,
-// exponential notation, NaN, and Infinity.
-// See https://stackoverflow.com/a/30987109 for more details
-const ONLY_NUMBER_REGEX = /^(NaN|-?((\d*\.\d+|\d+)([Ee][+-]?\d+)?|Infinity))$/;
-
-const parseNumberFromString = (value: string | number | null) => {
- if (typeof value === 'string' && ONLY_NUMBER_REGEX.test(value)) {
- return parseFloat(value);
- }
- return value;
-};
-
-const sortResults = (valueA: string | number, valueB: string | number) => {
- const aValue = parseNumberFromString(valueA);
- const bValue = parseNumberFromString(valueB);
-
- // equal items sort equally
- if (aValue === bValue) {
- return 0;
- }
-
- // nulls sort after anything else
- if (aValue === null) {
- return 1;
- }
- if (bValue === null) {
- return -1;
- }
-
- return aValue < bValue ? -1 : 1;
-};
+import { sortResults } from './sortResults';
export const FilterableTable = ({
orderedColumnKeys,
@@ -91,6 +61,11 @@ export const FilterableTable = ({
return values.some(v => v.includes(lowerCaseText));
};
+ const comparator = useCallback(
+ (a: CellDataType, b: CellDataType) => sortResults(a, b, data),
+ [data],
+ );
+
const columns = useMemo(
() =>
orderedColumnKeys.map(key => ({
@@ -98,7 +73,7 @@ export const FilterableTable = ({
label: key,
fieldName: key,
headerName: key,
- comparator: sortResults,
+ comparator,
render: ({ value, colDef }: { value: CellDataType; colDef: ColDef }) =>
renderResultCell({
cellData: value,
@@ -107,7 +82,7 @@ export const FilterableTable = ({
getCellContent,
}),
})),
- [orderedColumnKeys, allowHTML, getCellContent],
+ [orderedColumnKeys, allowHTML, getCellContent, comparator],
);
const keywordFilter = useCallback(
diff --git
a/superset-frontend/src/components/FilterableTable/sortResults.test.ts
b/superset-frontend/src/components/FilterableTable/sortResults.test.ts
new file mode 100644
index 00000000000..d003e5e33b6
--- /dev/null
+++ b/superset-frontend/src/components/FilterableTable/sortResults.test.ts
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { decimalParts, getCachedSortKey, sortResults } from './sortResults';
+
+test.each([
+ ['2', '10', -1],
+ [
+ '12345678901234567890.123456789012345678',
+ '12345678901234567890.123456789012345679',
+ -1,
+ ],
+ [
+ '-12345678901234567890.123456789012345678',
+ '-12345678901234567890.123456789012345679',
+ 1,
+ ],
+ ['1E-18', '0.000000000000000002', -1],
+ ['1E+100000', '9E+99999', 1],
+ ['1E-100000', '9E-100001', 1],
+ ['0.00000000000000000001', '0', 1],
+ ['0010.500', 10.5, 0],
+ ['-0.00', 0, 0],
+ ['.5', '0.50', 0],
+ ['9.99', 10, -1],
+ [null, '1.23', 1],
+ ['1.23', null, -1],
+ [null, null, 0],
+ ['apple', 'pear', -1],
+ ['2024-01-01', '2025-01-01', -1],
+ ['-Infinity', '-1.2', -1],
+ ['Infinity', '1.2', 1],
+] as const)('sorts %s vs %s exactly', (a, b, expected) => {
+ expect(sortResults(a, b)).toBe(expected);
+});
+
+test.each([
+ ['apple', 5],
+ ['apple', 'NaN'],
+ [NaN, 1.5],
+ ['2024-01-01', 7],
+] as const)('orders %s against %s in one direction only', (a, b) => {
+ expect(sortResults(a, b)).toBe(-sortResults(b, a));
+});
+
+test('treats two NaNs as equal rather than unordered', () => {
+ expect(sortResults(NaN, NaN)).toBe(0);
+ expect(sortResults('NaN', 'NaN')).toBe(0);
+});
+
+test('sorts a mixed text and number column transitively', () => {
+ const column = [5, 'apple', 'NaN', 10, 'banana'];
+ const ascending = [...column].sort(sortResults);
+ const descending = [...column].reverse().sort(sortResults);
+ expect(descending).toEqual(ascending);
+});
+
+test.each([
+ [[9, '5x', 10]],
+ [[10, 9, '5x']],
+ [['5x', 10, 9]],
+ [['9', '5x', '10']],
+ [['10', '9', '5x']],
+ [['5x', '10', '9']],
+])('sorts numbers before text whatever the input order: %p', column => {
+ const sorted = [...column].sort(sortResults).map(String);
+ expect(sorted).toEqual(['9', '10', '5x']);
+});
+
+test('orders numbers, then text, then NaN, then nulls', () => {
+ const column = [
+ 'NaN',
+ null,
+ 'apple',
+ 3,
+ NaN,
+ '2.5',
+ 'Infinity',
+ '-Infinity',
+ -Infinity,
+ '10',
+ ];
+ expect([...column].sort(sortResults)).toEqual([
+ '-Infinity',
+ -Infinity,
+ '2.5',
+ 3,
+ '10',
+ 'Infinity',
+ 'apple',
+ 'NaN',
+ NaN,
+ null,
+ ]);
+});
+
+test('is a consistent total order over mixed values', () => {
+ const values = [
+ null,
+ NaN,
+ 'NaN',
+ -Infinity,
+ '-Infinity',
+ Infinity,
+ 'Infinity',
+ -1.5,
+ '-1.50',
+ 0,
+ -0,
+ '0.000',
+ '1E-18',
+ 0.1,
+ '0.1',
+ 9,
+ '10',
+ 1e21,
+ '1E+21',
+ '',
+ '5x',
+ 'apple',
+ '2024-01-01',
+ ];
+ const sign = (n: number) => Math.sign(n) || 0;
+ values.forEach(a => {
+ expect(sortResults(a, a)).toBe(0);
+ values.forEach(b => {
+ const ab = sign(sortResults(a, b));
+ expect(sign(sortResults(b, a))).toBe(-ab || 0);
+ values.forEach(c => {
+ const bc = sign(sortResults(b, c));
+ if (ab <= 0 && bc <= 0) {
+ expect(sign(sortResults(a, c))).toBeLessThanOrEqual(0);
+ }
+ });
+ });
+ });
+});
+
+test.each([
+ [1, 2, -1],
+ [2, 1, 1],
+ [-0, 0, 0],
+ [-Infinity, -1e308, -1],
+ [Infinity, 1e308, 1],
+ [9007199254740993, 9007199254740992, 0],
+ [0.1 + 0.2, 0.3, 1],
+] as const)('compares plain numbers %p vs %p directly', (a, b, expected) => {
+ expect(sortResults(a, b)).toBe(expected);
+});
+
+test.each([
+ [0.1, '0.1'],
+ [1e21, '1E+21'],
+ [1e-7, '0.0000001'],
+ [-2.5, '-2.50'],
+ [Infinity, 'Infinity'],
+])('orders number %p and its exact string %p equally', (number, text) => {
+ expect(sortResults(number, text)).toBe(0);
+ expect(sortResults(text, number)).toBe(0);
+});
+
+// Reference regex the linear parser must agree with.
+const REFERENCE_DECIMAL =
+ /^([+-]?)(?:(\d+)(?:\.(\d*))?|\.(\d+))(?:[eE]([+-]?\d+))?$/;
+
+function strings(alphabet: string[], maxLength: number): string[] {
+ const all = [''];
+ let level = [''];
+ for (let length = 1; length <= maxLength; length += 1) {
+ level = level.flatMap(prefix => alphabet.map(token => prefix + token));
+ level.forEach(text => all.push(text));
+ }
+ return all;
+}
+
+test('linear parser accepts exactly what the reference regex matches', () => {
+ const alphabet = ['0', '7', '.', 'e', 'E', '+', '-', 'x', 'NaN', 'Infinity'];
+ const mismatches: string[] = [];
+ const inputs = strings(alphabet, 5);
+ expect(inputs).toHaveLength(111111);
+ expect(inputs).toEqual(
+ expect.arrayContaining(['NaN', 'Infinity', '-Infinity']),
+ );
+ inputs.forEach(text => {
+ if ((decimalParts(text) !== null) !== REFERENCE_DECIMAL.test(text)) {
+ mismatches.push(`decimal: ${text}`);
+ }
+ });
+ expect(mismatches).toEqual([]);
+});
+
+test('rejects long adversarial near-miss inputs', () => {
+ const inputs = [
+ `${'1'.repeat(200000)}x`,
+ `.${'1'.repeat(200000)}.`,
+ `1e${'1'.repeat(200000)}e`,
+ `-${'1'.repeat(100000)}.${'1'.repeat(100000)}E+`,
+ ];
+ inputs.forEach(text => {
+ expect(decimalParts(text)).toBeNull();
+ });
+});
+
+test('reuses sort keys only within the same result set', () => {
+ const rows = [{ value: '1.25' }, { value: '2.50' }];
+ expect(getCachedSortKey('1.25', rows)).toBeUndefined();
+ expect(sortResults('1.25', '2.50', rows)).toBe(-1);
+ const firstKey = getCachedSortKey('1.25', rows);
+ const secondKey = getCachedSortKey('2.50', rows);
+ expect(firstKey).toBeDefined();
+ expect(secondKey).toBeDefined();
+ expect(sortResults('2.50', '1.25', rows)).toBe(1);
+ expect(getCachedSortKey('1.25', rows)).toBe(firstKey);
+ expect(getCachedSortKey('2.50', rows)).toBe(secondKey);
+
+ const otherRows = [...rows];
+ expect(getCachedSortKey('1.25', otherRows)).toBeUndefined();
+ expect(sortResults('1.25', '2.50', otherRows)).toBe(-1);
+ expect(getCachedSortKey('1.25', otherRows)).toEqual(firstKey);
+ expect(getCachedSortKey('1.25', otherRows)).not.toBe(firstKey);
+ expect(sortResults('1.25', '2.50')).toBe(-1);
+ expect(getCachedSortKey('1.25', rows)).toBe(firstKey);
+});
diff --git a/superset-frontend/src/components/FilterableTable/sortResults.ts
b/superset-frontend/src/components/FilterableTable/sortResults.ts
new file mode 100644
index 00000000000..b93240f2844
--- /dev/null
+++ b/superset-frontend/src/components/FilterableTable/sortResults.ts
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+type CellValue = string | number | null;
+
+// Every pattern below is a single flat run of digits, so matching stays linear
+// in the input length; the optional parts are split off by hand instead of
+// nesting quantified groups, which could backtrack catastrophically.
+const DIGITS = /^\d*$/;
+const EXPONENT = /^[+-]?\d+$/;
+
+/** Split `text` at its first `e`/`E`; `null` when the exponent is malformed.
*/
+function splitExponent(text: string): [string, string | undefined] | null {
+ const at = text.search(/[eE]/);
+ if (at === -1) return [text, undefined];
+ const exponent = text.slice(at + 1);
+ return EXPONENT.test(exponent) ? [text.slice(0, at), exponent] : null;
+}
+
+// Compare significands and decimal orders, never coerce exact strings to
Number.
+// Keeping the exponent separate also avoids allocating 10**exponent zeroes.
+export function decimalParts(value: Exclude<CellValue, null>) {
+ const text = String(value);
+ const signed = text[0] === '+' || text[0] === '-';
+ const parts = splitExponent(signed ? text.slice(1) : text);
+ if (!parts) return null;
+ const [mantissa, exponent] = parts;
+ const dot = mantissa.indexOf('.');
+ const integer = dot === -1 ? mantissa : mantissa.slice(0, dot);
+ const fraction = dot === -1 ? '' : mantissa.slice(dot + 1);
+ if (
+ !(integer || fraction) ||
+ !DIGITS.test(integer) ||
+ !DIGITS.test(fraction)
+ ) {
+ return null;
+ }
+ const digits = `${integer}${fraction}`.replace(/^0+/, '');
+ return {
+ sign: digits ? (text[0] === '-' ? -1 : 1) : 0,
+ digits,
+ order:
+ BigInt(digits.length) + BigInt(exponent ?? '0') -
BigInt(fraction.length),
+ };
+}
+
+// Values sort by type first, so mixed columns stay transitive: numbers (exact
+// decimal strings and infinities included) before text, and NaN after both.
+const NUMERIC_RANK = 0;
+const TEXT_RANK = 1;
+const NAN_RANK = 2;
+
+type SortKey =
+ | {
+ rank: typeof NUMERIC_RANK;
+ // -1 or 1 for an infinity, 0 for a finite value.
+ infinity: number;
+ sign: number;
+ digits: string;
+ order: bigint;
+ }
+ | { rank: typeof TEXT_RANK; text: string }
+ | { rank: typeof NAN_RANK };
+
+function toSortKey(value: Exclude<CellValue, null>): SortKey {
+ if (typeof value === 'number' ? Number.isNaN(value) : value === 'NaN') {
+ return { rank: NAN_RANK };
+ }
+ if (value === Infinity || value === 'Infinity') {
+ return {
+ rank: NUMERIC_RANK,
+ infinity: 1,
+ sign: 1,
+ digits: '',
+ order: BigInt(0),
+ };
+ }
+ if (value === -Infinity || value === '-Infinity') {
+ return {
+ rank: NUMERIC_RANK,
+ infinity: -1,
+ sign: -1,
+ digits: '',
+ order: BigInt(0),
+ };
+ }
+ const parts = decimalParts(value);
+ return parts
+ ? { rank: NUMERIC_RANK, infinity: 0, ...parts }
+ : { rank: TEXT_RANK, text: String(value) };
+}
+
+// Parsing dominates the cost of sorting decimal strings, and a sort compares
+// each cell O(log n) times. Weak row-array keys let cached values be collected
+// with their result set instead of retaining them for the life of the page.
+const SORT_KEY_CACHE_LIMIT = 250_000;
+const sortKeyCaches = new WeakMap<
+ readonly unknown[],
+ Map<Exclude<CellValue, null>, SortKey>
+>();
+
+/** @internal Inspect cache reuse without coupling tests to parser details. */
+export function getCachedSortKey(
+ value: Exclude<CellValue, null>,
+ rows: readonly unknown[],
+): SortKey | undefined {
+ return sortKeyCaches.get(rows)?.get(value);
+}
+
+function sortKey(
+ value: Exclude<CellValue, null>,
+ rows?: readonly unknown[],
+): SortKey {
+ if (!rows) return toSortKey(value);
+ let sortKeyCache = sortKeyCaches.get(rows);
+ if (!sortKeyCache) {
+ sortKeyCache = new Map();
+ sortKeyCaches.set(rows, sortKeyCache);
+ }
+ let key = sortKeyCache.get(value);
+ if (key === undefined) {
+ if (sortKeyCache.size >= SORT_KEY_CACHE_LIMIT) sortKeyCache.clear();
+ key = toSortKey(value);
+ sortKeyCache.set(value, key);
+ }
+ return key;
+}
+
+function compareKeys(a: SortKey, b: SortKey): number {
+ if (a.rank !== b.rank) return a.rank < b.rank ? -1 : 1;
+ if (a.rank === TEXT_RANK && b.rank === TEXT_RANK) {
+ return a.text === b.text ? 0 : a.text < b.text ? -1 : 1;
+ }
+ if (a.rank !== NUMERIC_RANK || b.rank !== NUMERIC_RANK) return 0;
+ if (a.infinity !== b.infinity) return a.infinity < b.infinity ? -1 : 1;
+ if (a.infinity !== 0) return 0;
+ if (a.sign !== b.sign) return a.sign < b.sign ? -1 : 1;
+ if (a.sign === 0) return 0;
+ if (a.order !== b.order) return (a.order < b.order ? -1 : 1) * a.sign;
+ const width = Math.max(a.digits.length, b.digits.length);
+ const left = a.digits.padEnd(width, '0');
+ const right = b.digits.padEnd(width, '0');
+ return left === right ? 0 : (left < right ? -1 : 1) * a.sign;
+}
+
+export function sortResults(
+ valueA: CellValue,
+ valueB: CellValue,
+ rows?: readonly unknown[],
+): number {
+ // Plain JavaScript numbers need no parsing; their order agrees with the
+ // exact decimal order of their shortest representations.
+ if (
+ typeof valueA === 'number' &&
+ typeof valueB === 'number' &&
+ !Number.isNaN(valueA) &&
+ !Number.isNaN(valueB)
+ ) {
+ return valueA === valueB ? 0 : valueA < valueB ? -1 : 1;
+ }
+ if (valueA === valueB) return 0;
+ if (valueA === null) return 1;
+ if (valueB === null) return -1;
+ return compareKeys(sortKey(valueA, rows), sortKey(valueB, rows));
+}
diff --git a/superset/commands/sql_lab/export.py
b/superset/commands/sql_lab/export.py
index 93aea9d58c6..4b42cbc880d 100644
--- a/superset/commands/sql_lab/export.py
+++ b/superset/commands/sql_lab/export.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import logging
+from decimal import Decimal
from typing import Any, cast, TypedDict
import pandas as pd
@@ -37,6 +38,17 @@ from superset.views.utils import _deserialize_results_payload
logger = logging.getLogger(__name__)
+def _format_decimal_columns(df: pd.DataFrame) -> None:
+ """Format decimal object columns without changing other values or
dtypes."""
+ for name, column in df.items():
+ if pd.api.types.is_object_dtype(column.dtype) and any(
+ isinstance(value, Decimal) and value.is_finite() for value in
column
+ ):
+ # A finite decimal becomes a string, keeping map's result
object-typed
+ # so mixed floats, integers and None do not undergo dtype
inference.
+ df[name] = column.map(csv.format_decimal)
+
+
class SqlExportResult(TypedDict):
query: Query
count: int
@@ -140,6 +152,8 @@ class SqlResultExportCommand(BaseCommand):
self._query.schema,
)[:limit]
+ _format_decimal_columns(df)
+
# Manual encoding using the specified encoding (default to utf-8 if
not set)
csv_string = csv.df_to_escaped_csv(df, index=False,
**app.config["CSV_EXPORT"])
csv_data = csv_string.encode(app.config["CSV_EXPORT"].get("encoding",
"utf-8"))
diff --git a/superset/commands/streaming_export/base.py
b/superset/commands/streaming_export/base.py
index fbed887ecbc..d5814b1908f 100644
--- a/superset/commands/streaming_export/base.py
+++ b/superset/commands/streaming_export/base.py
@@ -34,7 +34,7 @@ from werkzeug.local import LocalProxy
from superset import db
from superset.commands.base import BaseCommand
-from superset.utils.csv import escape_value
+from superset.utils.csv import escape_value, format_decimal
logger = logging.getLogger(__name__)
@@ -192,10 +192,17 @@ class BaseStreamingCSVExportCommand(BaseCommand):
# (float, decimal.Decimal, numpy numeric types, ...). Booleans are
# technically a numeric type in Python but should never be
rewritten
# as numbers in CSV output.
+ elif isinstance(value, Decimal):
+ decimal_value = format_decimal(value)
+ if active_decimal_separator is not None:
+ decimal_value = str(decimal_value).replace(
+ ".", active_decimal_separator
+ )
+ formatted.append(decimal_value)
elif isinstance(value, bool):
formatted.append(value)
elif active_decimal_separator is not None and isinstance(
- value, (float, Decimal, Real)
+ value, (float, Real)
):
# Format numeric values with custom decimal separator
formatted.append(str(value).replace(".",
active_decimal_separator))
diff --git a/superset/common/query_context_processor.py
b/superset/common/query_context_processor.py
index 5f98c82ad0b..cc3ee36640b 100644
--- a/superset/common/query_context_processor.py
+++ b/superset/common/query_context_processor.py
@@ -592,6 +592,7 @@ class QueryContextProcessor:
# QueryObject post-processing has completed before this materialization
# boundary. Canonicalize its trusted missing/non-finite scalar outputs,
# while downstream envelope validation still rejects injected infinity.
+ # Chart consumers do arithmetic on decimals, so keep them JSON numbers.
return df_to_records(df)
@staticmethod
diff --git a/superset/dataframe.py b/superset/dataframe.py
index 12e24a69985..16dfe35abdd 100644
--- a/superset/dataframe.py
+++ b/superset/dataframe.py
@@ -85,8 +85,27 @@ def _is_trusted_missing_or_nonfinite(value: Any) -> bool:
return False
+def _convert_decimals(value: Any) -> Any:
+ """Keep SQL Lab decimals exact across JSON, MessagePack and browser
parsing."""
+ # Match exact types only, as below, so object-column values never get to
run
+ # overridden ``items``/``__iter__`` hooks during conversion.
+ value_type = type(value)
+ if value_type is Decimal:
+ return format(value, "f") if Decimal.is_finite(value) else None
+ if value_type is dict:
+ return {key: _convert_decimals(item) for key, item in
dict.items(value)}
+ if value_type is list:
+ return [_convert_decimals(item) for item in value]
+ if value_type is tuple:
+ return tuple(_convert_decimals(item) for item in value)
+ return value
+
+
def df_to_records(
- dframe: pd.DataFrame, *, convert_big_integers: bool = True
+ dframe: pd.DataFrame,
+ *,
+ convert_big_integers: bool = True,
+ convert_decimals: bool = False,
) -> list[dict[str, Any]]:
"""
Convert a DataFrame to a set of records.
@@ -97,6 +116,8 @@ def df_to_records(
:param dframe: the DataFrame to convert
:param convert_big_integers: whether integers outside JavaScript's safe
range
should be represented as strings
+ :param convert_decimals: whether ``Decimal`` values should be represented
as
+ exact strings rather than JSON numbers
:returns: a list of dictionaries reflecting each single row of the
DataFrame
"""
if not dframe.columns.is_unique:
@@ -118,12 +139,13 @@ def df_to_records(
# converting it to float can overflow or lose precision.
if type(value) in _NUMPY_NATIVE_TYPES:
value = value.item()
- dict.__setitem__(
- record,
- key,
- None
- if _is_trusted_missing_or_nonfinite(value)
- else (_convert_big_integers(value) if convert_big_integers
else value),
- )
+ if _is_trusted_missing_or_nonfinite(value):
+ value = None
+ else:
+ if convert_big_integers:
+ value = _convert_big_integers(value)
+ if convert_decimals:
+ value = _convert_decimals(value)
+ dict.__setitem__(record, key, value)
return records
diff --git a/superset/result_set.py b/superset/result_set.py
index e7eb527df30..b39473bfe33 100644
--- a/superset/result_set.py
+++ b/superset/result_set.py
@@ -18,6 +18,7 @@
import datetime
import logging
+from decimal import Decimal
from typing import Any, Optional
import numpy as np
@@ -88,6 +89,11 @@ def stringify_values(array: NDArray[Any]) -> NDArray[Any]:
# Non-JSON-serializable value (e.g. bytes, custom
# objects): fall back to str() to avoid crashing.
obj[...] = str(val)
+ elif isinstance(val, Decimal):
+ # str() switches to scientific notation for small or
+ # large exponents (-1.0E-7), which CSV export would
+ # formula-escape. NaN and Infinity have no exact value.
+ obj[...] = format(val, "f") if val.is_finite() else
None
else:
# for simple string conversions
# this handles odd character types better
diff --git a/superset/sql/execution/celery_task.py
b/superset/sql/execution/celery_task.py
index 7b56db8e4cc..bab85e025e1 100644
--- a/superset/sql/execution/celery_task.py
+++ b/superset/sql/execution/celery_task.py
@@ -323,7 +323,7 @@ def _serialize_result_set(
data = write_ipc_buffer(result_set.pa_table).to_pybytes()
else:
df = result_set.to_pandas_df()
- data = df_to_records(df) or []
+ data = df_to_records(df, convert_decimals=True) or []
return (data, result_set.columns)
diff --git a/superset/sql_lab.py b/superset/sql_lab.py
index 4fe37ccb983..ad64b27b7e0 100644
--- a/superset/sql_lab.py
+++ b/superset/sql_lab.py
@@ -424,7 +424,7 @@ def _serialize_and_expand_data(
all_columns, expanded_columns = (selected_columns, [])
else:
df = result_set.to_pandas_df()
- data = df_to_records(df) or []
+ data = df_to_records(df, convert_decimals=True) or []
if expand_data:
all_columns, data, expanded_columns = db_engine_spec.expand_data(
diff --git a/superset/utils/csv.py b/superset/utils/csv.py
index c0c9b10dcc8..bcb79a2c699 100644
--- a/superset/utils/csv.py
+++ b/superset/utils/csv.py
@@ -17,6 +17,7 @@
import logging
import urllib.request
from contextlib import closing
+from decimal import Decimal
from typing import Any, Optional, Union
from urllib.error import URLError
@@ -85,6 +86,15 @@ def escape_value(value: str) -> str:
return value
+def format_decimal(value: Any) -> Any:
+ """Return finite decimals as exact fixed-point text; preserve other
values."""
+ return (
+ format(value, "f")
+ if isinstance(value, Decimal) and value.is_finite()
+ else value
+ )
+
+
def df_to_escaped_csv(df: pd.DataFrame, **kwargs: Any) -> Any:
def escape_values(v: Any) -> Union[str, Any]:
return escape_value(v) if isinstance(v, str) else v
diff --git a/superset/views/utils.py b/superset/views/utils.py
index 63bb7037586..efaf14abf66 100644
--- a/superset/views/utils.py
+++ b/superset/views/utils.py
@@ -665,7 +665,7 @@ def _deserialize_results_payload(
raise SerializationError("Unable to deserialize table") from ex
df = result_set.SupersetResultSet.convert_table_to_df(pa_table)
- ds_payload["data"] = dataframe.df_to_records(df) or []
+ ds_payload["data"] = dataframe.df_to_records(df,
convert_decimals=True) or []
for column in ds_payload["selected_columns"]:
if "name" in column:
diff --git a/tests/unit_tests/commands/sql_lab/export_test.py
b/tests/unit_tests/commands/sql_lab/export_test.py
new file mode 100644
index 00000000000..22537969efb
--- /dev/null
+++ b/tests/unit_tests/commands/sql_lab/export_test.py
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for SQL Lab CSV export."""
+
+from decimal import Decimal
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pytest
+from pytest_mock import MockerFixture
+
+from superset.commands.sql_lab.export import SqlResultExportCommand
+from superset.utils import csv
+
+
[email protected](
+ ("value", "expected"),
+ [
+ ("0E-18", "0.000000000000000000"),
+ ("-1E-7", "-0.0000001"),
+ ("1E+30", "1000000000000000000000000000000"),
+ ("12.3400", "12.3400"),
+ ("NaN", ""),
+ ("Infinity", "Infinity"),
+ ("-Infinity", "-Infinity"),
+ ],
+)
+def test_csv_export_without_results_backend_preserves_decimals(
+ mocker: MockerFixture, value: str, expected: str
+) -> None:
+ """Re-executed query results use exact fixed-point decimal text."""
+ mocker.patch("superset.commands.sql_lab.export.results_backend", None)
+ mocker.patch.dict("superset.commands.sql_lab.export.app.config",
{"CSV_EXPORT": {}})
+ csv_spy = mocker.spy(csv, "df_to_escaped_csv")
+ decimal_value = Decimal(value)
+ query = MagicMock()
+ query.select_sql = "SELECT amount FROM numbers"
+ query.database.get_df.return_value = pd.DataFrame(
+ {"amount": [decimal_value], "other": ["unchanged"]}, dtype=object
+ )
+ mock_db = mocker.patch("superset.commands.sql_lab.export.db")
+ query_result = mock_db.session.query.return_value.filter_by.return_value
+ query_result.one_or_none.return_value = query
+
+ result = SqlResultExportCommand("client_id").run()
+
+ assert result["data"] == f"amount,other\n{expected},unchanged\n".encode()
+ assert result["count"] == 1
+ query.database.get_df.assert_called_once_with(
+ query.select_sql, query.catalog, query.schema
+ )
+ if not decimal_value.is_finite():
+ assert csv_spy.call_args.args[0].at[0, "amount"] is decimal_value
+
+
[email protected]("decimal_separator", [".", ","])
+def test_csv_export_mixed_frame_is_byte_identical(
+ mocker: MockerFixture, decimal_separator: str
+) -> None:
+ """Column mapping preserves mixed values and untouched numeric columns."""
+ mocker.patch("superset.commands.sql_lab.export.results_backend", None)
+ mocker.patch.dict(
+ "superset.commands.sql_lab.export.app.config",
+ {"CSV_EXPORT": {"sep": ";", "decimal": decimal_separator}},
+ )
+ query: MagicMock = MagicMock()
+ query.select_sql = "SELECT * FROM numbers"
+ frame: pd.DataFrame = pd.DataFrame(
+ {
+ "mixed": [Decimal("12.3400"), 1.25, "=formula", None,
Decimal("NaN")],
+ "float": [1.25, 2.5, 3.75, 4.0, 5.0],
+ "nullable": pd.Series([1, None, 3, 4, 5], dtype=object),
+ },
+ )
+ frame.index = [10, 20, 30, 40, 50]
+ query.database.get_df.return_value = frame
+ mock_db: MagicMock = mocker.patch("superset.commands.sql_lab.export.db")
+ query_result: MagicMock =
mock_db.session.query.return_value.filter_by.return_value
+ query_result.one_or_none.return_value = query
+ formatter: MagicMock = mocker.spy(csv, "format_decimal")
+
+ result = SqlResultExportCommand("client_id").run()
+
+ expected: str = (
+ "mixed;float;nullable\n"
+ "12.3400;1.25;1\n"
+ "1.25;2.5;\n"
+ "'=formula;3.75;3\n"
+ ";4.0;4\n"
+ ";5.0;5\n"
+ )
+ if decimal_separator == ",":
+ # Pandas applies the separator to float columns, not mixed object
cells.
+ expected = expected.replace(";1.25;", ";1,25;").replace(";2.5;",
";2,5;")
+ expected = expected.replace(";3.75;", ";3,75;").replace(";4.0;",
";4,0;")
+ expected = expected.replace(";5.0;", ";5,0;")
+ assert result["data"] == expected.encode()
+ assert result["count"] == 5
+ assert formatter.call_count == 5
diff --git a/tests/unit_tests/commands/sql_lab/streaming_export_command_test.py
b/tests/unit_tests/commands/sql_lab/streaming_export_command_test.py
index 83c5b2e9dce..c6aeb4ab0b0 100644
--- a/tests/unit_tests/commands/sql_lab/streaming_export_command_test.py
+++ b/tests/unit_tests/commands/sql_lab/streaming_export_command_test.py
@@ -16,7 +16,10 @@
# under the License.
"""Unit tests for SQL Lab Streaming CSV Export Command."""
+import csv
+import io
from decimal import Decimal
+from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
@@ -910,3 +913,74 @@ def
test_streaming_export_preserves_impersonated_user_context(
list(csv_generator_callable())
assert seen_usernames == ["alice"]
+
+
[email protected]("decimal_separator", [None, ".", ","])
+def test_streaming_csv_preserves_fixed_point_decimals(
+ decimal_separator: str | None,
+) -> None:
+ """Streaming CSV retains decimal digits and honors the decimal
separator."""
+ command: StreamingSqlResultExportCommand = StreamingSqlResultExportCommand(
+ "client_id"
+ )
+ result: MagicMock = MagicMock()
+ result.fetchmany.side_effect = [
+ [(Decimal("0E-18"), Decimal("-1E-7"), Decimal("1E+30"),
Decimal("12.3400"))],
+ [],
+ ]
+ buffer: io.StringIO = io.StringIO()
+ writer: Any = csv.writer(buffer, delimiter=";")
+
+ chunks: list[tuple[str, int, int]] = list(
+ command._process_rows(result, writer, buffer, None, decimal_separator)
+ )
+
+ expected: str = (
+
"0.000000000000000000;-0.0000001;1000000000000000000000000000000;12.3400"
+ )
+ if decimal_separator == ",":
+ expected = expected.replace(".", ",")
+ assert "".join(chunk[0] for chunk in chunks) == expected + "\r\n"
+
+
+def test_streaming_decimal_formatting_leaves_other_values_unchanged() -> None:
+ """Non-finite decimals and non-decimal values retain their
representations."""
+ command: StreamingSqlResultExportCommand = StreamingSqlResultExportCommand(
+ "client_id"
+ )
+ values: tuple[Any, ...] = (
+ Decimal("NaN"),
+ Decimal("Infinity"),
+ Decimal("-Infinity"),
+ 1,
+ 1e-7,
+ None,
+ True,
+ )
+
+ formatted: list[Any] = command._format_row_values(values, None)
+
+ assert all(
+ actual is original for actual, original in zip(formatted, values,
strict=True)
+ )
+
+
[email protected]("decimal_separator", [None, ".", ",", "::"])
+def test_streaming_non_finite_decimal_csv(decimal_separator: str | None) ->
None:
+ """Non-finite decimal CSV stays unchanged with custom decimal
separators."""
+ command: StreamingSqlResultExportCommand = StreamingSqlResultExportCommand(
+ "client_id"
+ )
+ values: tuple[Decimal, ...] = (
+ Decimal("NaN"),
+ Decimal("Infinity"),
+ Decimal("-Infinity"),
+ Decimal("12.3400"),
+ )
+ buffer: io.StringIO = io.StringIO()
+ writer: Any = csv.writer(buffer, delimiter=";")
+ writer.writerow(command._format_row_values(values, decimal_separator))
+ expected: str = "NaN;Infinity;-Infinity;12.3400\r\n"
+ if decimal_separator:
+ expected = expected.replace(".", decimal_separator)
+ assert buffer.getvalue() == expected
diff --git a/tests/unit_tests/common/test_query_context_processor.py
b/tests/unit_tests/common/test_query_context_processor.py
index d6fea922b55..27c273155db 100644
--- a/tests/unit_tests/common/test_query_context_processor.py
+++ b/tests/unit_tests/common/test_query_context_processor.py
@@ -16,6 +16,7 @@
# under the License.
from datetime import datetime, timedelta
+from decimal import Decimal
from typing import Any
from unittest.mock import MagicMock, patch
@@ -232,6 +233,17 @@ def test_get_data_json_preserves_browser_numeric_contract(
}
+def test_get_data_json_keeps_decimals_numeric(processor, mock_query_context)
-> None:
+ """Chart JSON keeps decimals as numbers; only SQL Lab quotes them."""
+ frame = pd.DataFrame({"amount": [Decimal("10.50")]})
+ mock_query_context.result_format = ChartDataResultFormat.JSON
+
+ result = processor.get_data(frame, [GenericDataType.NUMERIC])
+
+ assert type(result[0]["amount"]) is Decimal
+ assert superset_json.loads(superset_json.dumps(result)) == [{"amount":
10.5}]
+
+
def test_get_data_invalid_dataframe(processor, mock_query_context):
df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]})
coltypes = [GenericDataType.NUMERIC, GenericDataType.STRING]
diff --git a/tests/unit_tests/dataframe_test.py
b/tests/unit_tests/dataframe_test.py
index 5aa16c483be..02d974a061d 100644
--- a/tests/unit_tests/dataframe_test.py
+++ b/tests/unit_tests/dataframe_test.py
@@ -188,10 +188,11 @@ def
test_df_to_records_normalizes_only_exact_nonfinite_decimals() -> None:
]
frame = pd.DataFrame({"value": pd.Series(values, dtype=object)})
- records = df_to_records(frame)
+ records = df_to_records(frame, convert_decimals=True)
assert [records[index]["value"] for index in range(4)] == [None] * 4
- assert records[4]["value"] is finite
+ # SQL Lab quotes finite decimals so the browser keeps every digit.
+ assert records[4]["value"] == str(finite)
assert records[5]["value"] is hostile
strict_json = superset_json.dumps(records[:5], ignore_nan=False)
assert superset_json.loads(strict_json) == [
@@ -199,7 +200,7 @@ def
test_df_to_records_normalizes_only_exact_nonfinite_decimals() -> None:
{"value": None},
{"value": None},
{"value": None},
- {"value": 0.10000000000000000001},
+ {"value": "0.10000000000000000001"},
]
@@ -538,3 +539,79 @@ def
test_df_to_records_with_json_serialization_like_sql_lab() -> None:
)
parsed_no_flag = superset_json.loads(json_str_no_flag)
assert parsed_no_flag == parsed # Same result
+
+
[email protected](
+ "value",
+ [
+ "12345678901234567890.123456789012345678",
+ "-12345678901234567890.123456789012345678",
+ "0.000000000000000001",
+ "10.50",
+ "0.00",
+ "1E+30",
+ "0.000000000000000000",
+ "-0.000000100000000000",
+ ],
+)
+def test_decimal_records_keep_all_digits(value: str) -> None:
+ """Decimals become exact strings that survive a JSON round trip
unchanged."""
+ decimal_value: Decimal = Decimal(value)
+ frame = pd.DataFrame({"value": [decimal_value, None]})
+ records = df_to_records(frame, convert_decimals=True)
+ assert records == [{"value": format(decimal_value, "f")}, {"value": None}]
+ # Both the HTTP JSON and JSON cache must quote decimals for JavaScript.
+ assert superset_json.loads(superset_json.dumps(records)) == records
+ # SQL Lab conversion must not modify DataFrames used for chart arithmetic.
+ assert frame.iloc[0, 0] == decimal_value
+ assert isinstance(frame.iloc[0, 0], Decimal)
+
+
+def test_nested_decimal_records() -> None:
+ """Decimals nested inside dicts, lists and tuples also become exact
strings."""
+ value = Decimal("12345678901234567890.123456789012345678")
+ frame = pd.DataFrame({"value": [{"a": [value, None]}, (value,)]})
+ assert df_to_records(frame, convert_decimals=True) == [
+ {"value": {"a": [str(value), None]}},
+ {"value": (str(value),)},
+ ]
+
+
+def test_decimal_conversion_does_not_change_other_numbers() -> None:
+ """Ints, floats and bools pass through, and chart JSON still emits
numbers."""
+ frame = pd.DataFrame({"i": [2], "f": [0.5], "b": [True]})
+ assert df_to_records(frame, convert_decimals=True) == [
+ {"i": 2, "f": 0.5, "b": True}
+ ]
+ # Do not change the shared chart JSON serializer to emit decimal strings.
+ assert superset_json.loads(superset_json.dumps({"x": Decimal("10.50")}))
== {
+ "x": 10.5
+ }
+
+
+def test_decimal_conversion_is_disabled_by_default_for_chart_records() -> None:
+ """Chart records keep Decimal objects so the chart encoder emits
numbers."""
+ value = Decimal("10.50")
+ frame = pd.DataFrame({"value": [value, {"a": [value]}]})
+ records = df_to_records(frame)
+ assert records == [{"value": value}, {"value": {"a": [value]}}]
+ assert type(records[0]["value"]) is Decimal
+ assert superset_json.loads(superset_json.dumps(records)) == [
+ {"value": 10.5},
+ {"value": {"a": [10.5]}},
+ ]
+
+
+def test_high_scale_negative_decimal_csv() -> None:
+ """Small negative decimals export in fixed notation without formula
escaping."""
+ from superset.utils.csv import df_to_escaped_csv
+
+ records = df_to_records(
+ pd.DataFrame({"value": [Decimal("-0.000000100000000000")]}),
+ convert_decimals=True,
+ )
+ assert records == [{"value": "-0.000000100000000000"}]
+ assert (
+ df_to_escaped_csv(pd.DataFrame(records), index=False,
lineterminator="\n")
+ == "value\n-0.000000100000000000\n"
+ )
diff --git a/tests/unit_tests/result_set_test.py
b/tests/unit_tests/result_set_test.py
index 236fb376fa9..c178d84577a 100644
--- a/tests/unit_tests/result_set_test.py
+++ b/tests/unit_tests/result_set_test.py
@@ -18,9 +18,11 @@
# pylint: disable=import-outside-toplevel, unused-argument
from datetime import datetime, timezone
+from decimal import Decimal
import numpy as np
import pandas as pd
+import pytest
from numpy.core.multiarray import array
from pytest_mock import MockerFixture
@@ -643,3 +645,72 @@ def test_integers_outside_int64_are_stringified() -> None:
assert df["id"].tolist() == [1, 2, 3]
assert df["ubig"].iloc[:2].tolist() == ["18446744073709551615", "0"]
assert pd.isna(df["ubig"].iloc[2])
+
+
[email protected](
+ "values, expected",
+ [
+ (
+ [Decimal("Infinity"), Decimal("-0.00000010")],
+ [None, "-0.00000010"],
+ ),
+ (
+ [Decimal("-Infinity"), Decimal("NaN"), Decimal("1E+3")],
+ [None, None, "1000"],
+ ),
+ (
+ [Decimal("-0.00000010"), 1.5, Decimal("0E-18")],
+ ["-0.00000010", "1.5", "0.000000000000000000"],
+ ),
+ ],
+ ids=["infinity", "nan", "decimal-and-float"],
+)
+def test_stringified_decimals_use_fixed_point(
+ values: list[Decimal | float], expected: list[str | None]
+) -> None:
+ """
+ Decimal columns PyArrow cannot type (non-finite values, or Decimals mixed
+ with floats) are stringified without scientific notation, so CSV export
+ does not formula-escape small negatives. NaN and Infinity become null.
+ """
+ from superset.dataframe import df_to_records
+ from superset.utils.csv import df_to_escaped_csv
+
+ description = [("value", "numeric", None, None, None, None, True)]
+ data = [(value,) for value in values]
+ result_set = SupersetResultSet(data, description, BaseEngineSpec) # type:
ignore
+
+ records = df_to_records(result_set.to_pandas_df())
+ assert [record["value"] for record in records] == expected
+ csv = df_to_escaped_csv(pd.DataFrame(records), index=False,
lineterminator="\n")
+ assert "'" not in csv
+ assert "E-" not in csv
+ assert "E+" not in csv
+
+
[email protected](
+ "values, expected",
+ [
+ (
+ [Decimal("Infinity"), Decimal("-Infinity"), Decimal("NaN")],
+ [None, None, None],
+ ),
+ (
+ [Decimal("-0.00000010"), 1.5, Decimal("0E-18")],
+ ["-0.00000010", "1.5", "0.000000000000000000"],
+ ),
+ ],
+)
+def test_chart_dataframe_decimal_fallback(
+ mocker: MockerFixture,
+ values: list[Decimal | float],
+ expected: list[str | None],
+) -> None:
+ """Chart/dataset loading uses fixed-point strings and null non-finite
Decimals."""
+ from superset.models.core import Database
+
+ database = mocker.Mock(db_engine_spec=BaseEngineSpec)
+ description = [("value", "numeric", None, None, None, None, True)]
+ df = Database.load_into_dataframe(database, description, [(v,) for v in
values])
+
+ assert df["value"].tolist() == expected
diff --git a/tests/unit_tests/sql/execution/test_celery_task.py
b/tests/unit_tests/sql/execution/test_celery_task.py
index 1df4cf7755d..91a220f632d 100644
--- a/tests/unit_tests/sql/execution/test_celery_task.py
+++ b/tests/unit_tests/sql/execution/test_celery_task.py
@@ -896,7 +896,7 @@ def
test_serialize_result_set_json_normalizes_decimal_nonfinite(
data, _ = _serialize_result_set(mock_result_set)
assert isinstance(data, list)
- assert [row["value"] for row in data] == [None, None, None, None, finite]
+ assert [row["value"] for row in data] == [None, None, None, None,
str(finite)]
payload = _serialize_payload({"data": data})
payload_text = payload.decode() if isinstance(payload, bytes) else payload
diff --git a/tests/unit_tests/sql_lab_test.py b/tests/unit_tests/sql_lab_test.py
index f76ff4c1af8..77817152bf4 100644
--- a/tests/unit_tests/sql_lab_test.py
+++ b/tests/unit_tests/sql_lab_test.py
@@ -80,7 +80,7 @@ def test_sql_lab_and_view_json_normalize_decimal_nonfinite()
-> None:
result_set, BaseEngineSpec()
)
assert isinstance(data, list)
- assert [row["value"] for row in data] == [None, None, None, None, finite]
+ assert [row["value"] for row in data] == [None, None, None, None,
str(finite)]
payload = {
"data": data,
