Commit ·
6981ed3
1
Parent(s): 1195870
[KM-624] Compute Functions: fix non-JSON-safe scalars in mode & top_value
Browse filesTwo latent bugs in the same class as the Decimal crash: numpy/pandas scalars
leaked into tool output and would break JSON serialization (SSE / persistence).
- analyze_descriptive: `mode` of an integer column returned numpy.int64 (not
JSON-serializable); a datetime column returned a pandas.Timestamp. Added a
_clean pass (int64 -> int, Timestamp -> ISO string).
- analyze_profile: `top_value` of a datetime column returned a pandas.Timestamp
that failed json.dumps. Hardened _clean to coerce Timestamp -> ISO string.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
src/tools/analytics/descriptive.py
CHANGED
|
@@ -37,6 +37,22 @@ class ColumnNotFoundError(ValueError):
|
|
| 37 |
"""A requested column is absent from the DataFrame (maps to error_code COLUMN_NOT_FOUND)."""
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
def _describe_one(series: pd.Series, metrics: tuple[str, ...]) -> dict[str, object]:
|
| 41 |
"""Compute descriptive metrics for a single column.
|
| 42 |
|
|
@@ -59,7 +75,7 @@ def _describe_one(series: pd.Series, metrics: tuple[str, ...]) -> dict[str, obje
|
|
| 59 |
out["null_rate"] = float(series.isna().mean()) if total else 0.0
|
| 60 |
elif m == "mode":
|
| 61 |
modes = non_null.mode()
|
| 62 |
-
out["mode"] = modes.iloc[0] if not modes.empty else None
|
| 63 |
elif not is_numeric:
|
| 64 |
out[m] = None
|
| 65 |
elif m == "mean":
|
|
|
|
| 37 |
"""A requested column is absent from the DataFrame (maps to error_code COLUMN_NOT_FOUND)."""
|
| 38 |
|
| 39 |
|
| 40 |
+
def _clean(value: object) -> object:
|
| 41 |
+
"""Coerce a scalar to a JSON-clean Python value.
|
| 42 |
+
|
| 43 |
+
`mode` can be any dtype: an integer column yields `numpy.int64` (NOT
|
| 44 |
+
JSON-serializable), a datetime column yields `pandas.Timestamp`. The other
|
| 45 |
+
metrics are already wrapped in `float(...)`; mode is the one that needs this.
|
| 46 |
+
"""
|
| 47 |
+
if value is None:
|
| 48 |
+
return None
|
| 49 |
+
if isinstance(value, pd.Timestamp):
|
| 50 |
+
return value.isoformat()
|
| 51 |
+
if hasattr(value, "item"):
|
| 52 |
+
return value.item()
|
| 53 |
+
return value
|
| 54 |
+
|
| 55 |
+
|
| 56 |
def _describe_one(series: pd.Series, metrics: tuple[str, ...]) -> dict[str, object]:
|
| 57 |
"""Compute descriptive metrics for a single column.
|
| 58 |
|
|
|
|
| 75 |
out["null_rate"] = float(series.isna().mean()) if total else 0.0
|
| 76 |
elif m == "mode":
|
| 77 |
modes = non_null.mode()
|
| 78 |
+
out["mode"] = _clean(modes.iloc[0]) if not modes.empty else None
|
| 79 |
elif not is_numeric:
|
| 80 |
out[m] = None
|
| 81 |
elif m == "mean":
|
src/tools/analytics/quality.py
CHANGED
|
@@ -22,7 +22,13 @@ from src.tools.analytics.descriptive import ColumnNotFoundError
|
|
| 22 |
|
| 23 |
|
| 24 |
def _clean(value: object) -> object:
|
| 25 |
-
"""Convert numpy scalars to plain Python so the output is JSON-clean.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
if hasattr(value, "item"):
|
| 27 |
return value.item()
|
| 28 |
return value
|
|
|
|
| 22 |
|
| 23 |
|
| 24 |
def _clean(value: object) -> object:
|
| 25 |
+
"""Convert numpy/pandas scalars to plain Python so the output is JSON-clean.
|
| 26 |
+
|
| 27 |
+
`top_value` (most frequent value) can be a `pandas.Timestamp` when profiling
|
| 28 |
+
a datetime column — neither `Timestamp` nor numpy scalars are JSON-safe.
|
| 29 |
+
"""
|
| 30 |
+
if isinstance(value, pd.Timestamp):
|
| 31 |
+
return value.isoformat()
|
| 32 |
if hasattr(value, "item"):
|
| 33 |
return value.item()
|
| 34 |
return value
|