Skip to content

Uncertainty Diagnostics

The default uncertainty diagnostics preserve numerical NaN values for legitimate undefined edge cases, such as uncertainty-error correlation when all predictive standard deviations are constant. Use the JSON-safe helpers for API payloads, dashboards, and report manifests that require strict JSON.

matgpr.uncertainty

prediction_interval_bounds(y_pred, y_std, *, confidence_level=0.95)

Return Gaussian predictive interval bounds.

Parameters:

Name Type Description Default
y_pred

Predictive mean values.

 required
y_std

Predictive standard deviations in the same units as y_pred.

 required
confidence_level float

Central interval probability, for example 0.95.

 0.95
Source code in matgpr/uncertainty.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def prediction_interval_bounds(
    y_pred,
    y_std,
    *,
    confidence_level: float = 0.95,
) -> tuple[np.ndarray, np.ndarray]:
    """Return Gaussian predictive interval bounds.

    Parameters
    ----------
    y_pred
        Predictive mean values.
    y_std
        Predictive standard deviations in the same units as ``y_pred``.
    confidence_level
        Central interval probability, for example ``0.95``.
    """
    y_pred = to_1d_finite(y_pred, "y_pred")
    y_std = _to_positive_std(y_std)
    validate_same_length(y_pred, y_std, "y_pred", "y_std")
    confidence_level = validate_confidence_level(confidence_level)

    z_value = NormalDist().inv_cdf(0.5 + confidence_level / 2.0)
    return y_pred - z_value * y_std, y_pred + z_value * y_std

interval_coverage(y_true, y_pred, y_std, *, confidence_level=0.95)

Calculate observed coverage for a Gaussian prediction interval.

Source code in matgpr/uncertainty.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def interval_coverage(
    y_true,
    y_pred,
    y_std,
    *,
    confidence_level: float = 0.95,
) -> dict[str, float]:
    """Calculate observed coverage for a Gaussian prediction interval."""
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    confidence_level = validate_confidence_level(confidence_level)
    lower, upper = prediction_interval_bounds(
        y_pred,
        y_std,
        confidence_level=confidence_level,
    )
    covered = (y_true >= lower) & (y_true <= upper)
    interval_width = upper - lower
    observed_coverage = float(np.mean(covered))
    return {
        "confidence_level": float(confidence_level),
        "expected_coverage": float(confidence_level),
        "observed_coverage": observed_coverage,
        "coverage_error": observed_coverage - float(confidence_level),
        "mean_interval_width": float(np.mean(interval_width)),
        "median_interval_width": float(np.median(interval_width)),
    }

calibration_curve(y_true, y_pred, y_std, *, confidence_levels=None)

Calculate observed coverage across confidence levels.

The returned dataframe is useful for calibration plots. A well-calibrated uncertainty model should have observed coverage close to expected coverage across the full range of confidence levels.

Source code in matgpr/uncertainty.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def calibration_curve(
    y_true,
    y_pred,
    y_std,
    *,
    confidence_levels=None,
) -> pd.DataFrame:
    """Calculate observed coverage across confidence levels.

    The returned dataframe is useful for calibration plots. A well-calibrated
    uncertainty model should have observed coverage close to expected coverage
    across the full range of confidence levels.
    """
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    if confidence_levels is None:
        confidence_levels = np.linspace(0.1, 0.95, 18)
    confidence_levels = _to_confidence_levels(confidence_levels)

    rows = [
        interval_coverage(
            y_true,
            y_pred,
            y_std,
            confidence_level=level,
        )
        for level in confidence_levels
    ]
    return pd.DataFrame(rows)

gaussian_nlpd(y_true, y_pred, y_std, *, min_std=1e-12, reduction='mean')

Calculate Gaussian negative log predictive density.

Lower values indicate sharper and better-calibrated predictive distributions. reduction="none" returns one NLPD value per sample.

Source code in matgpr/uncertainty.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
def gaussian_nlpd(
    y_true,
    y_pred,
    y_std,
    *,
    min_std: float = 1e-12,
    reduction: str = "mean",
):
    """Calculate Gaussian negative log predictive density.

    Lower values indicate sharper and better-calibrated predictive
    distributions. ``reduction="none"`` returns one NLPD value per sample.
    """
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    if min_std <= 0:
        raise ValueError("min_std must be positive")
    if reduction not in {"mean", "sum", "none"}:
        raise ValueError("reduction must be one of: mean, sum, none")

    y_std = np.maximum(y_std, min_std)
    variance = y_std**2
    values = 0.5 * np.log(2.0 * np.pi * variance) + 0.5 * ((y_true - y_pred) ** 2) / variance

    if reduction == "mean":
        return float(np.mean(values))
    if reduction == "sum":
        return float(np.sum(values))
    return values

standardized_residuals(y_true, y_pred, y_std)

Return residuals divided by predictive standard deviation.

Source code in matgpr/uncertainty.py 
143
144
145
146
def standardized_residuals(y_true, y_pred, y_std) -> np.ndarray:
    """Return residuals divided by predictive standard deviation."""
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    return (y_true - y_pred) / y_std

uncertainty_error_correlation(y_true, y_pred, y_std, *, method='spearman')

Correlate predictive standard deviation with absolute prediction error.

Source code in matgpr/uncertainty.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def uncertainty_error_correlation(
    y_true,
    y_pred,
    y_std,
    *,
    method: str = "spearman",
) -> float:
    """Correlate predictive standard deviation with absolute prediction error."""
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    absolute_error = np.abs(y_true - y_pred)
    normalized = str(method).lower()
    if normalized not in {"spearman", "pearson"}:
        raise ValueError("method must be either 'spearman' or 'pearson'")

    if (
        len(y_true) < 2
        or np.allclose(absolute_error, absolute_error[0])
        or np.allclose(y_std, y_std[0])
    ):
        return np.nan

    if normalized == "spearman":
        return float(spearmanr(y_std, absolute_error).correlation)
    return float(pearsonr(y_std, absolute_error)[0])

uncertainty_diagnostics(y_true, y_pred, y_std, *, confidence_level=0.95)

Return a compact dictionary of uncertainty diagnostic metrics.

Source code in matgpr/uncertainty.py 
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
def uncertainty_diagnostics(
    y_true,
    y_pred,
    y_std,
    *,
    confidence_level: float = 0.95,
) -> dict[str, float]:
    """Return a compact dictionary of uncertainty diagnostic metrics."""
    y_true, y_pred, y_std = _validate_prediction_arrays(y_true, y_pred, y_std)
    coverage = interval_coverage(
        y_true,
        y_pred,
        y_std,
        confidence_level=confidence_level,
    )
    z_residuals = standardized_residuals(y_true, y_pred, y_std)
    return {
        "NLPD": gaussian_nlpd(y_true, y_pred, y_std),
        "mean_std": float(np.mean(y_std)),
        "median_std": float(np.median(y_std)),
        "mean_absolute_error": float(np.mean(np.abs(y_true - y_pred))),
        "mean_standardized_residual": float(np.mean(z_residuals)),
        "std_standardized_residual": float(np.std(z_residuals, ddof=0)),
        "uncertainty_error_spearman": uncertainty_error_correlation(
            y_true,
            y_pred,
            y_std,
            method="spearman",
        ),
        **coverage,
    }

json_safe_diagnostics(diagnostics, *, include_warnings=True, warning_key='diagnostic_warnings')

Return strict-JSON-safe uncertainty diagnostics.

Undefined diagnostic values, such as uncertainty-error correlation for constant uncertainties, are converted from NaN to None.

Source code in matgpr/uncertainty.py 
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def json_safe_diagnostics(
    diagnostics: dict[str, float],
    *,
    include_warnings: bool = True,
    warning_key: str = "diagnostic_warnings",
) -> dict[str, object]:
    """Return strict-JSON-safe uncertainty diagnostics.

    Undefined diagnostic values, such as uncertainty-error correlation for
    constant uncertainties, are converted from ``NaN`` to ``None``.
    """
    return json_safe_metrics(
        diagnostics,
        include_warnings=include_warnings,
        warning_key=warning_key,
    )

json_safe_uncertainty_diagnostics(y_true, y_pred, y_std, *, confidence_level=0.95, include_warnings=True, warning_key='diagnostic_warnings')

Calculate uncertainty diagnostics and return a strict-JSON-safe dictionary.

Source code in matgpr/uncertainty.py 
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def json_safe_uncertainty_diagnostics(
    y_true,
    y_pred,
    y_std,
    *,
    confidence_level: float = 0.95,
    include_warnings: bool = True,
    warning_key: str = "diagnostic_warnings",
) -> dict[str, object]:
    """Calculate uncertainty diagnostics and return a strict-JSON-safe dictionary."""
    return json_safe_diagnostics(
        uncertainty_diagnostics(
            y_true,
            y_pred,
            y_std,
            confidence_level=confidence_level,
        ),
        include_warnings=include_warnings,
        warning_key=warning_key,
    )