Algorithm Insufficiency Identification

"""Analyse ML model insufficiencies for SOTIF."""
import numpy as np

def analyse_classifier_performance(predictions: list,
                                    ground_truth: list,
                                    conditions: list) -> dict:
    """
    Analyse classifier performance by condition.
    Identifies conditions where performance is insufficient.
    """
    results = {}
    unique_conditions = set(conditions)

    for cond in unique_conditions:
        mask = [i for i, c in enumerate(conditions) if c == cond]
        preds = [predictions[i] for i in mask]
        truth = [ground_truth[i] for i in mask]

        tp = sum(1 for p, t in zip(preds, truth) if p == 1 and t == 1)
        fn = sum(1 for p, t in zip(preds, truth) if p == 0 and t == 1)
        fp = sum(1 for p, t in zip(preds, truth) if p == 1 and t == 0)

        recall    = tp / (tp + fn) if (tp + fn) > 0 else 0.0
        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0

        results[cond] = {
            "recall":    round(recall, 3),
            "precision": round(precision, 3),
            "n_samples": len(mask),
            "sufficient": recall >= 0.95  # SOTIF target
        }
    return results

# Example: analyse AEB detector across weather conditions
conditions = (["daylight"]*500 + ["light_rain"]*200 +
              ["heavy_rain"]*50  + ["fog"]*30)
# Simulated: heavy rain recall = 0.71 (insufficient)
gt    = [1]*780
preds = ([1]*495 + [0]*5 +     # daylight: recall 0.99
         [1]*186 + [0]*14 +    # light rain: recall 0.93
         [1]*35  + [0]*15 +    # heavy rain: recall 0.70 FAIL
         [1]*22  + [0]*8)      # fog: recall 0.73 FAIL

results = analyse_classifier_performance(preds, gt, conditions)
for cond, r in results.items():
    status = "OK" if r["sufficient"] else "INSUFFICIENT"
    print(f"{cond:15s}: recall={r['recall']:.2f}  [{status}]")