fix: remove denormalization in eval scripts
NormalizeVelocity transform is disabled, so model outputs are already in original m/s space. Denormalizing inflates RMSE by ~3.5x. - evaluate.py: compute RMSE in model output space directly - benchmark/evaluate.py: same, plus fix misleading "normalized" comment Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -1,132 +0,0 @@
|
||||
"""
|
||||
Diagnostic: stateless (windowed, h=0) vs stateful (rolled GRU) evaluation.
|
||||
|
||||
Goal: determine whether the long-horizon tracking failure is caused by a
|
||||
train/inference mismatch in the GRU hidden state.
|
||||
|
||||
A. Stateless -- model.forward() over non-overlapping windows of seq_len,
|
||||
hidden state reset to zero at the start of each window.
|
||||
This is the SAME operating regime the model was trained in.
|
||||
|
||||
B. Stateful -- model.step() frame-by-frame, hidden state carried across
|
||||
the whole scene (the long-horizon regime that fails).
|
||||
|
||||
Both paths share identical inputs (same val transform, same event params,
|
||||
same frames). The only variable is whether the GRU state persists across
|
||||
windows / accumulates over the whole scene.
|
||||
|
||||
Run:
|
||||
uv run python -m src.velocity_prediction.diag_state_mismatch \
|
||||
--checkpoint checkpoints/<run>/best.pt --device cuda:7
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from src.velocity_prediction.model import VelocityPredictionModel
|
||||
from src.velocity_prediction.dataset import create_val_loader
|
||||
from src.velocity_prediction.config import train_cfg, TRAIN_SCENES
|
||||
|
||||
|
||||
def _rmse(pred: np.ndarray, target: np.ndarray) -> tuple[float, float, float]:
|
||||
"""Per-axis and combined RMSE."""
|
||||
rx = float(np.sqrt(np.mean((pred[:, 0] - target[:, 0]) ** 2)))
|
||||
ry = float(np.sqrt(np.mean((pred[:, 1] - target[:, 1]) ** 2)))
|
||||
rxy = float(np.sqrt(np.mean(np.sum((pred - target) ** 2, axis=1))))
|
||||
return rx, ry, rxy
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def eval_stateless(model, loader, device) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""A. Windowed forward, GRU hidden state reset to zero each window."""
|
||||
model.eval()
|
||||
preds, targets = [], []
|
||||
for batch in loader:
|
||||
events = batch["events"].to(device) # (B, S, 1, H, W)
|
||||
tilt = batch["tilt"].to(device) # (B, S, 3)
|
||||
target = batch["v_body_target"].to(device) # (B, S, 2)
|
||||
pred, _ = model(events, tilt) # (B, S, 2) h=0 internally
|
||||
preds.append(pred.reshape(-1, 2).cpu().numpy())
|
||||
targets.append(target.reshape(-1, 2).cpu().numpy())
|
||||
return np.concatenate(preds, 0), np.concatenate(targets, 0)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def eval_stateful(model, loader, device) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""B. Single-frame step, GRU hidden state carried across the whole scene."""
|
||||
model.eval()
|
||||
preds, targets = [], []
|
||||
h = None
|
||||
for batch in loader:
|
||||
events = batch["events"].to(device) # (1, 1, 1, H, W)
|
||||
tilt = batch["tilt"].to(device) # (1, 1, 3)
|
||||
target = batch["v_body_target"].to(device) # (1, 1, 2)
|
||||
pred, h = model.step(events, tilt, h) # (1, 2), (L, 1, H_gru)
|
||||
preds.append(pred.cpu().numpy())
|
||||
targets.append(target[:, -1, :].cpu().numpy())
|
||||
return np.concatenate(preds, 0), np.concatenate(targets, 0)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--checkpoint", type=str, required=True)
|
||||
ap.add_argument("--device", type=str, default="cuda")
|
||||
ap.add_argument("--seq-len", type=int, default=8,
|
||||
help="window length for stateless eval (default: train_cfg.seq_len)")
|
||||
args = ap.parse_args()
|
||||
|
||||
device = torch.device(args.device if torch.cuda.is_available() and "cuda" in args.device else "cpu")
|
||||
seq_len = args.seq_len or train_cfg.seq_len
|
||||
# print(f"Device: {device} seq_len(window)={seq_len} threshold={train_cfg.event_threshold} "
|
||||
# f"use_log={train_cfg.event_use_log}")
|
||||
print(f"Device: {device} seq_len(window)={seq_len} threshold=0 "
|
||||
f"use_log={train_cfg.event_use_log}")
|
||||
|
||||
|
||||
model = VelocityPredictionModel()
|
||||
ckpt = torch.load(args.checkpoint, map_location="cpu")
|
||||
model.load_state_dict(ckpt["model_state_dict"])
|
||||
model.to(device)
|
||||
print(f"Loaded {args.checkpoint} (epoch={ckpt.get('epoch', '?')}, "
|
||||
f"val_loss={ckpt.get('val_loss', '?')})\n")
|
||||
|
||||
th = 0
|
||||
log = train_cfg.event_use_log
|
||||
|
||||
rows = []
|
||||
for scene in TRAIN_SCENES:
|
||||
# A: non-overlapping windows, h reset each window (training regime)
|
||||
la = create_val_loader(scene_names=[scene], seq_len=seq_len, stride=seq_len,
|
||||
batch_size=8, num_workers=0,
|
||||
event_threshold=th, event_use_log=log)
|
||||
pa, ta = eval_stateless(model, la, device)
|
||||
|
||||
# B: single-frame, h carried (long-horizon regime)
|
||||
lb = create_val_loader(scene_names=[scene], seq_len=1, stride=1,
|
||||
batch_size=1, num_workers=0,
|
||||
event_threshold=th, event_use_log=log)
|
||||
pb, tb = eval_stateful(model, lb, device)
|
||||
|
||||
# Coverage note: stateless drops the trailing < seq_len frames.
|
||||
n_min = min(len(pa), len(pb))
|
||||
ax, ay, axy = _rmse(pa[:n_min], ta[:n_min])
|
||||
bx, by, bxy = _rmse(pb[:n_min], tb[:n_min])
|
||||
|
||||
rows.append((scene, len(pa), len(pb), ax, ay, axy, bx, by, bxy))
|
||||
print(f"[{scene}] A(stateless)={len(pa)} frames B(stateful)={len(pb)} frames")
|
||||
print(f" A vx={ax:.4f} vy={ay:.4f} xy={axy:.4f}")
|
||||
print(f" B vx={bx:.4f} vy={by:.4f} xy={bxy:.4f}")
|
||||
|
||||
print("\n================ SUMMARY (RMSE xy, m/s) ================")
|
||||
print(f"{'scene':<22}{'A stateless':>14}{'B stateful':>14}{'B/A':>8}")
|
||||
for scene, na, nb, ax, ay, axy, bx, by, bxy in rows:
|
||||
ratio = bxy / axy if axy > 0 else float("inf")
|
||||
print(f"{scene:<22}{axy:>14.4f}{bxy:>14.4f}{ratio:>8.2f}")
|
||||
print("=======================================================")
|
||||
print("If B >> A -> hidden-state distribution mismatch confirmed.")
|
||||
print("If A ~ B and both bad -> problem lies elsewhere (CNN/BN/...).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,7 +15,7 @@ import torch.nn as nn
|
||||
|
||||
from src.velocity_prediction.model import VelocityPredictionModel
|
||||
from src.velocity_prediction.dataset import create_val_loader
|
||||
from src.velocity_prediction.config import train_cfg, VELOCITY_MEAN, VELOCITY_STD
|
||||
from src.velocity_prediction.config import train_cfg
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -54,19 +54,13 @@ def evaluate_stateful(
|
||||
preds = np.concatenate(all_preds, axis=0)
|
||||
targets = np.concatenate(all_targets, axis=0)
|
||||
|
||||
# Denormalize predictions back to original velocity space
|
||||
mean = np.array(VELOCITY_MEAN, dtype=np.float32)
|
||||
std = np.array(VELOCITY_STD, dtype=np.float32)
|
||||
preds_denorm = preds * std + mean
|
||||
targets_denorm = targets * std + mean
|
||||
|
||||
# ── Diagnostics (in normalized space) ────────────────────────
|
||||
print("\n========== Evaluation Diagnostics (normalized space) ==========")
|
||||
# ── Diagnostics (in model output space — original m/s) ─────────
|
||||
print("\n========== Evaluation Diagnostics (model output space) ==========")
|
||||
print(f"Total samples: {len(preds)}")
|
||||
print(f"\n--- Targets (normalized) ---")
|
||||
print(f"\n--- Targets ---")
|
||||
print(f" vx: mean={targets[:, 0].mean():.6f}, std={targets[:, 0].std():.6f}")
|
||||
print(f" vy: mean={targets[:, 1].mean():.6f}, std={targets[:, 1].std():.6f}")
|
||||
print(f"\n--- Predictions (normalized) ---")
|
||||
print(f"\n--- Predictions ---")
|
||||
print(f" vx: mean={preds[:, 0].mean():.6f}, std={preds[:, 0].std():.6f}, "
|
||||
f"min={preds[:, 0].min():.6f}, max={preds[:, 0].max():.6f}")
|
||||
print(f" vy: mean={preds[:, 1].mean():.6f}, std={preds[:, 1].std():.6f}, "
|
||||
@@ -83,14 +77,14 @@ def evaluate_stateful(
|
||||
print(f" pred vy mean ≈ 0? {abs(preds[:, 1].mean()):.6f} diff from zero")
|
||||
print("=============================================\n")
|
||||
|
||||
# Per-axis and overall RMSE (in original velocity space)
|
||||
rmse_x = np.sqrt(np.mean((preds_denorm[:, 0] - targets_denorm[:, 0]) ** 2))
|
||||
rmse_y = np.sqrt(np.mean((preds_denorm[:, 1] - targets_denorm[:, 1]) ** 2))
|
||||
rmse_xy = np.sqrt(np.mean(np.sum((preds_denorm - targets_denorm) ** 2, axis=1)))
|
||||
# Per-axis and overall RMSE (in original m/s space)
|
||||
rmse_x = np.sqrt(np.mean((preds[:, 0] - targets[:, 0]) ** 2))
|
||||
rmse_y = np.sqrt(np.mean((preds[:, 1] - targets[:, 1]) ** 2))
|
||||
rmse_xy = np.sqrt(np.mean(np.sum((preds - targets) ** 2, axis=1)))
|
||||
|
||||
return {
|
||||
"preds": preds_denorm, # denormalized for plotting
|
||||
"targets": targets_denorm, # denormalized for plotting
|
||||
"preds": preds, # original m/s space
|
||||
"targets": targets, # original m/s space
|
||||
"rmse_x": rmse_x,
|
||||
"rmse_y": rmse_y,
|
||||
"rmse_xy": rmse_xy,
|
||||
|
||||
Reference in New Issue
Block a user