feat: add TBPTT training with cross-batch hidden state carryover

- config: seq_len=128, batch_size=4 for long-sequence TBPTT
- dataset: create_tbptt_loader with non-overlapping windows, strict temporal order
- model: forward() accepts/exposes hidden state h; add step() for single-frame stateful inference
- train: carry detached hidden state across batches, reset at epoch boundary
- benchmark: fix model call for new (v_body, h) return signature

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>

src/velocity_prediction/config.py

@@ -39,7 +39,8 @@ VAL_SCENES = [
     # "indoor_forward_3", "indoor_forward_9", "indoor_forward_10",  # Easy
 ]
 TEST_SCENES = [
-    "indoor_forward_9","indoor_forward_3",
+    "indoor_forward_9",
+    # "indoor_forward_9","indoor_forward_3",
     # "indoor_forward_7",      # Hard 室内
     # "outdoor_forward_1",     # Easy 室外
     # "outdoor_forward_5"      # Hard 室外
@@ -92,8 +93,8 @@ class ModelConfig:
 
 @dataclass
 class TrainConfig:
-    seq_len: int = 8                    # frames per training sequence
-    batch_size: int = 32
+    seq_len: int = 128                    # frames per training sequence
+    batch_size: int = 4
     epochs: int = 300
     lr: float = 1e-3
     weight_decay: float = 1e-5

src/velocity_prediction/dataset.py

@@ -145,3 +145,48 @@ def create_val_loader(
         shuffle=False,
     )
     return loader
+
+
+def create_tbptt_loader(
+    scene_names: Optional[List[str]] = None,
+    seq_len: int = 8,
+    batch_size: int = 32,
+    event_threshold: float = 0.1,
+    event_use_log: bool = True,
+):
+    """Create a DataLoader for TBPTT training.
+
+    Non-overlapping windows (stride=seq_len), no shuffle, single worker
+    for strict temporal order.  Scene order is shuffled per epoch but
+    frames within each scene are always in temporal sequence, so GRU
+    hidden state can be carried across consecutive batches.
+
+    Frames per scene = floor(scene_frames / seq_len) * seq_len;
+    trailing frames that don't fill a full window are dropped.
+    """
+    import random
+    if scene_names is None:
+        from src.velocity_prediction.config import TRAIN_SCENES
+        scene_names = TRAIN_SCENES
+
+    urls = _scene_urls(scene_names)
+    # Scene-level shuffle: randomize scene order, but each scene's frames
+    # are strictly in temporal order (essential for cross-batch TBPTT).
+    random.shuffle(urls)
+
+    transform = build_train_transform(
+        event_threshold=event_threshold,
+        event_use_log=event_use_log,
+    )
+    pipeline = _build_pipeline(
+        urls, transform, seq_len=seq_len, stride=seq_len,
+        shuffle=0, deterministic=True,
+    )
+
+    loader = wds.WebLoader(
+        pipeline,
+        batch_size=batch_size,
+        num_workers=0,          # strict temporal ordering
+        shuffle=False,
+    )
+    return loader

src/velocity_prediction/diag_state_mismatch.py

@@ -0,0 +1,132 @@
+"""
+Diagnostic: stateless (windowed, h=0) vs stateful (rolled GRU) evaluation.
+
+Goal: determine whether the long-horizon tracking failure is caused by a
+train/inference mismatch in the GRU hidden state.
+
+  A. Stateless  -- model.forward() over non-overlapping windows of seq_len,
+                   hidden state reset to zero at the start of each window.
+                   This is the SAME operating regime the model was trained in.
+
+  B. Stateful   -- model.step() frame-by-frame, hidden state carried across
+                   the whole scene (the long-horizon regime that fails).
+
+Both paths share identical inputs (same val transform, same event params,
+same frames). The only variable is whether the GRU state persists across
+windows / accumulates over the whole scene.
+
+Run:
+    uv run python -m src.velocity_prediction.diag_state_mismatch \
+        --checkpoint checkpoints/<run>/best.pt --device cuda:7
+"""
+
+import argparse
+import numpy as np
+import torch
+
+from src.velocity_prediction.model import VelocityPredictionModel
+from src.velocity_prediction.dataset import create_val_loader
+from src.velocity_prediction.config import train_cfg, TRAIN_SCENES
+
+
+def _rmse(pred: np.ndarray, target: np.ndarray) -> tuple[float, float, float]:
+    """Per-axis and combined RMSE."""
+    rx = float(np.sqrt(np.mean((pred[:, 0] - target[:, 0]) ** 2)))
+    ry = float(np.sqrt(np.mean((pred[:, 1] - target[:, 1]) ** 2)))
+    rxy = float(np.sqrt(np.mean(np.sum((pred - target) ** 2, axis=1))))
+    return rx, ry, rxy
+
+
+@torch.no_grad()
+def eval_stateless(model, loader, device) -> tuple[np.ndarray, np.ndarray]:
+    """A. Windowed forward, GRU hidden state reset to zero each window."""
+    model.eval()
+    preds, targets = [], []
+    for batch in loader:
+        events = batch["events"].to(device)          # (B, S, 1, H, W)
+        tilt = batch["tilt"].to(device)              # (B, S, 3)
+        target = batch["v_body_target"].to(device)   # (B, S, 2)
+        pred, _ = model(events, tilt)                   # (B, S, 2)  h=0 internally
+        preds.append(pred.reshape(-1, 2).cpu().numpy())
+        targets.append(target.reshape(-1, 2).cpu().numpy())
+    return np.concatenate(preds, 0), np.concatenate(targets, 0)
+
+
+@torch.no_grad()
+def eval_stateful(model, loader, device) -> tuple[np.ndarray, np.ndarray]:
+    """B. Single-frame step, GRU hidden state carried across the whole scene."""
+    model.eval()
+    preds, targets = [], []
+    h = None
+    for batch in loader:
+        events = batch["events"].to(device)          # (1, 1, 1, H, W)
+        tilt = batch["tilt"].to(device)              # (1, 1, 3)
+        target = batch["v_body_target"].to(device)   # (1, 1, 2)
+        pred, h = model.step(events, tilt, h)        # (1, 2), (L, 1, H_gru)
+        preds.append(pred.cpu().numpy())
+        targets.append(target[:, -1, :].cpu().numpy())
+    return np.concatenate(preds, 0), np.concatenate(targets, 0)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--checkpoint", type=str, required=True)
+    ap.add_argument("--device", type=str, default="cuda")
+    ap.add_argument("--seq-len", type=int, default=8,
+                    help="window length for stateless eval (default: train_cfg.seq_len)")
+    args = ap.parse_args()
+
+    device = torch.device(args.device if torch.cuda.is_available() and "cuda" in args.device else "cpu")
+    seq_len = args.seq_len or train_cfg.seq_len
+    # print(f"Device: {device}  seq_len(window)={seq_len}  threshold={train_cfg.event_threshold} "
+    #       f"use_log={train_cfg.event_use_log}")
+    print(f"Device: {device}  seq_len(window)={seq_len}  threshold=0 "
+          f"use_log={train_cfg.event_use_log}")
+
+
+    model = VelocityPredictionModel()
+    ckpt = torch.load(args.checkpoint, map_location="cpu")
+    model.load_state_dict(ckpt["model_state_dict"])
+    model.to(device)
+    print(f"Loaded {args.checkpoint} (epoch={ckpt.get('epoch', '?')}, "
+          f"val_loss={ckpt.get('val_loss', '?')})\n")
+
+    th = 0
+    log = train_cfg.event_use_log
+
+    rows = []
+    for scene in TRAIN_SCENES:
+        # A: non-overlapping windows, h reset each window (training regime)
+        la = create_val_loader(scene_names=[scene], seq_len=seq_len, stride=seq_len,
+                               batch_size=8, num_workers=0,
+                               event_threshold=th, event_use_log=log)
+        pa, ta = eval_stateless(model, la, device)
+
+        # B: single-frame, h carried (long-horizon regime)
+        lb = create_val_loader(scene_names=[scene], seq_len=1, stride=1,
+                               batch_size=1, num_workers=0,
+                               event_threshold=th, event_use_log=log)
+        pb, tb = eval_stateful(model, lb, device)
+
+        # Coverage note: stateless drops the trailing < seq_len frames.
+        n_min = min(len(pa), len(pb))
+        ax, ay, axy = _rmse(pa[:n_min], ta[:n_min])
+        bx, by, bxy = _rmse(pb[:n_min], tb[:n_min])
+
+        rows.append((scene, len(pa), len(pb), ax, ay, axy, bx, by, bxy))
+        print(f"[{scene}]  A(stateless)={len(pa)} frames  B(stateful)={len(pb)} frames")
+        print(f"    A  vx={ax:.4f}  vy={ay:.4f}  xy={axy:.4f}")
+        print(f"    B  vx={bx:.4f}  vy={by:.4f}  xy={bxy:.4f}")
+
+    print("\n================ SUMMARY (RMSE xy, m/s) ================")
+    print(f"{'scene':<22}{'A stateless':>14}{'B stateful':>14}{'B/A':>8}")
+    for scene, na, nb, ax, ay, axy, bx, by, bxy in rows:
+        ratio = bxy / axy if axy > 0 else float("inf")
+        print(f"{scene:<22}{axy:>14.4f}{bxy:>14.4f}{ratio:>8.2f}")
+    print("=======================================================")
+    print("If B >> A  -> hidden-state distribution mismatch confirmed.")
+    print("If A ~ B and both bad  -> problem lies elsewhere (CNN/BN/...).")
+
+
+if __name__ == "__main__":
+    main()

src/velocity_prediction/model.py

@@ -124,14 +124,17 @@ class VelocityPredictionModel(nn.Module):
         # nn.init.uniform_(self.head[-1].weight, -0.001, 0.001)
         # nn.init.zeros_(self.head[-1].bias)
 
-    def forward(self, events: torch.Tensor, tilt: torch.Tensor) -> torch.Tensor:
+    def forward(self, events: torch.Tensor, tilt: torch.Tensor,
+                h: torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Args:
             events: (B, S, 1, H, W)
             tilt:   (B, S, 3)
+            h:      (num_layers, B, hidden_size) or None → zeros (TBPTT state)
 
         Returns:
             v_body: (B, S, 2)  predicted body-frame [v_right, v_forward] at every timestep
+            h_new:  (num_layers, B, hidden_size)  final GRU hidden state (detached by caller)
         """
         B, S = events.shape[:2]
 
@@ -145,12 +148,12 @@ class VelocityPredictionModel(nn.Module):
         # Fuse per frame
         fused = torch.cat([cnn_feat, pose_feat], dim=-1)  # (B, S, 320)
 
-        # GRU temporal modelling
-        gru_out, _ = self.gru(fused)           # (B, S, 128)
+        # GRU temporal modelling — accepts external hidden state for TBPTT
+        gru_out, h_new = self.gru(fused, h)    # (B, S, 128), (num_layers, B, 128)
 
         # Head regression over every timestep (single-layer GRU → gru_out == h_n[-1] at t=-1)
         v_body = self.head(gru_out)           # (B, S, 128) → (B, S, 2)
-        return v_body
+        return v_body, h_new
 
     @torch.no_grad()
     def step(self, events: torch.Tensor, tilt: torch.Tensor,
@@ -195,7 +198,7 @@ if __name__ == "__main__":
     B, S, H, W = 4, 8, 240, 320
     events = torch.randn(B, S, 1, H, W)
     tilt = torch.randn(B, S, 3)
-    out = model(events, tilt)
+    out, h = model(events, tilt)
     print(f"Input events: {events.shape}")
     print(f"Input tilt:   {tilt.shape}")
     print(f"Output:       {out.shape}  (should be [4, 8, 2])")

src/velocity_prediction/train.py

@@ -18,7 +18,7 @@ from torch.utils.tensorboard import SummaryWriter
 
 from src.velocity_prediction.config import train_cfg, model_cfg
 from src.velocity_prediction.model import VelocityPredictionModel, count_parameters
-from src.velocity_prediction.dataset import create_train_loader, create_val_loader
+from src.velocity_prediction.dataset import create_val_loader, create_tbptt_loader
 
 
 def set_seed(seed: int):
@@ -39,8 +39,9 @@ def train_one_epoch(
     log_interval: int = 50,
     global_step: int = 0,
     use_amp: bool = True,
-) -> tuple[float, int]:
-    """Train for one epoch. Returns (avg_loss, updated_global_step)."""
+    h_state: torch.Tensor = None,
+) -> tuple[float, int, torch.Tensor]:
+    """Train for one epoch. Returns (avg_loss, updated_global_step, h_state_final)."""
     model.train()
     total_loss = 0.0
     num_batches = 0
@@ -51,11 +52,15 @@ def train_one_epoch(
         tilt = batch["tilt"].to(device)              # (B, S, 3)
         target = batch["v_body_target"].to(device)   # (B, S, 2)
 
-        # Per-step supervision over the whole sequence
+        # Drop carried state if batch size changed (partial last batch / new scene)
+        if h_state is not None and h_state.shape[1] != events.shape[0]:
+            h_state = None
+
+        # Per-step supervision over the whole sequence — TBPTT hidden state
         with torch.amp.autocast(device.type, enabled=use_amp):
-            pred_seq = model(events, tilt)               # (B, S, 2)
-            loss_per_step = criterion(pred_seq, target)  # (B, S, 2)
-            loss_per_step = loss_per_step.mean(-1)       # (B, S)
+            pred_seq, h_new = model(events, tilt, h_state)   # (B, S, 2), (L, B, H)
+            loss_per_step = criterion(pred_seq, target)      # (B, S, 2)
+            loss_per_step = loss_per_step.mean(-1)           # (B, S)
             loss = loss_per_step.mean()
 
         optimizer.zero_grad()
@@ -63,6 +68,9 @@ def train_one_epoch(
         scaler.step(optimizer)
         scaler.update()
 
+        # Detach hidden state for next batch — gradient only back to current seq_len
+        h_state = h_new.detach()
+
         total_loss += loss.item()
         num_batches += 1
         global_step += 1
@@ -79,7 +87,7 @@ def train_one_epoch(
 
     avg_loss = total_loss / max(num_batches, 1)
     print(f"  Epoch {epoch} | Avg Loss: {avg_loss:.6f}")
-    return avg_loss, global_step
+    return avg_loss, global_step, h_state
 
 
 @torch.no_grad()
@@ -101,7 +109,7 @@ def validate(
         target = batch["v_body_target"].to(device)
 
         with torch.amp.autocast(device.type, enabled=use_amp):
-            pred_seq = model(events, tilt)               # (B, S, 2)
+            pred_seq, _ = model(events, tilt)          # (B, S, 2), h=None stateless
             loss_per_step = criterion(pred_seq, target)  # (B, S, 2)
             loss = loss_per_step.mean(-1).mean()
 
@@ -142,12 +150,10 @@ def main():
     print(f"Model parameters: {total_params:,} ({total_params/1e6:.3f} M)")
     print(f"AMP: {'enabled' if use_amp else 'disabled'}")
 
-    # Data loaders
-    train_loader = create_train_loader(
+    # Data loaders — TBPTT training requires strict temporal order
+    train_loader = create_tbptt_loader(
         seq_len=train_cfg.seq_len,
-        stride=train_cfg.sliding_window_stride,
         batch_size=train_cfg.batch_size,
-        num_workers=train_cfg.num_workers,
         event_threshold=event_threshold,
         event_use_log=train_cfg.event_use_log,
     )
@@ -217,11 +223,13 @@ def main():
     for epoch in range(start_epoch, train_cfg.epochs + 1):
         epoch_start = time.time()
 
-        train_loss, global_step = train_one_epoch(
+        # Reset GRU hidden state at epoch start — each epoch begins with h=0
+        train_loss, global_step, _ = train_one_epoch(
             model, train_loader, optimizer, criterion, scaler, device, epoch, writer,
             log_interval=train_cfg.log_interval,
             global_step=global_step,
             use_amp=use_amp,
+            h_state=None,
         )
         val_loss = validate(model, val_loader, criterion, device, use_amp=use_amp)
         scheduler.step()