feat: add checkpoint resume and fix train_loss tracking

- Add --resume CLI arg to resume training from a checkpoint
- Restore model, optimizer, scheduler state; continue from saved epoch+1
- Preserve global_step and best_val_loss across resume
- Save run_id in checkpoints for TensorBoard log continuity
- Use logs/run_<timestamp>/ subdirectories to isolate experiment logs
- Fix: replace train_loss in checkpoint dict with global_step to avoid
  KeyError when loading; track global_step through train_one_epoch
- Fix: use global_step (not batch_idx) as TensorBoard x-axis for batch loss
- Fix: print average loss at end of each epoch

Generated by Mistral Vibe (ds-v4-flash).
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-06-04 22:55:31 +08:00
parent 0a504d648e
commit ec143868d0
2 changed files with 61 additions and 21 deletions
+9 -8
View File
@@ -10,9 +10,10 @@ from pathlib import Path
DATASET_ROOT = Path(__file__).resolve().parents[2] / "dataset"
# Velocity normalization stats (computed from training set)
VELOCITY_MEAN = [0.859184, -0.783945] # [vx, vy]
VELOCITY_STD = [2.244513, 1.088335] # [vx, vy]
# Velocity normalization stats (computed from forward scenes only, 28363 frames)
# Yaw-compensated horizontal velocity: [v_right, v_forward]
VELOCITY_MEAN = [-2.902497, 3.837231] # [v_right, v_forward]
VELOCITY_STD = [3.453774, 3.722085] # [v_right, v_forward]
# TRAIN_SCENES = [
# "indoor_forward_3", "indoor_forward_5", "indoor_forward_6",
@@ -38,10 +39,10 @@ VAL_SCENES = [
# "indoor_forward_3", "indoor_forward_9", "indoor_forward_10", # Easy
]
TEST_SCENES = [
"indoor_forward_7", # Hard 室内
"outdoor_forward_1", # Easy 室外
"outdoor_forward_5" # Hard 室外
# "indoor_forward_3", "indoor_forward_9", "indoor_forward_10", # Easy
# "indoor_forward_7", # Hard 室内
# "outdoor_forward_1", # Easy 室外
# "outdoor_forward_5" # Hard 室外
"indoor_forward_3", "indoor_forward_9", "indoor_forward_10", # Easy
]
@@ -75,7 +76,7 @@ class GRUConfig:
class HeadConfig:
input_dim: int = 128 # GRU hidden_size
hidden_dim: int = 64
output_dim: int = 2 # [vx_body, vy_body]
output_dim: int = 2 # [v_right, v_forward]
@dataclass