feat: 更新框架，新增数据及定义和backbone

roboimi/demos/vla_scripts/train_vla.py

@@ -16,7 +16,7 @@ from hydra.utils import instantiate
 
 log = logging.getLogger(__name__)
 
-@hydra.main(version_base=None, config_path="../../../roboimi/vla/conf", config_name="config")
+@hydra.main(version_base=None, config_path="../../vla/conf", config_name="config")
 def main(cfg: DictConfig):
     print(OmegaConf.to_yaml(cfg))
     log.info(f"🚀 Starting VLA Training with Real Data (Device: {cfg.train.device})")
@@ -64,21 +64,30 @@ def main(cfg: DictConfig):
         # 我们在这里做一个映射，模拟多模态融合前的处理
         
         # 假设我们只用配置里的第一个 key 作为主视觉
-        primary_cam_key = cfg.data.obs_keys[0] 
+        # primary_cam_key = cfg.data.obs_keys[0] 
         
         # Dataset 返回 shape: (B, Obs_Horizon, C, H, W)
         # DebugBackbone 期望: (B, C, H, W) 或者 (B, Seq, Dim)
         # 这里我们取 Obs_Horizon 的最后一帧 (Current Frame)
-        input_img = batch['obs'][primary_cam_key][:, -1, :, :, :]
+        # input_img = batch['obs'][primary_cam_key][:, -1, :, :, :]
         
+        # agent_input = {
+        #     "obs": {
+        #         "image": input_img,
+        #         "text": batch["language"] # 传递语言指令
+        #     },
+        #     "actions": batch["actions"] # (B, Chunk, Dim)
+        # }
         agent_input = {
-            "obs": {
-                "image": input_img,
-                "text": batch["language"] # 传递语言指令
-            },
-            "actions": batch["actions"] # (B, Chunk, Dim)
+            "action": batch["action"],
+            "qpos": batch["qpos"],
+            "images": {}
         }
 
+        for cam_name in cfg.data.camera_names:
+            key = f"image_{cam_name}"
+            agent_input["images"][cam_name] = batch[key].squeeze(1)
+
         # --- 5. Forward & Backward ---
         outputs = agent(agent_input)