chore(camera): 添加front相机

roboimi/assets/models/manipulators/DianaMed/table_square.xml

@@ -8,5 +8,6 @@
       </body>
       <camera name="top" pos="0.0 1.0 2.0" fovy="44" mode="targetbody" target="table"/>
       <camera name="angle" pos="0.0 0.0 2.0" fovy="37" mode="targetbody" target="table"/>
+      <camera name="front" pos="0 0 0.9" fovy="55" mode="fixed" quat="0.7071 0.7071 0 0"/>
   </worldbody>
 </mujoco>

roboimi/assets/robots/diana_med.py

@@ -58,8 +58,8 @@ class BiDianaMed(ArmBase):
     def __init__(self):
         super().__init__(
             name="Bidiana",
-            urdf_path="./assets/models/manipulators/DianaMed/DualDianaMed.urdf",
+            urdf_path="roboimi/assets/models/manipulators/DianaMed/DualDianaMed.urdf",
-            xml_path="./assets/models/manipulators/DianaMed/bi_diana_transfer_ee.xml",
+            xml_path="roboimi/assets/models/manipulators/DianaMed/bi_diana_transfer_ee.xml",
             gripper=None
         )
         self.left_arm = self.Arm(self, 'single', self.urdf_path)

roboimi/demos/diana_record_sim_episodes.py

@@ -21,7 +21,7 @@ def main():
     render_cam_name = 'angle'
 
     episode_len  = 700      #SIM_TASK_CONFIGS[task_name]['episode_len']
-    camera_names = ['angle','r_vis', 'top']   #SIM_TASK_CONFIGS[task_name]['camera_names']
+    camera_names = ['angle','r_vis', 'top', 'front']   #SIM_TASK_CONFIGS[task_name]['camera_names']
     if task_name == 'sim_transfer':
         policy = TestPickAndTransferPolicy(inject_noise)
         print(task_name)

roboimi/demos/eval_vla.py

@@ -29,7 +29,7 @@ class VLAEvaluator:
         self,
         agent: torch.nn.Module,
         device: str = 'cuda',
-        camera_names: List[str] = ['r_vis', 'top'],
+        camera_names: List[str] = ['r_vis', 'top', 'front'],
         num_queries: int = 1,
         obs_horizon: int = 2,
         pred_horizon: int = 16,
@@ -351,7 +351,7 @@ def evaluate_policy(
     max_timesteps: int = 700,
     task_name: str = 'sim_transfer',
     device: str = 'cuda',
-    camera_names: List[str] = ['r_vis', 'top'],
+    camera_names: List[str] = ['r_vis', 'top', 'front'],
     num_queries: int = 1,
     obs_horizon: int = 2,
     save_video: bool = True
@@ -500,7 +500,7 @@ def main():
                         help='Maximum timesteps per episode')
     parser.add_argument('--device', type=str, default='cuda',
                         help='Device for inference')
-    parser.add_argument('--camera_names', nargs='+', default=['r_vis', 'top'],
+    parser.add_argument('--camera_names', nargs='+', default=['r_vis', 'top', 'front'],
                         help='Camera names to use')
     parser.add_argument('--num_queries', type=int, default=16,
                         help='Policy query frequency (timesteps)')

roboimi/envs/double_base.py

@@ -53,6 +53,7 @@ class DualDianaMed(MujocoEnv):
         self.l_vis = None
         self.top = None
         self.angle = None
+        self.front = None
         self.obs = None
         
         self.rew = None
@@ -168,6 +169,7 @@ class DualDianaMed(MujocoEnv):
         obs['images']['angle'] = self.angle
         obs['images']['r_vis'] = self.r_vis
         obs['images']['l_vis'] = self.l_vis
+        obs['images']['front'] = self.front
         return obs
 
     def _get_image_obs(self):
@@ -177,6 +179,7 @@ class DualDianaMed(MujocoEnv):
         obs['images']['angle'] = self.angle
         obs['images']['r_vis'] = self.r_vis
         obs['images']['l_vis'] = self.l_vis
+        obs['images']['front'] = self.front
         return obs
     
     def _get_qpos_obs(self):
@@ -202,6 +205,8 @@ class DualDianaMed(MujocoEnv):
             return self.r_vis
         elif self.cam == 'l_vis':
             return self.l_vis
+        elif self.cam == 'front':
+            return self.front
         else:
             raise AttributeError("please input right name")
 
@@ -222,6 +227,9 @@ class DualDianaMed(MujocoEnv):
             img_renderer.update_scene(self.mj_data,camera="angle")
             self.angle = img_renderer.render()
             self.angle = self.angle[:, :, ::-1]
+            img_renderer.update_scene(self.mj_data,camera="front")
+            self.front = img_renderer.render()
+            self.front = self.front[:, :, ::-1]
             cv2.imshow('Cam view', self.cam_view)
             cv2.waitKey(1)
 

roboimi/envs/double_pos_ctrl_env.py

@@ -77,7 +77,8 @@ class DualDianaMed_Pos_Ctrl(DualDianaMed):
         while self.cam_flage:
             if(type(self.top)==type(None) 
                or type(self.angle)==type(None) 
-               or type(self.r_vis)==type(None)):
+               or type(self.r_vis)==type(None)
+               or type(self.front)==type(None)):
                 time.sleep(0.001)
                 t+=1
             else:

roboimi/utils/constants.py

@@ -20,7 +20,7 @@ SIM_TASK_CONFIGS = {
         'dataset_dir': DATASET_DIR + '/sim_transfer',
         'num_episodes': 20,
         'episode_len': 700,
-        'camera_names': ['top','r_vis'],
+        'camera_names': ['top','r_vis','front'],
         'xml_dir': HOME_PATH + '/assets'
     },
 

roboimi/vla/conf/agent/resnet_diffusion.yaml

@@ -19,4 +19,4 @@ obs_horizon: 2     # How many historical observations to use
 diffusion_steps: 100  # Number of diffusion timesteps for training
 
 # Camera Configuration
-num_cams: 2  # Number of cameras (e.g., r_vis, top)
+num_cams: 3  # Number of cameras (e.g., r_vis, top)

roboimi/vla/conf/config.yaml

@@ -4,7 +4,7 @@ defaults:
   - data: resnet_dataset
 
 train:
-  batch_size: 32       # Batch size for training
+  batch_size: 16       # Batch size for training
   lr: 1e-4            # Learning rate
   max_steps: 20000    # Maximum training steps
   log_freq: 100       # Log frequency (steps)

roboimi/vla/conf/data/resnet_dataset.yaml

@@ -13,6 +13,7 @@ action_horizon: 8 # Action execution horizon (used during evaluation)
 camera_names:
   - r_vis
   - top
+  - front
 
 # Normalization Type: 'gaussian' (mean/std) or 'min_max' ([-1, 1])
 normalization_type: gaussian

roboimi/vla/conf/data/siglip2.yaml

@@ -4,5 +4,5 @@ dataset_dir: "/home/d51/workspace/work/robo-imi-act/roboimi/demos/dataset/sim_tr
 pred_horizon: 16
 obs_horizon: 1
 action_horizon: 8
-camera_names: ['r_vis', 'top'] # ['angle', 'r_vis', 'top']
+camera_names: ['r_vis', 'top', 'front'] # ['angle', 'r_vis', 'top']
 normalization_type: 'gaussian' # 'min_max' or 'gaussian'

roboimi/vla/data/dataset.py

@@ -13,7 +13,7 @@ class RobotDiffusionDataset(Dataset):
                  pred_horizon=16, 
                  obs_horizon=2, 
                  action_horizon=8,
-                 camera_names=['r_vis', 'top'],
+                 camera_names=['r_vis', 'top', 'front'],
                  normalization_type='gaussian'): 
         """
         Args: