feat(vla): add SmolVLA conditioning and experiment artifacts
This commit is contained in:
Logic
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"agent": {
|
||||
"vision_backbone": {
|
||||
"_target_": "roboimi.vla.models.backbones.resnet_diffusion.ResNetDiffusionBackbone",
|
||||
"vision_backbone": "resnet18",
|
||||
"pretrained_backbone_weights": null,
|
||||
"vision_backbone_mode": "resnet",
|
||||
"freeze_backbone": false,
|
||||
"input_shape": [
|
||||
3,
|
||||
224,
|
||||
224
|
||||
],
|
||||
"crop_shape": null,
|
||||
"crop_is_random": true,
|
||||
"use_group_norm": true,
|
||||
"spatial_softmax_num_keypoints": 32,
|
||||
"use_separate_rgb_encoder_per_camera": true,
|
||||
"output_tokens_per_camera": false,
|
||||
"num_cameras": 3,
|
||||
"attnres_stem_dim": 64,
|
||||
"attnres_stage_dims": [
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
512
|
||||
],
|
||||
"attnres_stage_depths": [
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"attnres_stage_heads": [
|
||||
4,
|
||||
4,
|
||||
8,
|
||||
8
|
||||
],
|
||||
"attnres_stage_kv_heads": [
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"attnres_stage_window_sizes": [
|
||||
7,
|
||||
7,
|
||||
7,
|
||||
7
|
||||
],
|
||||
"attnres_dropout": 0.0,
|
||||
"attnres_ffn_mult": 2.667,
|
||||
"attnres_eps": 1e-06,
|
||||
"attnres_rope_theta": 10000.0,
|
||||
"camera_names": [
|
||||
"r_vis",
|
||||
"top",
|
||||
"front"
|
||||
]
|
||||
},
|
||||
"state_encoder": {
|
||||
"_target_": "roboimi.vla.modules.encoders.IdentityStateEncoder"
|
||||
},
|
||||
"action_encoder": {
|
||||
"_target_": "roboimi.vla.modules.encoders.IdentityActionEncoder"
|
||||
},
|
||||
"head": {
|
||||
"_target_": "roboimi.vla.models.heads.imf_transformer1d.IMFTransformer1D",
|
||||
"_partial_": true,
|
||||
"input_dim": 16,
|
||||
"output_dim": 16,
|
||||
"horizon": 32,
|
||||
"n_obs_steps": 2,
|
||||
"cond_dim": 208,
|
||||
"n_layer": 12,
|
||||
"n_head": 1,
|
||||
"n_emb": 384,
|
||||
"p_drop_emb": 0.1,
|
||||
"p_drop_attn": 0.1,
|
||||
"causal_attn": false,
|
||||
"time_as_cond": true,
|
||||
"obs_as_cond": true,
|
||||
"n_cond_layers": 0,
|
||||
"backbone_type": "attnres_full",
|
||||
"n_kv_head": 1,
|
||||
"attn_res_ffn_mult": 2.667,
|
||||
"attn_res_eps": 1e-06,
|
||||
"attn_res_rope_theta": 10000.0
|
||||
},
|
||||
"_target_": "roboimi.vla.agent_imf.IMFVLAAgent",
|
||||
"action_dim": 16,
|
||||
"obs_dim": 16,
|
||||
"normalization_type": "min_max",
|
||||
"pred_horizon": 32,
|
||||
"obs_horizon": 2,
|
||||
"num_action_steps": 16,
|
||||
"camera_names": [
|
||||
"r_vis",
|
||||
"top",
|
||||
"front"
|
||||
],
|
||||
"num_cams": 3,
|
||||
"diffusion_steps": 100,
|
||||
"inference_steps": 1,
|
||||
"head_type": "transformer",
|
||||
"loss_type": "pseudo_huber",
|
||||
"pseudo_huber_delta": 1.0
|
||||
},
|
||||
"data": {
|
||||
"_target_": "roboimi.vla.data.simpe_robot_dataset.SimpleRobotDataset",
|
||||
"dataset_dir": "/home/droid/project/diana_sim/sim_transfer",
|
||||
"pred_horizon": 32,
|
||||
"obs_horizon": 2,
|
||||
"camera_names": [
|
||||
"r_vis",
|
||||
"top",
|
||||
"front"
|
||||
],
|
||||
"image_resize_shape": [
|
||||
224,
|
||||
224
|
||||
]
|
||||
},
|
||||
"eval": {
|
||||
"ckpt_path": "/home/droid/project/roboimi/.worktrees/feat-imf-attnres-policy/runs/sim-transfer-imf-attnres-ph32-exec16-emb384-l12-infer1-unfreeze-step50k-roll5x5-20260403-141304/checkpoints/vla_model_best.pt",
|
||||
"num_episodes": 3,
|
||||
"num_workers": 1,
|
||||
"cuda_devices": null,
|
||||
"response_timeout_s": 300.0,
|
||||
"server_startup_timeout_s": 300.0,
|
||||
"max_timesteps": 700,
|
||||
"device": "cuda",
|
||||
"task_name": "sim_transfer",
|
||||
"task_description": null,
|
||||
"num_queries": 16,
|
||||
"obs_horizon": 2,
|
||||
"camera_names": [
|
||||
"r_vis",
|
||||
"top",
|
||||
"front"
|
||||
],
|
||||
"use_smoothing": false,
|
||||
"smooth_method": "ema",
|
||||
"smooth_alpha": 0.3,
|
||||
"headless": true,
|
||||
"verbose_action": false,
|
||||
"artifact_dir": null,
|
||||
"save_artifacts": false,
|
||||
"save_timing": false,
|
||||
"save_trajectory": false,
|
||||
"save_summary_json": false,
|
||||
"save_trajectory_npz": false,
|
||||
"save_trajectory_image": false,
|
||||
"trajectory_image_camera": null,
|
||||
"trajectory_image_camera_name": null,
|
||||
"record_video": false,
|
||||
"video_camera": null,
|
||||
"video_camera_name": null,
|
||||
"video_fps": 30
|
||||
},
|
||||
"train": {
|
||||
"batch_size": 16,
|
||||
"lr": 0.0001,
|
||||
"max_steps": 100000,
|
||||
"device": "cuda",
|
||||
"disable_cudnn": false,
|
||||
"num_workers": 12,
|
||||
"val_split": 0.0,
|
||||
"val_episode_indices": null,
|
||||
"action_mse_val_freq_epochs": 0,
|
||||
"seed": 42,
|
||||
"log_freq": 100,
|
||||
"save_freq": 2000,
|
||||
"use_swanlab": false,
|
||||
"swanlab_project": "roboimi-vla",
|
||||
"swanlab_run_name": null,
|
||||
"rollout_val_freq_epochs": 50,
|
||||
"rollout_validate_on_checkpoint": false,
|
||||
"rollout_num_episodes": 3,
|
||||
"rollout_device": "cuda",
|
||||
"rollout_num_workers": null,
|
||||
"rollout_cuda_devices": null,
|
||||
"rollout_response_timeout_s": 300.0,
|
||||
"rollout_server_startup_timeout_s": 300.0,
|
||||
"warmup_steps": 2000,
|
||||
"scheduler_type": "cosine",
|
||||
"min_lr": 1e-06,
|
||||
"weight_decay": 1e-05,
|
||||
"grad_clip": 1.0,
|
||||
"pretrained_ckpt": null
|
||||
},
|
||||
"experiment": {
|
||||
"name": "vla_diffusion",
|
||||
"notes": "",
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user