JiajunLI/roboimi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
5ae9f5fa482fc7c5b1c659d7defe5eb7ed190c03
roboimi/workspace/metrics.json
T

398 lines
9.5 KiB
JSON
Raw Blame History

{
"tables": [
{
"label": "Socket peg insertion, 100-rollout performance and speed",
"headers": [
"run_name",
"method_family",
"horizon_or_steps",
"avg_reward",
"median_reward",
"avg_max_reward",
"max_reward",
"nonzero_reward",
"success_like",
"avg_inference_fps",
"avg_control_fps",
"avg_inference_time_ms"
],
"rows": [
[
"socket-insert-imf-attnres-ph32-exec16-emb384-l12-infer1-unfreeze-step50k-roll1x10-5880g01-20260506-195806",
"iMF-AttnRes",
"infer1, ph32, exec16, 50k",
"1275.22",
"1490.5",
"3.12",
"2250.0",
"84/100",
"0/100",
"64.226",
"5.863",
"16.186"
],
[
"socket-insert-no-pretrain-ph16-emb384-l18-infer100-unfreeze-step150k-roll1x10-5090-20260506-193753",
"Diffusion-style VLA",
"infer100, ph16, 150k",
"861.53",
"668.0",
"2.58",
"2241.0",
"97/100",
"2/100",
"2.956",
"2.591",
"338.291"
],
[
"socket-insert-no-pretrain-ph32-exec16-emb384-l18-infer100-unfreeze-step150k-roll1x10-5880g1-20260507-092227",
"Diffusion-style VLA",
"infer100, ph32, exec16, 150k",
"1019.39",
"804.0",
"2.47",
"2385.0",
"92/100",
"4/100",
"2.516",
"1.928",
"397.912"
],
[
"socket-insert-imf-attnres-ph32-exec16-emb384-l12-infer2-drop005-unfreeze-step150k-roll1x10-5880g01-20260508-165022",
"iMF-AttnRes",
"infer2, ph32, exec16, 150k",
"1472.62",
"1825.0",
"3.29",
"2365.0",
"83/100",
"5/100",
"71.579",
"7.181",
"14.409"
],
[
"socket-insert-imf-attnres-ph32-exec16-emb384-l12-infer3-drop005-unfreeze-step150k-roll1x10-5880g01-20260509-172423",
"iMF-AttnRes",
"infer3, ph32, exec16, 150k",
"1513.56",
"1901.5",
"3.28",
"2285.0",
"83/100",
"3/100",
"67.891",
"7.247",
"15.122"
],
[
"act-socket-peg-224-20260508-170237",
"ACT",
"action chunking",
"289.6",
"13.0",
"1.29",
"2091.0",
"55/100",
"1/100",
"10.5171",
"2.2782",
"100.2116"
],
[
"smolvla_socket_peg_bs80_100k_20260511_145727",
"SmolVLA",
"100k",
"466.16",
"106.0",
"1.72",
"4.0",
"89/100",
"0/100",
"382.53",
"16.84",
"2.614"
]
]
},
{
"label": "Object transfer / sim_transfer, 100-rollout performance and speed",
"headers": [
"run_name",
"method_family",
"horizon_or_steps",
"avg_reward",
"median_reward",
"avg_max_reward",
"max_reward",
"nonzero_reward",
"success_like",
"avg_inference_fps",
"avg_control_fps",
"avg_inference_time_ms"
],
"rows": [
[
"diffusion_policy_native_dit_ddpm_resnet_best",
"Diffusion Policy native DiT + DDPM + ResNet",
"best checkpoint",
"319.2",
"6.0",
"1.68",
"1416.0",
"55/100",
"29/100",
"32.09",
"16.70",
"n/a"
],
[
"embed384_layer18_best_checkpoint",
"Diffusion-style VLA",
"emb384, layer18",
"233.52",
"0.0",
"1.12",
"1436.0",
"39/100",
"17/100",
"1.859",
"1.649",
"n/a"
],
[
"resnet18-multitoken-imf-emb256-l16-ph16-ex08-roll10-l20g2-20260406-112815",
"iMF multi-token ResNet18",
"step34999, ph16, exec08",
"260.66",
"0.0",
"1.12",
"1422.0",
"33/100",
"23/100",
"441.80",
"13.37",
"n/a"
],
[
"imf-p2-full-attnres-vision-ph16-ex08-emb384-l12-b40-lr1p25e4-ms50k-l20g3-20260405-002424",
"iMF full AttnRes vision",
"ph16, exec08, 50k",
"228.42",
"0.0",
"0.94",
"1482.0",
"31/100",
"16/100",
"55.997",
"4.301",
"n/a"
],
[
"imf-p1-ph16-ex16-emb384-l12-ms50k-l20g0-20260404-131223",
"iMF-AttnRes DiT only",
"ph16, exec16, 50k",
"240.64",
"0.0",
"1.02",
"1428.0",
"32/100",
"19/100",
"137.996",
"4.523",
"n/a"
],
[
"imf-p1-ph32-ex08-emb384-l12-ms50k-l20g1-20260404-131223",
"iMF-AttnRes DiT only",
"ph32, exec08, 50k",
"163.28",
"0.0",
"0.76",
"1566.0",
"26/100",
"12/100",
"85.638",
"4.317",
"n/a"
],
[
"imf-p1-ph32-ex32-emb384-l12-ms50k-5090-20260404-13122",
"iMF-AttnRes DiT only",
"ph32, exec32, 50k",
"260.72",
"0.0",
"1.18",
"1342.0",
"38/100",
"21/100",
"9.557",
"3.299",
"n/a"
],
[
"imf-p1-ph16-ex08-emb384-l12-ms50k-5880g1-20260404-131223",
"iMF-AttnRes DiT only",
"ph16, exec08, 50k",
"229.56",
"0.0",
"1.18",
"1564.0",
"41/100",
"18/100",
"69.984",
"8.053",
"n/a"
],
[
"imf-p1-ph08-ex08-emb384-l12-ms50k-5880g0-20260404-131223",
"iMF-AttnRes DiT only",
"ph08, exec08, 50k",
"237.02",
"0.0",
"1.32",
"1400.0",
"48/100",
"18/100",
"69.192",
"8.173",
"n/a"
],
[
"imf-p1-ph32-ex16-emb384-l12-ms50k-l20g2-20260404-131223",
"iMF-AttnRes DiT only",
"ph32, exec16, 50k",
"49.88",
"0.0",
"0.32",
"1284.0",
"13/100",
"4/100",
"138.903",
"4.482",
"n/a"
],
[
"sim-transfer-imf-attnres-ph32-exec16-emb384-l12-infer1-unfreeze-step50k-roll5x5-20260403-14130",
"iMF-AttnRes",
"infer1, ph32, exec16, 50k",
"526.22",
"86.0",
"2.14",
"1460.0",
"63/100",
"44/100",
"8.911",
"3.193",
"n/a"
]
]
},
{
"label": "Derived comparisons used in the paper draft",
"headers": [
"comparison",
"metric",
"numerator_run",
"denominator_run",
"numerator_value",
"denominator_value",
"derived_ratio_or_delta"
],
"rows": [
[
"socket best iMF vs ph16 diffusion baseline",
"avg_reward ratio",
"iMF infer3 150k",
"diffusion ph16 infer100 150k",
"1513.56",
"861.53",
"1.76x"
],
[
"socket best iMF vs ph32 diffusion baseline",
"avg_reward ratio",
"iMF infer3 150k",
"diffusion ph32 infer100 150k",
"1513.56",
"1019.39",
"1.48x"
],
[
"socket best iMF vs ACT",
"avg_reward ratio",
"iMF infer3 150k",
"ACT",
"1513.56",
"289.6",
"5.23x"
],
[
"socket best iMF vs SmolVLA",
"avg_reward ratio",
"iMF infer3 150k",
"SmolVLA",
"1513.56",
"466.16",
"3.25x"
],
[
"socket iMF infer2 vs ph16 diffusion baseline",
"inference time reduction",
"iMF infer2 150k",
"diffusion ph16 infer100 150k",
"14.409",
"338.291",
"23.48x lower latency"
],
[
"socket iMF infer3 vs ph16 diffusion baseline",
"inference fps ratio",
"iMF infer3 150k",
"diffusion ph16 infer100 150k",
"67.891",
"2.956",
"22.97x"
],
[
"socket iMF infer2 vs ph32 diffusion baseline",
"inference fps ratio",
"iMF infer2 150k",
"diffusion ph32 infer100 150k",
"71.579",
"2.516",
"28.45x"
],
[
"sim_transfer best iMF vs native diffusion policy",
"avg_reward ratio",
"sim-transfer iMF infer1",
"native diffusion policy",
"526.22",
"319.2",
"1.65x"
],
[
"sim_transfer best iMF vs native diffusion policy",
"success_like delta",
"sim-transfer iMF infer1",
"native diffusion policy",
"44",
"29",
"+15 episodes"
],
[
"sim_transfer ResNet18 multitoken iMF vs embed384 layer18 baseline",
"inference fps ratio",
"ResNet18 multitoken iMF",
"embed384 layer18",
"441.80",
"1.859",
"237.65x"
]
]
}
]
}
Reference in New Issue View Git Blame Copy Permalink