Uiwang-right DINOv3-ViTB16 FlowMatch diffusion policy (200k)

README.md

@@ -0,0 +1,21 @@
+---
+library_name: lerobot
+pipeline_tag: robotics
+tags: [lerobot, diffusion-policy, flow-matching, dinov3, robotics, manipulation]
+---
+
+# Hyundai Uiwang — Diffusion Policy (FlowMatch), RIGHT arm — **DINOv3 backbone**
+
+LeRobot Diffusion Policy trained on the Hyundai Uiwang **right**-arm dataset
+(118 episodes / 77,490 frames, 30 Hz), FlowMatch scheduler (`num_inference_steps=1`).
+Robot: HDRB + INSPIRE-RH56. Vision encoder: **facebook/dinov3-vitb16-pretrain-lvd1689m**
+(ViT-B/16), pretrained + frozen. Inputs: front_rgb + wrist_rgb + state(26) +
+gripper_sensor(30) + wrist_ft_sensor(6) -> action(26).
+
+## Load-time requirements (important)
+1. **DINOv3 gated access**: loading constructs the backbone via
+   `AutoModel.from_pretrained("facebook/dinov3-vitb16-pretrain-lvd1689m")`, which is gated —
+   accept the license on that model page with your HF account first.
+2. **lerobot with DINOv3 register-token support**: the `Dinov2Backbone` wrapper must strip
+   CLS + register tokens (DINOv3 has 4 registers). Use the snu-hyundai fork branch that
+   includes this patch, or the loaded patch features will be misaligned.

config.json

@@ -0,0 +1,125 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "input_features": {
+        "observation.images.front_rgb": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.wrist_rgb": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                26
+            ]
+        },
+        "observation.gripper_sensor": {
+            "type": "STATE",
+            "shape": [
+                30
+            ]
+        },
+        "observation.wrist_ft_sensor": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                26
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "drop_n_last_frames": 7,
+    "vision_backbone": "dinov2",
+    "theia_model_name": "theaiinstitute/theia-tiny-patch16-224-cdiv",
+    "dinov2_model_name": "facebook/dinov3-vitb16-pretrain-lvd1689m",
+    "freeze_vision_backbone": true,
+    "resize_shape": [
+        240,
+        320
+    ],
+    "crop_ratio": 0.9,
+    "crop_shape": [
+        216,
+        288
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 64,
+    "use_separate_rgb_encoder_per_camera": false,
+    "use_mask": false,
+    "mask_encoder_base_dim": 64,
+    "mask_feature_dim": 128,
+    "mask_height": 240,
+    "mask_width": 320,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "FlowMatch",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": 1,
+    "use_adaflow_inference": false,
+    "adaflow_min_steps": 1,
+    "adaflow_max_steps": 4,
+    "adaflow_convergence_threshold": 0.01,
+    "use_rtc": false,
+    "rtc_inference_delay": 0,
+    "rtc_execution_horizon": 0,
+    "compile_model": false,
+    "compile_mode": "reduce-overhead",
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

inference_example.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Portable inference example for the Hyundai Uiwang FlowMatch Diffusion Policy.
+
+Runs with ONLY the Hugging Face model id — no dataset download, no robot, no
+local checkpoint needed. The uploaded model bundles its normalization stats
+(policy_preprocessor / policy_postprocessor), so `make_pre_post_processors`
+loads everything straight from the Hub.
+
+What you need to provide at run time:
+  * front_rgb : np.uint8 (H, W, 3) RGB  — scene/zivid camera view
+  * wrist_rgb : np.uint8 (H, W, 3) RGB  — wrist camera view
+  * state     : np.float32 (26,)        — arm joints (6) + hand joints (20)
+
+The policy resizes images internally (to 240x320 then center-crops), so the
+input camera resolution does not need to match training exactly — just pass the
+raw RGB frames.
+
+Output: np.float32 (26,) action = target arm joints (6) + target hand joints (20), at 30 Hz.
+
+Usage:
+    # self-contained demo with synthetic frames (verifies the model loads + runs):
+    python examples/hyundai_uiwang/inference_example.py
+
+    # specify a different model / device:
+    python examples/hyundai_uiwang/inference_example.py --model-id Ngseo/hyundai-uiwang-right-flowmatch-dinov3 --device cuda
+
+Install (once):
+    pip install lerobot   # or use this repo with PYTHONPATH=src
+"""
+
+from __future__ import annotations
+
+import argparse
+
+import numpy as np
+import torch
+
+from lerobot.policies.diffusion.modeling_diffusion import DiffusionPolicy
+from lerobot.policies.factory import make_pre_post_processors
+
+DEFAULT_MODEL_ID = "Ngseo/hyundai-uiwang-right-flowmatch-dinov3"
+# Camera feature keys the model was trained with (see the model card / config.json).
+FRONT_KEY = "observation.images.front_rgb"
+WRIST_KEY = "observation.images.wrist_rgb"
+STATE_KEY = "observation.state"
+
+
+def load_policy(model_id: str = DEFAULT_MODEL_ID, device: str = "cuda"):
+    """Load the policy + pre/post processors from the Hugging Face Hub."""
+    device = device if (device != "cuda" or torch.cuda.is_available()) else "cpu"
+    policy = DiffusionPolicy.from_pretrained(model_id)
+    policy.config.device = device  # saved config pins device=cuda; align it to the runtime device
+    policy.to(device)
+    policy.eval()
+    policy.reset()  # clears the internal observation/action queues
+    # pretrained_path=model_id -> normalization stats are loaded from the Hub repo.
+    # Override the saved device_processor step so preprocessing targets `device` too.
+    preprocess, postprocess = make_pre_post_processors(
+        policy.config, model_id, preprocessor_overrides={"device_processor": {"device": device}}
+    )
+    return policy, preprocess, postprocess, device
+
+
+@torch.no_grad()
+def predict_action(
+    policy,
+    preprocess,
+    postprocess,
+    front_rgb: np.ndarray,
+    wrist_rgb: np.ndarray,
+    state: np.ndarray,
+    device: str = "cuda",
+) -> np.ndarray:
+    """Run one inference step and return a 26-d action as np.float32.
+
+    Note: the policy keeps an internal queue (n_obs_steps / n_action_steps), so
+    call this repeatedly at the control loop rate; `policy.reset()` starts a new
+    episode.
+    """
+    # Raw frame dict in the format expected by the preprocessor:
+    #   images: uint8 (H, W, C);  state: float32 (D,)  — batching/normalization
+    #   are handled by the processor pipeline.
+    obs = {
+        FRONT_KEY: torch.from_numpy(front_rgb).to(torch.float32).div(255).permute(2, 0, 1).unsqueeze(0).to(device),
+        WRIST_KEY: torch.from_numpy(wrist_rgb).to(torch.float32).div(255).permute(2, 0, 1).unsqueeze(0).to(device),
+        STATE_KEY: torch.from_numpy(state).to(torch.float32).unsqueeze(0).to(device),
+        "task": "",
+        "robot_type": "",
+    }
+    obs = preprocess(obs)
+    action = policy.select_action(obs)  # (1, 26), normalized
+    action = postprocess(action)  # unnormalized
+    return action.squeeze(0).float().cpu().numpy()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model-id", default=DEFAULT_MODEL_ID)
+    ap.add_argument("--device", default="cuda", choices=["cuda", "cpu", "mps"])
+    ap.add_argument("--steps", type=int, default=4, help="number of demo inference steps")
+    args = ap.parse_args()
+
+    print(f"Loading {args.model_id} ...")
+    policy, preprocess, postprocess, device = load_policy(args.model_id, args.device)
+
+    # Report what the model expects (handy when porting to a new robot).
+    img_keys = [k for k in policy.config.input_features if "image" in k]
+    state_dim = policy.config.input_features[STATE_KEY].shape[0]
+    action_dim = policy.config.output_features["action"].shape[0]
+    print(f"device={device} | cameras={img_keys} | state_dim={state_dim} | action_dim={action_dim}")
+
+    # --- demo with synthetic frames (replace these with real camera/robot data) ---
+    rng = np.random.default_rng(0)
+    for t in range(args.steps):
+        front_rgb = rng.integers(0, 256, size=(480, 640, 3), dtype=np.uint8)
+        wrist_rgb = rng.integers(0, 256, size=(480, 640, 3), dtype=np.uint8)
+        state = rng.standard_normal(state_dim).astype(np.float32)
+
+        action = predict_action(policy, preprocess, postprocess, front_rgb, wrist_rgb, state, device)
+        print(f"step {t}: action[26] = {np.array2string(action, precision=3, max_line_width=120)}")
+
+    print("\nOK — model runs from the HF id alone. Swap the synthetic frames for your robot's cameras/state.")
+
+
+if __name__ == "__main__":
+    main()

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa8794ffd5bd4a5aae4b9c5898b46087c92acccaa97b464994d8f67acd92eea
+size 1398891600

policy_postprocessor.json

@@ -0,0 +1,32 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              26
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_0_unnormalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58e7433b278337b293786ff7ab0591a02bb8e2d2b2584f22aa7f95c9957fe45d
+size 12320

policy_preprocessor.json

@@ -0,0 +1,76 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.images.front_rgb": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "observation.images.wrist_rgb": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              26
+            ]
+          },
+          "observation.gripper_sensor": {
+            "type": "STATE",
+            "shape": [
+              30
+            ]
+          },
+          "observation.wrist_ft_sensor": {
+            "type": "STATE",
+            "shape": [
+              6
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              26
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors"
+    }
+  ]
+}

policy_preprocessor_step_3_normalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58e7433b278337b293786ff7ab0591a02bb8e2d2b2584f22aa7f95c9957fe45d
+size 12320

train_config.json

@@ -0,0 +1,376 @@
+{
+    "dataset": {
+        "repo_id": "local/hyundai_uiwang_right",
+        "root": "data/lerobot/hyundai_uiwang_right",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "p_apply": 1.0,
+            "domain_randomization": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "lighting": {
+                    "weight": 0.0,
+                    "type": "RandomLighting",
+                    "kwargs": {
+                        "gain_range": [
+                            0.7,
+                            1.3
+                        ]
+                    }
+                },
+                "gamma": {
+                    "weight": 0.0,
+                    "type": "RandomGamma",
+                    "kwargs": {
+                        "gamma_range": [
+                            0.7,
+                            1.5
+                        ]
+                    }
+                },
+                "sensor_noise": {
+                    "weight": 0.0,
+                    "type": "RandomSensorNoise",
+                    "kwargs": {
+                        "shot_range": [
+                            0.0,
+                            0.04
+                        ],
+                        "read_range": [
+                            0.0,
+                            0.01
+                        ]
+                    }
+                },
+                "gaussian_blur": {
+                    "weight": 0.0,
+                    "type": "GaussianBlur",
+                    "kwargs": {
+                        "kernel_size": 5,
+                        "sigma": [
+                            0.1,
+                            2.0
+                        ]
+                    }
+                },
+                "intrinsics": {
+                    "weight": 0.0,
+                    "type": "RandomIntrinsics",
+                    "kwargs": {
+                        "focal_scale_range": [
+                            0.8,
+                            1.2
+                        ],
+                        "distortion_range": [
+                            -0.3,
+                            0.0
+                        ]
+                    }
+                },
+                "unprocessor": {
+                    "weight": 0.0,
+                    "type": "RandomUnprocessor",
+                    "kwargs": {
+                        "gamma_range": [
+                            1.8,
+                            2.6
+                        ],
+                        "ccm_noise": 0.05,
+                        "shot_range": [
+                            0.0,
+                            0.0005
+                        ],
+                        "read_range": [
+                            0.0,
+                            0.0001
+                        ]
+                    }
+                },
+                "corruption": {
+                    "weight": 0.0,
+                    "type": "RandomCorruption",
+                    "kwargs": {
+                        "severity_range": [
+                            1,
+                            3
+                        ],
+                        "corruption_types": [
+                            "noise",
+                            "blur",
+                            "digital"
+                        ]
+                    }
+                },
+                "corner_crop": {
+                    "weight": 0.0,
+                    "type": "RandomCornerCrop",
+                    "kwargs": {
+                        "crop_ratios": [
+                            0.05,
+                            0.1,
+                            0.2
+                        ]
+                    }
+                },
+                "warping": {
+                    "weight": 0.0,
+                    "type": "RandomWarp",
+                    "kwargs": {
+                        "distortion_scale_range": [
+                            0.1,
+                            0.4
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "streaming": false
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "input_features": {
+            "observation.images.front_rgb": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.wrist_rgb": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    26
+                ]
+            },
+            "observation.gripper_sensor": {
+                "type": "STATE",
+                "shape": [
+                    30
+                ]
+            },
+            "observation.wrist_ft_sensor": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    26
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "drop_n_last_frames": 7,
+        "vision_backbone": "dinov2",
+        "theia_model_name": "theaiinstitute/theia-tiny-patch16-224-cdiv",
+        "dinov2_model_name": "facebook/dinov3-vitb16-pretrain-lvd1689m",
+        "freeze_vision_backbone": true,
+        "resize_shape": [
+            240,
+            320
+        ],
+        "crop_ratio": 0.9,
+        "crop_shape": [
+            216,
+            288
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 64,
+        "use_separate_rgb_encoder_per_camera": false,
+        "use_mask": false,
+        "mask_encoder_base_dim": 64,
+        "mask_feature_dim": 128,
+        "mask_height": 240,
+        "mask_width": 320,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "FlowMatch",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": 1,
+        "use_adaflow_inference": false,
+        "adaflow_min_steps": 1,
+        "adaflow_max_steps": 4,
+        "adaflow_convergence_threshold": 0.01,
+        "use_rtc": false,
+        "rtc_inference_delay": 0,
+        "rtc_execution_horizon": 0,
+        "compile_model": false,
+        "compile_mode": "reduce-overhead",
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/uiwang_right_dinov3b_flowmatch",
+    "job_name": "uiwang_right_dinov3b_flowmatch",
+    "resume": false,
+    "seed": 1000,
+    "cudnn_deterministic": false,
+    "num_workers": 8,
+    "batch_size": 64,
+    "steps": 200000,
+    "eval_freq": 0,
+    "log_freq": 200,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": false,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": null,
+        "mode": null
+    },
+    "peft": null,
+    "use_rabc": false,
+    "rabc_progress_path": null,
+    "rabc_kappa": 0.01,
+    "rabc_epsilon": 1e-06,
+    "rabc_head_mode": "sparse",
+    "rename_map": {},
+    "checkpoint_path": null
+}