镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
feat(logging): add swanlab backend for training runs (#17)
* feat(logging): add swanlab backend + evo1 stage2 optimizations - Add SwanLab logging support (auto-selects when SWANLAB_API_KEY is set) - Add stage2 training optimizations: tensor fastpath, gradient checkpointing - Document stage2 config override behavior and runnable value-model alternative Cherry-picked from:bfdc28c,bcecabb* refactor(logging): separate SwanLabLogger from WandBLogger - Restore WandBLogger to its original clean form with all comments - Add SwanLabLogger as a separate class with the same interface - Add make_logger() factory function to select backend based on SWANLAB_API_KEY environment variable - Move evo1 stage2 runtime logging from lerobot_train.py into EVO1Policy.__init__ where it belongs - Add ImportError handling for swanlab dependency * fix(logging): gracefully fall back to WandB when swanlab is not installed If SWANLAB_API_KEY is set but the swanlab package is not installed, make_logger() now logs a warning and falls back to WandBLogger instead of crashing with an ImportError at training startup. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Elvinyk <223015152@link.cuhk.edu.cn> Co-authored-by: MrBlue0632 <2028207642@qq.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
这个提交包含在:
@ -33,6 +33,26 @@ Validated in this repo / recent experiments:
|
||||
- `training_stage=stage2` finetunes both the VLM and the action head.
|
||||
- In this repo, `resume_pretrain=true` is the Evo1-style stage handoff: load weights only, but start a fresh optimizer/scheduler state.
|
||||
- We currently recommend `dataset.image_transforms.enable=false` in the validated training commands because the image augmentation pipeline was not stable in the tested environment.
|
||||
- Stage2 now defaults to `embedder_tensor_fastpath=true`, `enable_vlm_gradient_checkpointing=true`, and
|
||||
`gradient_checkpointing_use_reentrant=false`.
|
||||
- When stage2 is launched from `--policy.path=<stage1_ckpt>`, the stage1 checkpoint config is loaded first.
|
||||
In practice, it is safer to explicitly pass the stage2 memory/speed flags on the CLI so they override any
|
||||
stage1-era values saved in `config.json`.
|
||||
|
||||
## Optimization Summary
|
||||
|
||||
The current Evo1 integration includes a small set of stage2-oriented training optimizations merged from the Evo_2 branch while preserving the LeRobot/Evo-RL structure:
|
||||
|
||||
- Runtime config switches were added for `embedder_tensor_fastpath`, `enable_vlm_gradient_checkpointing`, and `gradient_checkpointing_use_reentrant`.
|
||||
- Evo1 image collection during training no longer forces a `GPU -> CPU -> GPU` round trip before the VLM embedder.
|
||||
- InternVL3 stage2 now enables a more memory-friendly checkpointing path and disables `use_cache` when checkpointing is active.
|
||||
- The main `lerobot_train` entrypoint was kept intact; only lightweight Evo1 stage2 runtime logging was added.
|
||||
- These Evo1-specific optimizations apply to `stage2` training and to the final `RL policy train` stage when it uses
|
||||
`policy.type=evo1` with `training_stage=stage2`. They do not apply to `value-train` or `value-infer`, which use
|
||||
the separate value-model pipeline.
|
||||
- In the current local environment, the default `pistar06` language backbone `google/gemma-3-270m` may fail with
|
||||
a gated-repo `403`. For runnable value/RL scripts, prefer explicitly setting
|
||||
`--value.language_repo_id=Qwen/Qwen2.5-0.5B`.
|
||||
|
||||
## Stage 1 Command
|
||||
|
||||
@ -89,6 +109,10 @@ PYTHONPATH=src /opt/conda/bin/accelerate launch \
|
||||
--resume_pretrain=true \
|
||||
--policy.vlm_model_name=/root/private_data/models/InternVL3-1B \
|
||||
--policy.device=cuda \
|
||||
--policy.use_flash_attn=true \
|
||||
--policy.embedder_tensor_fastpath=true \
|
||||
--policy.enable_vlm_gradient_checkpointing=true \
|
||||
--policy.gradient_checkpointing_use_reentrant=false \
|
||||
--policy.push_to_hub=false \
|
||||
--policy.dropout=0.2 \
|
||||
--policy.optimizer_lr=1e-5 \
|
||||
|
||||
@ -56,6 +56,9 @@ class Evo1Config(PreTrainedConfig):
|
||||
vlm_num_layers: int | None = 14
|
||||
vlm_dtype: str = "bfloat16"
|
||||
use_flash_attn: bool = True
|
||||
embedder_tensor_fastpath: bool = True
|
||||
enable_vlm_gradient_checkpointing: bool | None = None
|
||||
gradient_checkpointing_use_reentrant: bool = False
|
||||
action_head: str = "flowmatching"
|
||||
embed_dim: int = 896
|
||||
hidden_dim: int = 1024
|
||||
@ -97,6 +100,9 @@ class Evo1Config(PreTrainedConfig):
|
||||
self.finetune_vlm = True
|
||||
self.finetune_action_head = True
|
||||
|
||||
if self.enable_vlm_gradient_checkpointing is None:
|
||||
self.enable_vlm_gradient_checkpointing = self.training_stage == "stage2"
|
||||
|
||||
if self.n_action_steps > self.chunk_size:
|
||||
raise ValueError(
|
||||
f"n_action_steps ({self.n_action_steps}) must be <= chunk_size ({self.chunk_size})"
|
||||
|
||||
@ -22,6 +22,9 @@ class EVO1(nn.Module):
|
||||
num_language_layers=config.get("vlm_num_layers", 14),
|
||||
model_dtype=config.get("vlm_dtype", "bfloat16"),
|
||||
use_flash_attn=config.get("use_flash_attn", True),
|
||||
enable_gradient_checkpointing=config.get("enable_vlm_gradient_checkpointing", False),
|
||||
enable_tensor_fastpath=config.get("embedder_tensor_fastpath", True),
|
||||
gradient_checkpointing_use_reentrant=config.get("gradient_checkpointing_use_reentrant", False),
|
||||
)
|
||||
|
||||
action_head_type = config.get("action_head", "flowmatching").lower()
|
||||
|
||||
@ -88,12 +88,18 @@ class InternVL3Embedder(nn.Module):
|
||||
num_language_layers: int | None = 14,
|
||||
model_dtype: str | torch.dtype = "bfloat16",
|
||||
use_flash_attn: bool = True,
|
||||
enable_gradient_checkpointing: bool = False,
|
||||
enable_tensor_fastpath: bool = True,
|
||||
gradient_checkpointing_use_reentrant: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self._requested_device = device
|
||||
self.image_size = image_size
|
||||
self.num_language_layers = num_language_layers
|
||||
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
|
||||
self.enable_gradient_checkpointing = bool(enable_gradient_checkpointing)
|
||||
self.enable_tensor_fastpath = bool(enable_tensor_fastpath)
|
||||
self.gradient_checkpointing_use_reentrant = bool(gradient_checkpointing_use_reentrant)
|
||||
self.transform = build_transform(image_size)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
|
||||
if isinstance(model_dtype, str):
|
||||
@ -123,12 +129,53 @@ class InternVL3Embedder(nn.Module):
|
||||
else:
|
||||
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
||||
self.model.language_model.lm_head = torch.nn.Identity()
|
||||
|
||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||
self._configure_memory_features()
|
||||
|
||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||
|
||||
def _configure_memory_features(self) -> None:
|
||||
checkpoint_kwargs = {"use_reentrant": self.gradient_checkpointing_use_reentrant}
|
||||
|
||||
def _enable_ckpt(module) -> bool:
|
||||
if module is None or not hasattr(module, "gradient_checkpointing_enable"):
|
||||
return False
|
||||
module.gradient_checkpointing_enable(gradient_checkpointing_kwargs=checkpoint_kwargs)
|
||||
return True
|
||||
|
||||
if not self.enable_gradient_checkpointing:
|
||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||
return
|
||||
|
||||
enabled_any = False
|
||||
enabled_any = _enable_ckpt(self.model) or enabled_any
|
||||
|
||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||
self.model.vision_model.encoder.gradient_checkpointing = True
|
||||
enabled_any = True
|
||||
|
||||
if hasattr(self.model, "language_model"):
|
||||
language_model = self.model.language_model
|
||||
enabled_any = _enable_ckpt(language_model) or enabled_any
|
||||
if hasattr(language_model, "model"):
|
||||
enabled_any = _enable_ckpt(language_model.model) or enabled_any
|
||||
if hasattr(language_model, "config"):
|
||||
language_model.config.use_cache = False
|
||||
|
||||
if hasattr(self.model, "config"):
|
||||
self.model.config.use_cache = False
|
||||
|
||||
if hasattr(self.model, "enable_input_require_grads"):
|
||||
self.model.enable_input_require_grads()
|
||||
|
||||
if enabled_any:
|
||||
logging.info(
|
||||
"Enabled InternVL3 gradient checkpointing (use_reentrant=%s).",
|
||||
self.gradient_checkpointing_use_reentrant,
|
||||
)
|
||||
else:
|
||||
logging.warning("Requested InternVL3 gradient checkpointing, but no checkpointable module was found.")
|
||||
|
||||
def _dynamic_preprocess_tensor(
|
||||
self, image_t, min_num=1, max_num=1, use_thumbnail=False
|
||||
):
|
||||
@ -191,7 +238,7 @@ class InternVL3Embedder(nn.Module):
|
||||
for image_tensors in image_tensors_batch:
|
||||
num_tiles_list = []
|
||||
for image in image_tensors:
|
||||
if isinstance(image, torch.Tensor):
|
||||
if isinstance(image, torch.Tensor) and self.enable_tensor_fastpath:
|
||||
if (
|
||||
image.ndim == 3
|
||||
and image.shape[0] == 3
|
||||
@ -216,6 +263,8 @@ class InternVL3Embedder(nn.Module):
|
||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||
tile_tensors = (tiles - mean) / std
|
||||
else:
|
||||
if isinstance(image, torch.Tensor):
|
||||
image = to_pil_image(image.detach().to(device="cpu"))
|
||||
image_t = TF.to_tensor(image).to(device=self.device, dtype=torch.float32)
|
||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||
tile_tensors = (tiles - mean) / std
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import builtins
|
||||
import logging
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
@ -29,6 +30,16 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
self.config = config
|
||||
self.model = EVO1(self._build_model_config(config))
|
||||
self.model.set_finetune_flags()
|
||||
|
||||
if config.training_stage == "stage2":
|
||||
logging.info(
|
||||
"Evo1 stage2 runtime settings: embedder_tensor_fastpath=%s "
|
||||
"enable_vlm_gradient_checkpointing=%s gradient_checkpointing_use_reentrant=%s",
|
||||
config.embedder_tensor_fastpath,
|
||||
config.enable_vlm_gradient_checkpointing,
|
||||
config.gradient_checkpointing_use_reentrant,
|
||||
)
|
||||
|
||||
self.reset()
|
||||
|
||||
@classmethod
|
||||
@ -70,6 +81,9 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
"vlm_num_layers": config.vlm_num_layers,
|
||||
"vlm_dtype": config.vlm_dtype,
|
||||
"use_flash_attn": config.use_flash_attn,
|
||||
"embedder_tensor_fastpath": config.embedder_tensor_fastpath,
|
||||
"enable_vlm_gradient_checkpointing": config.enable_vlm_gradient_checkpointing,
|
||||
"gradient_checkpointing_use_reentrant": config.gradient_checkpointing_use_reentrant,
|
||||
"action_head": config.action_head,
|
||||
"action_horizon": config.chunk_size,
|
||||
"per_action_dim": config.max_action_dim,
|
||||
@ -300,7 +314,7 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch across cameras: key={camera_key} batch={image.shape[0]} expected={batch_size}"
|
||||
)
|
||||
sample_images.append(image[batch_index].detach().cpu())
|
||||
sample_images.append(image[batch_index])
|
||||
if not sample_images:
|
||||
raise ValueError("EVO1 received a batch without any image tensor.")
|
||||
while len(sample_images) < self.config.max_views:
|
||||
|
||||
@ -150,13 +150,11 @@ def train(cfg: TrainRLServerPipelineConfig, job_name: str | None = None):
|
||||
logging.info(f"Learner logging initialized, writing to {log_file}")
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
# Setup WandB logging if enabled
|
||||
if cfg.wandb.enable and cfg.wandb.project:
|
||||
from lerobot.rl.wandb_utils import WandBLogger
|
||||
# Setup logging
|
||||
from lerobot.rl.wandb_utils import make_logger
|
||||
|
||||
wandb_logger = WandBLogger(cfg)
|
||||
else:
|
||||
wandb_logger = None
|
||||
wandb_logger = make_logger(cfg)
|
||||
if wandb_logger is None:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
|
||||
# Handle resume logic
|
||||
|
||||
@ -201,3 +201,106 @@ class WandBLogger:
|
||||
|
||||
wandb_video = self._wandb.Video(video_path, fps=self.env_fps, format="mp4")
|
||||
self._wandb.log({f"{mode}/video": wandb_video}, step=step)
|
||||
|
||||
|
||||
class SwanLabLogger:
|
||||
"""A helper class to log training metrics using SwanLab."""
|
||||
|
||||
def __init__(self, cfg: TrainPipelineConfig):
|
||||
try:
|
||||
import swanlab
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"SwanLab is required when SWANLAB_API_KEY is set. "
|
||||
"Install it with: pip install swanlab"
|
||||
)
|
||||
|
||||
self.cfg = cfg.wandb
|
||||
self.log_dir = cfg.output_dir
|
||||
self.job_name = cfg.job_name
|
||||
self.env_fps = cfg.env.fps if cfg.env else None
|
||||
|
||||
swanlab.login(api_key=os.environ["SWANLAB_API_KEY"])
|
||||
self._run = swanlab.init(
|
||||
project=self.cfg.project,
|
||||
workspace=self.cfg.entity,
|
||||
experiment_name=self.job_name,
|
||||
description=self.cfg.notes,
|
||||
tags=cfg_to_group(cfg, return_list=True, truncate_tags=True),
|
||||
logdir=str(self.log_dir),
|
||||
config=cfg.to_dict(),
|
||||
mode=self.cfg.mode if self.cfg.mode in ["offline", "disabled"] else "cloud",
|
||||
id=cfg.wandb.run_id,
|
||||
resume="must" if cfg.resume else None,
|
||||
)
|
||||
run_id = getattr(self._run, "id", None)
|
||||
cfg.wandb.run_id = run_id
|
||||
logging.info(colored("Logs will be synced with swanlab.", "blue", attrs=["bold"]))
|
||||
try:
|
||||
import swanlab.data as swanlab_data
|
||||
|
||||
run_url = swanlab_data.get_url()
|
||||
if run_url:
|
||||
logging.info(f"Track this run --> {colored(run_url, 'yellow', attrs=['bold'])}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def log_policy(self, checkpoint_dir: Path):
|
||||
return
|
||||
|
||||
def log_dict(
|
||||
self, d: dict, step: int | None = None, mode: str = "train", custom_step_key: str | None = None
|
||||
):
|
||||
if mode not in {"train", "eval"}:
|
||||
raise ValueError(mode)
|
||||
if step is None and custom_step_key is None:
|
||||
raise ValueError("Either step or custom_step_key must be provided.")
|
||||
|
||||
data = {}
|
||||
if custom_step_key is not None and custom_step_key in d:
|
||||
value_custom_step = d[custom_step_key]
|
||||
data[f"{mode}/{custom_step_key}"] = value_custom_step
|
||||
if step is None:
|
||||
step = int(value_custom_step)
|
||||
for k, v in d.items():
|
||||
if not isinstance(v, (int | float | str)):
|
||||
logging.warning(
|
||||
f'SwanLab logging of key "{k}" was ignored as its type "{type(v)}" is not handled by this wrapper.'
|
||||
)
|
||||
continue
|
||||
if custom_step_key is not None and k == custom_step_key:
|
||||
continue
|
||||
data[f"{mode}/{k}"] = v
|
||||
if data:
|
||||
self._run.log(data, step=step)
|
||||
|
||||
def log_video(self, video_path: str, step: int, mode: str = "train"):
|
||||
if mode not in {"train", "eval"}:
|
||||
raise ValueError(mode)
|
||||
|
||||
try:
|
||||
import swanlab
|
||||
|
||||
self._run.log({f"{mode}/video": swanlab.Video(video_path)}, step=step)
|
||||
except Exception as exc:
|
||||
logging.warning("Failed to log video to SwanLab: %s", exc)
|
||||
|
||||
|
||||
def make_logger(cfg: TrainPipelineConfig) -> WandBLogger | SwanLabLogger | None:
|
||||
"""Create a training logger based on config and environment.
|
||||
|
||||
Returns SwanLabLogger if SWANLAB_API_KEY is set, WandBLogger otherwise.
|
||||
Returns None if logging is disabled.
|
||||
"""
|
||||
if not (cfg.wandb.enable and cfg.wandb.project):
|
||||
return None
|
||||
|
||||
if os.getenv("SWANLAB_API_KEY"):
|
||||
try:
|
||||
return SwanLabLogger(cfg)
|
||||
except ImportError:
|
||||
logging.warning(
|
||||
"swanlab package is not installed but SWANLAB_API_KEY is set. "
|
||||
"Falling back to WandB. Install swanlab with: pip install swanlab"
|
||||
)
|
||||
return WandBLogger(cfg)
|
||||
|
||||
@ -37,7 +37,7 @@ from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.rl.acp_dataset_stats import compute_acp_indicator_stats
|
||||
from lerobot.rl.acp_hook import build_acp_raw_batch_hook
|
||||
from lerobot.rl.wandb_utils import WandBLogger
|
||||
from lerobot.rl.wandb_utils import make_logger
|
||||
from lerobot.scripts.lerobot_eval import eval_policy_all
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
|
||||
@ -200,13 +200,10 @@ def train(
|
||||
if is_main_process:
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
# Initialize wandb only on main process
|
||||
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
|
||||
wandb_logger = WandBLogger(cfg)
|
||||
else:
|
||||
wandb_logger = None
|
||||
if is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
# Initialize logger only on main process
|
||||
wandb_logger = make_logger(cfg) if is_main_process else None
|
||||
if wandb_logger is None and is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
|
||||
if cfg.seed is not None:
|
||||
set_seed(cfg.seed, accelerator=accelerator)
|
||||
|
||||
@ -18,7 +18,7 @@ from lerobot.datasets.utils import cycle
|
||||
from lerobot.optim.factory import make_optimizer_and_scheduler
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.rl.wandb_utils import WandBLogger
|
||||
from lerobot.rl.wandb_utils import make_logger
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
@ -98,12 +98,9 @@ def value_train(
|
||||
if is_main_process:
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
|
||||
wandb_logger = WandBLogger(cfg)
|
||||
else:
|
||||
wandb_logger = None
|
||||
if is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
wandb_logger = make_logger(cfg) if is_main_process else None
|
||||
if wandb_logger is None and is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
|
||||
if cfg.seed is not None:
|
||||
set_seed(cfg.seed, accelerator=accelerator)
|
||||
|
||||
在新工单中引用
屏蔽一个用户