镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
feat(value-viz): add configurable multiview and smoothing controls
- add viz.video_keys for selecting and ordering camera views - add viz.smooth_window with default 1 (disabled) - wire new viz options through lerobot_value_infer export path - migrate validated overlay style and multiview export logic
这个提交包含在:
@ -93,9 +93,11 @@ class ValueInferenceVizConfig:
|
|||||||
enable: bool = False
|
enable: bool = False
|
||||||
episodes: str = "all"
|
episodes: str = "all"
|
||||||
video_key: str | None = None
|
video_key: str | None = None
|
||||||
|
video_keys: str | None = None
|
||||||
overwrite: bool = False
|
overwrite: bool = False
|
||||||
vcodec: str = "libsvtav1"
|
vcodec: str = "libsvtav1"
|
||||||
frame_storage_mode: str = "memory"
|
frame_storage_mode: str = "memory"
|
||||||
|
smooth_window: int = 1
|
||||||
|
|
||||||
def validate(self) -> None:
|
def validate(self) -> None:
|
||||||
if not self.episodes:
|
if not self.episodes:
|
||||||
@ -104,6 +106,8 @@ class ValueInferenceVizConfig:
|
|||||||
raise ValueError("'viz.vcodec' must be non-empty.")
|
raise ValueError("'viz.vcodec' must be non-empty.")
|
||||||
if self.frame_storage_mode not in {"memory", "disk"}:
|
if self.frame_storage_mode not in {"memory", "disk"}:
|
||||||
raise ValueError("'viz.frame_storage_mode' must be one of {'memory', 'disk'}.")
|
raise ValueError("'viz.frame_storage_mode' must be one of {'memory', 'disk'}.")
|
||||||
|
if self.smooth_window < 1:
|
||||||
|
raise ValueError("'viz.smooth_window' must be >= 1. Use 1 to disable smoothing.")
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@ -440,10 +440,12 @@ def _export_visualization_outputs(
|
|||||||
indicator_field=cfg.acp.indicator_field,
|
indicator_field=cfg.acp.indicator_field,
|
||||||
viz_episodes=cfg.viz.episodes,
|
viz_episodes=cfg.viz.episodes,
|
||||||
video_key=cfg.viz.video_key,
|
video_key=cfg.viz.video_key,
|
||||||
|
video_keys=cfg.viz.video_keys,
|
||||||
output_dir=viz_output_dir,
|
output_dir=viz_output_dir,
|
||||||
overwrite=cfg.viz.overwrite,
|
overwrite=cfg.viz.overwrite,
|
||||||
vcodec=cfg.viz.vcodec,
|
vcodec=cfg.viz.vcodec,
|
||||||
frame_storage_mode=cfg.viz.frame_storage_mode,
|
frame_storage_mode=cfg.viz.frame_storage_mode,
|
||||||
|
smooth_window=cfg.viz.smooth_window,
|
||||||
)
|
)
|
||||||
logging.info("Exported %d overlay videos to %s", len(written_videos), viz_output_dir)
|
logging.info("Exported %d overlay videos to %s", len(written_videos), viz_output_dir)
|
||||||
return [str(path) for path in written_videos]
|
return [str(path) for path in written_videos]
|
||||||
|
|||||||
@ -4,6 +4,7 @@ from pathlib import Path
|
|||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
from scipy.signal import savgol_filter
|
||||||
from PIL import Image, ImageDraw, ImageFont
|
from PIL import Image, ImageDraw, ImageFont
|
||||||
from tqdm.auto import tqdm
|
from tqdm.auto import tqdm
|
||||||
|
|
||||||
@ -11,6 +12,20 @@ from lerobot.datasets.video_utils import decode_video_frames, encode_video_frame
|
|||||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||||
|
|
||||||
|
|
||||||
|
def _smooth_1d(arr: np.ndarray, window: int = 1) -> np.ndarray:
|
||||||
|
"""Savitzky-Golay smoothing for 1-D array. Use window=1 to disable."""
|
||||||
|
if window <= 1 or arr.shape[0] < 5:
|
||||||
|
return arr.copy()
|
||||||
|
|
||||||
|
w = min(int(window), int(arr.shape[0]))
|
||||||
|
if w % 2 == 0:
|
||||||
|
w -= 1
|
||||||
|
if w < 5:
|
||||||
|
return arr.copy()
|
||||||
|
|
||||||
|
return savgol_filter(arr, window_length=w, polyorder=3).astype(arr.dtype)
|
||||||
|
|
||||||
|
|
||||||
def _select_video_key(camera_keys: list[str], requested_video_key: str | None) -> str:
|
def _select_video_key(camera_keys: list[str], requested_video_key: str | None) -> str:
|
||||||
if len(camera_keys) == 0:
|
if len(camera_keys) == 0:
|
||||||
raise ValueError("No camera key found in dataset.")
|
raise ValueError("No camera key found in dataset.")
|
||||||
@ -26,6 +41,33 @@ def _select_video_key(camera_keys: list[str], requested_video_key: str | None) -
|
|||||||
return camera_keys[0]
|
return camera_keys[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _select_video_keys(
|
||||||
|
camera_keys: list[str],
|
||||||
|
requested_video_keys: str | None,
|
||||||
|
requested_video_key: str | None,
|
||||||
|
) -> list[str]:
|
||||||
|
if len(camera_keys) == 0:
|
||||||
|
raise ValueError("No camera key found in dataset.")
|
||||||
|
|
||||||
|
if requested_video_keys is None or not requested_video_keys.strip():
|
||||||
|
return [_select_video_key(camera_keys, requested_video_key)]
|
||||||
|
|
||||||
|
resolved: list[str] = []
|
||||||
|
for raw_key in requested_video_keys.split(','):
|
||||||
|
key = raw_key.strip()
|
||||||
|
if not key:
|
||||||
|
continue
|
||||||
|
if key not in camera_keys:
|
||||||
|
raise ValueError(f"Unknown video key '{key}'. Available camera keys: {camera_keys}")
|
||||||
|
if key not in resolved:
|
||||||
|
resolved.append(key)
|
||||||
|
|
||||||
|
if not resolved:
|
||||||
|
raise ValueError("'viz.video_keys' is empty after parsing. Provide comma-separated camera keys.")
|
||||||
|
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
def _parse_episodes_arg(episodes_arg: str, total_episodes: int) -> list[int]:
|
def _parse_episodes_arg(episodes_arg: str, total_episodes: int) -> list[int]:
|
||||||
value = episodes_arg.strip().lower()
|
value = episodes_arg.strip().lower()
|
||||||
if value == "all":
|
if value == "all":
|
||||||
@ -116,70 +158,88 @@ def _draw_overlay(
|
|||||||
y_max: float,
|
y_max: float,
|
||||||
indicators: np.ndarray | None = None,
|
indicators: np.ndarray | None = None,
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
|
# Keep signature/call-sites stable; style does not use these signals.
|
||||||
|
_ = (advantage_t, acp_t, highlight_current_point, indicators)
|
||||||
|
|
||||||
rgba = frame.convert("RGBA")
|
rgba = frame.convert("RGBA")
|
||||||
overlay = Image.new("RGBA", rgba.size, (0, 0, 0, 0))
|
overlay = Image.new("RGBA", rgba.size, (0, 0, 0, 0))
|
||||||
draw = ImageDraw.Draw(overlay)
|
draw = ImageDraw.Draw(overlay)
|
||||||
|
|
||||||
width, height = rgba.size
|
width, height = rgba.size
|
||||||
margin = max(10, width // 80)
|
n = len(values)
|
||||||
chart_h = max(72, height // 4)
|
if n == 0:
|
||||||
chart_w = width - 2 * margin
|
return Image.alpha_composite(rgba, overlay).convert("RGB")
|
||||||
chart_x0 = margin
|
|
||||||
chart_y0 = height - margin - chart_h
|
|
||||||
|
|
||||||
draw.rectangle(
|
margin_x = max(12, width // 96)
|
||||||
(chart_x0 - 4, chart_y0 - 4, chart_x0 + chart_w + 4, chart_y0 + chart_h + 4),
|
margin_top = max(8, height // 72)
|
||||||
fill=(0, 0, 0, 110),
|
margin_bottom = max(24, height // 36)
|
||||||
)
|
chart_x0 = margin_x
|
||||||
draw.line((chart_x0, chart_y0, chart_x0, chart_y0 + chart_h), fill=(200, 200, 200, 160), width=1)
|
chart_x1 = width - margin_x
|
||||||
draw.line(
|
chart_y0 = margin_top
|
||||||
(chart_x0, chart_y0 + chart_h, chart_x0 + chart_w, chart_y0 + chart_h),
|
chart_y1 = height - margin_bottom
|
||||||
fill=(200, 200, 200, 160),
|
chart_w = max(1, chart_x1 - chart_x0)
|
||||||
width=1,
|
chart_h = max(1, chart_y1 - chart_y0)
|
||||||
|
|
||||||
|
denom_y = max(1e-6, y_max - y_min)
|
||||||
|
padding = denom_y * 0.05
|
||||||
|
val_min = y_min - padding
|
||||||
|
val_max = y_max + padding
|
||||||
|
|
||||||
|
last = min(current_step, n - 1)
|
||||||
|
denom_x = max(1, n - 1)
|
||||||
|
|
||||||
|
# Light grid + current-frame vertical cursor.
|
||||||
|
grid_color = (255, 255, 255, 30)
|
||||||
|
for frac in (0.25, 0.5, 0.75):
|
||||||
|
gy = int(round(chart_y0 + frac * chart_h))
|
||||||
|
draw.line([(chart_x0, gy), (chart_x1, gy)], fill=grid_color, width=1)
|
||||||
|
cx = int(round(chart_x0 + chart_w * (last / denom_x)))
|
||||||
|
draw.line([(cx, chart_y0), (cx, chart_y1)], fill=(255, 255, 255, 40), width=1)
|
||||||
|
|
||||||
|
points = _curve_points(
|
||||||
|
values=values,
|
||||||
|
current_step=last,
|
||||||
|
x0=chart_x0,
|
||||||
|
y0=chart_y0,
|
||||||
|
width=chart_w,
|
||||||
|
height=chart_h,
|
||||||
|
y_min=val_min,
|
||||||
|
y_max=val_max,
|
||||||
)
|
)
|
||||||
|
|
||||||
mid_y = chart_y0 + chart_h // 2
|
curve_width = max(2, width // 600)
|
||||||
draw.line((chart_x0, mid_y, chart_x0 + chart_w, mid_y), fill=(120, 120, 120, 120), width=1)
|
|
||||||
|
|
||||||
points = _curve_points(values, current_step, chart_x0, chart_y0, chart_w, chart_h, y_min, y_max)
|
|
||||||
curve_width = max(2, width // 320)
|
|
||||||
white = (255, 255, 255, 255)
|
|
||||||
red = (255, 80, 80, 255)
|
|
||||||
if len(points) >= 2:
|
if len(points) >= 2:
|
||||||
if indicators is not None:
|
draw.line(points, fill=(100, 200, 255, 220), width=curve_width)
|
||||||
for i in range(len(points) - 1):
|
|
||||||
seg_color = red if int(indicators[i]) == 1 or int(indicators[i + 1]) == 1 else white
|
|
||||||
draw.line([points[i], points[i + 1]], fill=seg_color, width=curve_width)
|
|
||||||
else:
|
|
||||||
draw.line(points, fill=white, width=curve_width)
|
|
||||||
point_color = red if highlight_current_point else white
|
|
||||||
if len(points) == 1:
|
|
||||||
x, y = points[0]
|
|
||||||
draw.ellipse((x - curve_width, y - curve_width, x + curve_width, y + curve_width), fill=point_color)
|
|
||||||
elif len(points) > 1:
|
|
||||||
x, y = points[-1]
|
|
||||||
radius = max(2, curve_width + 1)
|
|
||||||
draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=point_color)
|
|
||||||
|
|
||||||
font_size = max(18, height // 26)
|
if len(points) >= 1:
|
||||||
font = _load_font(font_size)
|
px, py = points[-1]
|
||||||
lines = [f"advantage: {advantage_t:+.4f}", f"acp_indicator: {int(acp_t)}"]
|
radius = max(4, curve_width + 2)
|
||||||
line_sizes = [draw.textbbox((0, 0), text, font=font) for text in lines]
|
draw.ellipse((px - radius, py - radius, px + radius, py + radius), fill=(255, 255, 100, 255))
|
||||||
text_w = max(box[2] - box[0] for box in line_sizes)
|
draw.ellipse(
|
||||||
text_h = sum(box[3] - box[1] for box in line_sizes) + max(4, font_size // 4)
|
(px - radius - 2, py - radius - 2, px + radius + 2, py + radius + 2),
|
||||||
box_pad = max(8, font_size // 3)
|
outline=(255, 255, 100, 180),
|
||||||
box_x1 = width - margin
|
width=1,
|
||||||
box_x0 = box_x1 - text_w - 2 * box_pad
|
)
|
||||||
box_y0 = margin
|
|
||||||
box_y1 = box_y0 + text_h + 2 * box_pad
|
|
||||||
draw.rectangle((box_x0, box_y0, box_x1, box_y1), fill=(0, 0, 0, 150))
|
|
||||||
|
|
||||||
text_y = box_y0 + box_pad
|
value_font = _load_font(max(16, height // 22))
|
||||||
for idx, text in enumerate(lines):
|
value_text = f"{float(values[last]):.4f}"
|
||||||
box = line_sizes[idx]
|
value_bbox = draw.textbbox((0, 0), value_text, font=value_font)
|
||||||
line_h = box[3] - box[1]
|
text_w = value_bbox[2] - value_bbox[0]
|
||||||
draw.text((box_x0 + box_pad, text_y), text, fill=(255, 255, 255, 255), font=font)
|
text_h = value_bbox[3] - value_bbox[1]
|
||||||
text_y += line_h + max(4, font_size // 4)
|
tx = min(px + 10, width - text_w - 10)
|
||||||
|
ty = max(py - text_h - 12, chart_y0 + 2)
|
||||||
|
draw.rectangle((tx - 4, ty - 2, tx + text_w + 4, ty + text_h + 2), fill=(0, 0, 0, 160))
|
||||||
|
draw.text((tx, ty), value_text, fill=(255, 255, 100, 255), font=value_font)
|
||||||
|
|
||||||
|
small_font = _load_font(max(13, height // 30))
|
||||||
|
frame_text = f"frame {last}/{n - 1}"
|
||||||
|
frame_bbox = draw.textbbox((0, 0), frame_text, font=small_font)
|
||||||
|
frame_w = frame_bbox[2] - frame_bbox[0]
|
||||||
|
frame_h = frame_bbox[3] - frame_bbox[1]
|
||||||
|
fx = width - frame_w - margin_x - 4
|
||||||
|
fy = height - frame_h - 6
|
||||||
|
draw.rectangle((fx - 3, fy - 1, fx + frame_w + 3, fy + frame_h + 1), fill=(0, 0, 0, 140))
|
||||||
|
draw.text((fx, fy), frame_text, fill=(200, 200, 200, 220), font=small_font)
|
||||||
|
|
||||||
return Image.alpha_composite(rgba, overlay).convert("RGB")
|
return Image.alpha_composite(rgba, overlay).convert("RGB")
|
||||||
|
|
||||||
@ -190,6 +250,17 @@ def _build_output_video_path(output_dir: Path, repo_id: str, video_key: str, epi
|
|||||||
return output_dir / f"{repo_tag}_episode_{episode_index:04d}_{key_tag}.mp4"
|
return output_dir / f"{repo_tag}_episode_{episode_index:04d}_{key_tag}.mp4"
|
||||||
|
|
||||||
|
|
||||||
|
def _build_output_video_path_multiview(
|
||||||
|
output_dir: Path,
|
||||||
|
repo_id: str,
|
||||||
|
video_keys: list[str],
|
||||||
|
episode_index: int,
|
||||||
|
) -> Path:
|
||||||
|
repo_tag = repo_id.replace("/", "_")
|
||||||
|
keys_tag = "__".join(key.replace(".", "_") for key in video_keys)
|
||||||
|
return output_dir / f"{repo_tag}_episode_{episode_index:04d}_{keys_tag}_multiview.mp4"
|
||||||
|
|
||||||
|
|
||||||
def _decode_frames_at_timestamps(
|
def _decode_frames_at_timestamps(
|
||||||
video_file: Path,
|
video_file: Path,
|
||||||
timestamps_s: np.ndarray,
|
timestamps_s: np.ndarray,
|
||||||
@ -309,7 +380,10 @@ def _export_single_episode(
|
|||||||
video_backend: str | None,
|
video_backend: str | None,
|
||||||
frame_storage_mode: str = "memory",
|
frame_storage_mode: str = "memory",
|
||||||
temp_dir_root: Path | None = None,
|
temp_dir_root: Path | None = None,
|
||||||
|
smooth_window: int = 1,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
|
ep_values = _smooth_1d(ep_values, smooth_window)
|
||||||
|
ep_advantages = _smooth_1d(ep_advantages, smooth_window)
|
||||||
y_min, y_max = _get_episode_value_bounds(ep_values)
|
y_min, y_max = _get_episode_value_bounds(ep_values)
|
||||||
decoded_frames = _decode_frames_at_timestamps(
|
decoded_frames = _decode_frames_at_timestamps(
|
||||||
video_file=src_video_path,
|
video_file=src_video_path,
|
||||||
@ -370,6 +444,104 @@ def _export_single_episode(
|
|||||||
return dst_video_path
|
return dst_video_path
|
||||||
|
|
||||||
|
|
||||||
|
def _export_single_episode_multiview(
|
||||||
|
src_video_paths: list[Path],
|
||||||
|
camera_labels: list[str],
|
||||||
|
dst_video_path: Path,
|
||||||
|
ep_values: np.ndarray,
|
||||||
|
ep_advantages: np.ndarray,
|
||||||
|
ep_indicators: np.ndarray,
|
||||||
|
episode_timestamps_per_cam: list[np.ndarray],
|
||||||
|
fps: int,
|
||||||
|
vcodec: str,
|
||||||
|
tolerance_s: float,
|
||||||
|
video_backend: str | None,
|
||||||
|
frame_storage_mode: str = "memory",
|
||||||
|
temp_dir_root: Path | None = None,
|
||||||
|
smooth_window: int = 1,
|
||||||
|
) -> Path:
|
||||||
|
ep_values = _smooth_1d(ep_values, smooth_window)
|
||||||
|
ep_advantages = _smooth_1d(ep_advantages, smooth_window)
|
||||||
|
y_min, y_max = _get_episode_value_bounds(ep_values)
|
||||||
|
|
||||||
|
all_cam_frames: list[list[Image.Image]] = []
|
||||||
|
for src_path, ts in zip(src_video_paths, episode_timestamps_per_cam):
|
||||||
|
frames = _decode_frames_at_timestamps(
|
||||||
|
video_file=src_path,
|
||||||
|
timestamps_s=ts,
|
||||||
|
tolerance_s=tolerance_s,
|
||||||
|
backend=video_backend,
|
||||||
|
)
|
||||||
|
all_cam_frames.append(frames)
|
||||||
|
|
||||||
|
n_frames = min(len(f) for f in all_cam_frames)
|
||||||
|
n_frames = min(n_frames, len(ep_values))
|
||||||
|
if n_frames == 0:
|
||||||
|
raise ValueError(f"No decoded frames for multiview video: {dst_video_path}")
|
||||||
|
|
||||||
|
single_w = all_cam_frames[0][0].width
|
||||||
|
single_h = all_cam_frames[0][0].height
|
||||||
|
n_cams = len(all_cam_frames)
|
||||||
|
total_w = single_w * n_cams
|
||||||
|
|
||||||
|
label_font_size = max(14, single_h // 30)
|
||||||
|
label_font = _load_font(label_font_size)
|
||||||
|
|
||||||
|
def _compose_frame(i: int) -> Image.Image:
|
||||||
|
wide_frame = Image.new("RGB", (total_w, single_h))
|
||||||
|
for cam_idx, cam_frames in enumerate(all_cam_frames):
|
||||||
|
wide_frame.paste(cam_frames[i].resize((single_w, single_h)), (cam_idx * single_w, 0))
|
||||||
|
|
||||||
|
label_draw = ImageDraw.Draw(wide_frame)
|
||||||
|
for cam_idx, label in enumerate(camera_labels):
|
||||||
|
short_label = label.split(".")[-1]
|
||||||
|
lx = cam_idx * single_w + 8
|
||||||
|
ly = 4
|
||||||
|
bbox = label_draw.textbbox((lx, ly), short_label, font=label_font)
|
||||||
|
label_draw.rectangle(
|
||||||
|
(bbox[0] - 2, bbox[1] - 2, bbox[2] + 4, bbox[3] + 2),
|
||||||
|
fill=(0, 0, 0, 180),
|
||||||
|
)
|
||||||
|
label_draw.text((lx, ly), short_label, fill=(255, 255, 200), font=label_font)
|
||||||
|
|
||||||
|
return _draw_overlay(
|
||||||
|
frame=wide_frame,
|
||||||
|
values=ep_values,
|
||||||
|
current_step=i,
|
||||||
|
advantage_t=float(ep_advantages[i]) if i < len(ep_advantages) else 0.0,
|
||||||
|
acp_t=int(ep_indicators[i]) if i < len(ep_indicators) else 0,
|
||||||
|
highlight_current_point=False,
|
||||||
|
y_min=y_min,
|
||||||
|
y_max=y_max,
|
||||||
|
indicators=ep_indicators,
|
||||||
|
)
|
||||||
|
|
||||||
|
if frame_storage_mode == "disk":
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
dir=str(temp_dir_root) if temp_dir_root is not None else None,
|
||||||
|
prefix=f"{dst_video_path.stem}-frames-",
|
||||||
|
) as temp_dir:
|
||||||
|
temp_path = Path(temp_dir)
|
||||||
|
for i in range(n_frames):
|
||||||
|
_compose_frame(i).save(temp_path / f"frame-{i:06d}.png")
|
||||||
|
|
||||||
|
encode_video_frames(
|
||||||
|
imgs_dir=temp_path,
|
||||||
|
video_path=dst_video_path,
|
||||||
|
fps=fps,
|
||||||
|
vcodec=vcodec,
|
||||||
|
overwrite=True,
|
||||||
|
)
|
||||||
|
return dst_video_path
|
||||||
|
|
||||||
|
composed_frames: list[Image.Image] = []
|
||||||
|
for i in range(n_frames):
|
||||||
|
composed_frames.append(_compose_frame(i))
|
||||||
|
|
||||||
|
_encode_pil_to_video(composed_frames, dst_video_path, fps, vcodec)
|
||||||
|
return dst_video_path
|
||||||
|
|
||||||
|
|
||||||
def _export_overlay_videos(
|
def _export_overlay_videos(
|
||||||
dataset: LeRobotDataset,
|
dataset: LeRobotDataset,
|
||||||
value_field: str,
|
value_field: str,
|
||||||
@ -377,12 +549,19 @@ def _export_overlay_videos(
|
|||||||
indicator_field: str,
|
indicator_field: str,
|
||||||
viz_episodes: str,
|
viz_episodes: str,
|
||||||
video_key: str | None,
|
video_key: str | None,
|
||||||
|
video_keys: str | None,
|
||||||
output_dir: Path,
|
output_dir: Path,
|
||||||
overwrite: bool,
|
overwrite: bool,
|
||||||
vcodec: str,
|
vcodec: str,
|
||||||
frame_storage_mode: str = "memory",
|
frame_storage_mode: str = "memory",
|
||||||
|
smooth_window: int = 1,
|
||||||
) -> list[Path]:
|
) -> list[Path]:
|
||||||
selected_video_key = _select_video_key(dataset.meta.camera_keys, video_key)
|
selected_video_keys = _select_video_keys(
|
||||||
|
camera_keys=list(dataset.meta.camera_keys),
|
||||||
|
requested_video_keys=video_keys,
|
||||||
|
requested_video_key=video_key,
|
||||||
|
)
|
||||||
|
multiview_mode = len(selected_video_keys) > 1
|
||||||
|
|
||||||
raw_dataset = dataset.hf_dataset.with_format(None)
|
raw_dataset = dataset.hf_dataset.with_format(None)
|
||||||
column_names = set(raw_dataset.column_names)
|
column_names = set(raw_dataset.column_names)
|
||||||
@ -400,6 +579,7 @@ def _export_overlay_videos(
|
|||||||
indicators_all = _to_1d_int(raw_dataset[indicator_field])
|
indicators_all = _to_1d_int(raw_dataset[indicator_field])
|
||||||
else:
|
else:
|
||||||
indicators_all = np.zeros(values_all.shape[0], dtype=np.int64)
|
indicators_all = np.zeros(values_all.shape[0], dtype=np.int64)
|
||||||
|
|
||||||
episode_indices_all = np.asarray(raw_dataset["episode_index"], dtype=np.int64).reshape(-1)
|
episode_indices_all = np.asarray(raw_dataset["episode_index"], dtype=np.int64).reshape(-1)
|
||||||
frame_indices_all = np.asarray(raw_dataset["frame_index"], dtype=np.int64).reshape(-1)
|
frame_indices_all = np.asarray(raw_dataset["frame_index"], dtype=np.int64).reshape(-1)
|
||||||
if "timestamp" in column_names:
|
if "timestamp" in column_names:
|
||||||
@ -420,58 +600,141 @@ def _export_overlay_videos(
|
|||||||
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
tasks = []
|
|
||||||
fps = int(dataset.fps)
|
fps = int(dataset.fps)
|
||||||
for ep in episodes:
|
tolerance_s = float(getattr(dataset, "tolerance_s", 1e-4))
|
||||||
ep_positions = np.flatnonzero(episode_indices_all == ep)
|
video_backend = getattr(dataset, "video_backend", None)
|
||||||
if ep_positions.shape[0] > 1:
|
written_paths: list[Path] = []
|
||||||
|
|
||||||
|
if multiview_mode:
|
||||||
|
tasks = []
|
||||||
|
for ep in episodes:
|
||||||
|
ep_positions = np.flatnonzero(episode_indices_all == ep)
|
||||||
|
if ep_positions.shape[0] == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
ep_frame_indices = frame_indices_all[ep_positions]
|
ep_frame_indices = frame_indices_all[ep_positions]
|
||||||
if bool(np.any(np.diff(ep_frame_indices) < 0)):
|
if bool(np.any(np.diff(ep_frame_indices) < 0)):
|
||||||
ep_positions = ep_positions[np.argsort(ep_frame_indices, kind="stable")]
|
ep_positions = ep_positions[np.argsort(ep_frame_indices, kind="stable")]
|
||||||
|
|
||||||
ep_values = values_all[ep_positions]
|
ep_values = values_all[ep_positions]
|
||||||
if ep_values.shape[0] == 0:
|
if ep_values.shape[0] == 0:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
ep_timestamps = timestamps_all[ep_positions]
|
ep_timestamps = timestamps_all[ep_positions]
|
||||||
from_ts, to_ts = _get_episode_video_time_bounds(dataset, ep, selected_video_key)
|
src_paths: list[Path] = []
|
||||||
ep_video_timestamps = from_ts + ep_timestamps
|
ts_per_cam: list[np.ndarray] = []
|
||||||
if to_ts is not None:
|
for cam_key in selected_video_keys:
|
||||||
ep_video_timestamps = np.minimum(ep_video_timestamps, to_ts)
|
src_path = Path(dataset.root) / dataset.meta.get_video_file_path(ep, cam_key)
|
||||||
|
src_paths.append(src_path)
|
||||||
|
from_ts, to_ts = _get_episode_video_time_bounds(dataset, ep, cam_key)
|
||||||
|
cam_ts = from_ts + ep_timestamps
|
||||||
|
if to_ts is not None:
|
||||||
|
cam_ts = np.minimum(cam_ts, to_ts)
|
||||||
|
ts_per_cam.append(cam_ts)
|
||||||
|
|
||||||
dst_path = _build_output_video_path(output_dir, dataset.repo_id, selected_video_key, ep)
|
dst_path = _build_output_video_path_multiview(
|
||||||
if dst_path.exists() and not overwrite:
|
output_dir=output_dir,
|
||||||
continue
|
repo_id=dataset.repo_id,
|
||||||
|
video_keys=selected_video_keys,
|
||||||
src_path = Path(dataset.root) / dataset.meta.get_video_file_path(ep, selected_video_key)
|
episode_index=ep,
|
||||||
tasks.append(
|
|
||||||
(
|
|
||||||
src_path,
|
|
||||||
dst_path,
|
|
||||||
ep_values,
|
|
||||||
advantages_all[ep_positions],
|
|
||||||
indicators_all[ep_positions],
|
|
||||||
ep_video_timestamps,
|
|
||||||
)
|
)
|
||||||
)
|
if dst_path.exists() and not overwrite:
|
||||||
|
continue
|
||||||
|
|
||||||
written_paths: list[Path] = []
|
tasks.append(
|
||||||
for src, dst, vals, advs, inds, ts in tqdm(tasks, total=len(tasks), desc="Export value overlay videos", leave=False):
|
(
|
||||||
written_paths.append(
|
src_paths,
|
||||||
_export_single_episode(
|
dst_path,
|
||||||
src,
|
ep_values,
|
||||||
dst,
|
advantages_all[ep_positions],
|
||||||
vals,
|
indicators_all[ep_positions],
|
||||||
advs,
|
ts_per_cam,
|
||||||
inds,
|
)
|
||||||
ts,
|
)
|
||||||
fps,
|
|
||||||
vcodec,
|
desc_keys = ",".join(key.split(".")[-1] for key in selected_video_keys)
|
||||||
float(getattr(dataset, "tolerance_s", 1e-4)),
|
for srcs, dst, vals, advs, inds, ts_per_cam in tqdm(
|
||||||
getattr(dataset, "video_backend", None),
|
tasks,
|
||||||
frame_storage_mode,
|
total=len(tasks),
|
||||||
output_dir,
|
desc=f"Export overlay multiview [{desc_keys}]",
|
||||||
|
leave=False,
|
||||||
|
):
|
||||||
|
written_paths.append(
|
||||||
|
_export_single_episode_multiview(
|
||||||
|
src_video_paths=srcs,
|
||||||
|
camera_labels=selected_video_keys,
|
||||||
|
dst_video_path=dst,
|
||||||
|
ep_values=vals,
|
||||||
|
ep_advantages=advs,
|
||||||
|
ep_indicators=inds,
|
||||||
|
episode_timestamps_per_cam=ts_per_cam,
|
||||||
|
fps=fps,
|
||||||
|
vcodec=vcodec,
|
||||||
|
tolerance_s=tolerance_s,
|
||||||
|
video_backend=video_backend,
|
||||||
|
frame_storage_mode=frame_storage_mode,
|
||||||
|
temp_dir_root=output_dir,
|
||||||
|
smooth_window=smooth_window,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
selected_video_key = selected_video_keys[0]
|
||||||
|
tasks = []
|
||||||
|
for ep in episodes:
|
||||||
|
ep_positions = np.flatnonzero(episode_indices_all == ep)
|
||||||
|
if ep_positions.shape[0] > 1:
|
||||||
|
ep_frame_indices = frame_indices_all[ep_positions]
|
||||||
|
if bool(np.any(np.diff(ep_frame_indices) < 0)):
|
||||||
|
ep_positions = ep_positions[np.argsort(ep_frame_indices, kind="stable")]
|
||||||
|
|
||||||
|
ep_values = values_all[ep_positions]
|
||||||
|
if ep_values.shape[0] == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
ep_timestamps = timestamps_all[ep_positions]
|
||||||
|
from_ts, to_ts = _get_episode_video_time_bounds(dataset, ep, selected_video_key)
|
||||||
|
ep_video_timestamps = from_ts + ep_timestamps
|
||||||
|
if to_ts is not None:
|
||||||
|
ep_video_timestamps = np.minimum(ep_video_timestamps, to_ts)
|
||||||
|
|
||||||
|
dst_path = _build_output_video_path(output_dir, dataset.repo_id, selected_video_key, ep)
|
||||||
|
if dst_path.exists() and not overwrite:
|
||||||
|
continue
|
||||||
|
|
||||||
|
src_path = Path(dataset.root) / dataset.meta.get_video_file_path(ep, selected_video_key)
|
||||||
|
tasks.append(
|
||||||
|
(
|
||||||
|
src_path,
|
||||||
|
dst_path,
|
||||||
|
ep_values,
|
||||||
|
advantages_all[ep_positions],
|
||||||
|
indicators_all[ep_positions],
|
||||||
|
ep_video_timestamps,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for src, dst, vals, advs, inds, ts in tqdm(
|
||||||
|
tasks,
|
||||||
|
total=len(tasks),
|
||||||
|
desc=f"Export overlay [{selected_video_key}]",
|
||||||
|
leave=False,
|
||||||
|
):
|
||||||
|
written_paths.append(
|
||||||
|
_export_single_episode(
|
||||||
|
src,
|
||||||
|
dst,
|
||||||
|
vals,
|
||||||
|
advs,
|
||||||
|
inds,
|
||||||
|
ts,
|
||||||
|
fps,
|
||||||
|
vcodec,
|
||||||
|
tolerance_s,
|
||||||
|
video_backend,
|
||||||
|
frame_storage_mode,
|
||||||
|
output_dir,
|
||||||
|
smooth_window=smooth_window,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
|
||||||
|
|
||||||
return written_paths
|
return written_paths
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户