ToDo
这个提交包含在:
10
TODO.MD
普通文件
10
TODO.MD
普通文件
@ -0,0 +1,10 @@
|
|||||||
|
# ToDos
|
||||||
|
## data_controller
|
||||||
|
- 给手臂驱动加上真实的控制机器人的部分
|
||||||
|
- _get_robot_state 按照机器人实际情况改
|
||||||
|
|
||||||
|
## trainer
|
||||||
|
- 不能训练平均值,需要生成完整的动作序列
|
||||||
|
|
||||||
|
## microphone
|
||||||
|
- 移除语言识别部分
|
||||||
518
annotation_tool.py
普通文件
518
annotation_tool.py
普通文件
@ -0,0 +1,518 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
数据标注工具 - 用于为录制的机器人动作数据添加语音指令标注
|
||||||
|
基于LeRobot框架的语音输入法训练系统
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import tkinter as tk
|
||||||
|
from tkinter import ttk, filedialog, messagebox
|
||||||
|
import h5py
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
|
||||||
|
from matplotlib.figure import Figure
|
||||||
|
import cv2
|
||||||
|
from PIL import Image, ImageTk
|
||||||
|
from typing import Dict, List, Optional, Any
|
||||||
|
from dataclasses import dataclass, asdict
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Annotation:
|
||||||
|
"""标注数据结构"""
|
||||||
|
start_time: float
|
||||||
|
end_time: float
|
||||||
|
text: str
|
||||||
|
confidence: float = 1.0
|
||||||
|
annotator: str = "default"
|
||||||
|
timestamp: float = None
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
if self.timestamp is None:
|
||||||
|
self.timestamp = time.time()
|
||||||
|
|
||||||
|
|
||||||
|
class AnnotationTool:
|
||||||
|
"""数据标注工具GUI"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.root = tk.Tk()
|
||||||
|
self.root.title("语音机器人训练系统 - 数据标注工具")
|
||||||
|
self.root.geometry("1200x800")
|
||||||
|
|
||||||
|
# 数据相关变量
|
||||||
|
self.session_dir = None
|
||||||
|
self.robot_data = None
|
||||||
|
self.metadata = None
|
||||||
|
self.annotations = []
|
||||||
|
self.current_time = 0.0
|
||||||
|
self.total_time = 0.0
|
||||||
|
self.playing = False
|
||||||
|
self.play_thread = None
|
||||||
|
|
||||||
|
# GUI组件
|
||||||
|
self.setup_gui()
|
||||||
|
|
||||||
|
# 绑定键盘事件
|
||||||
|
self.root.bind('<Key>', self.on_key_press)
|
||||||
|
self.root.focus_set()
|
||||||
|
|
||||||
|
def setup_gui(self):
|
||||||
|
"""设置GUI界面"""
|
||||||
|
# 主框架
|
||||||
|
main_frame = ttk.Frame(self.root)
|
||||||
|
main_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
|
||||||
|
|
||||||
|
# 顶部控制面板
|
||||||
|
control_frame = ttk.Frame(main_frame)
|
||||||
|
control_frame.pack(fill=tk.X, pady=(0, 10))
|
||||||
|
|
||||||
|
# 文件操作按钮
|
||||||
|
ttk.Button(control_frame, text="加载数据", command=self.load_data).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
ttk.Button(control_frame, text="保存标注", command=self.save_annotations).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
ttk.Button(control_frame, text="导出数据", command=self.export_data).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
|
||||||
|
# 播放控制按钮
|
||||||
|
ttk.Separator(control_frame, orient=tk.VERTICAL).pack(side=tk.LEFT, padx=10, fill=tk.Y)
|
||||||
|
ttk.Button(control_frame, text="播放/暂停", command=self.toggle_play).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
ttk.Button(control_frame, text="重置", command=self.reset_playback).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
|
||||||
|
# 时间显示
|
||||||
|
self.time_label = ttk.Label(control_frame, text="时间: 0.00 / 0.00 秒")
|
||||||
|
self.time_label.pack(side=tk.RIGHT)
|
||||||
|
|
||||||
|
# 中间内容区域
|
||||||
|
content_frame = ttk.Frame(main_frame)
|
||||||
|
content_frame.pack(fill=tk.BOTH, expand=True)
|
||||||
|
|
||||||
|
# 左侧:数据可视化
|
||||||
|
left_frame = ttk.LabelFrame(content_frame, text="数据可视化")
|
||||||
|
left_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=(0, 5))
|
||||||
|
|
||||||
|
# 创建matplotlib图表
|
||||||
|
self.fig = Figure(figsize=(8, 6), dpi=100)
|
||||||
|
self.canvas = FigureCanvasTkAgg(self.fig, left_frame)
|
||||||
|
self.canvas.get_tk_widget().pack(fill=tk.BOTH, expand=True)
|
||||||
|
|
||||||
|
# 时间滑块
|
||||||
|
self.time_scale = ttk.Scale(left_frame, from_=0, to=100, orient=tk.HORIZONTAL, command=self.on_time_change)
|
||||||
|
self.time_scale.pack(fill=tk.X, pady=5)
|
||||||
|
|
||||||
|
# 右侧:标注面板
|
||||||
|
right_frame = ttk.LabelFrame(content_frame, text="标注面板", width=300)
|
||||||
|
right_frame.pack(side=tk.RIGHT, fill=tk.Y, padx=(5, 0))
|
||||||
|
right_frame.pack_propagate(False)
|
||||||
|
|
||||||
|
# 当前标注输入
|
||||||
|
ttk.Label(right_frame, text="语音指令:").pack(anchor=tk.W, pady=(10, 0))
|
||||||
|
self.text_entry = tk.Text(right_frame, height=3, wrap=tk.WORD)
|
||||||
|
self.text_entry.pack(fill=tk.X, pady=5)
|
||||||
|
|
||||||
|
# 时间范围设置
|
||||||
|
time_frame = ttk.Frame(right_frame)
|
||||||
|
time_frame.pack(fill=tk.X, pady=5)
|
||||||
|
|
||||||
|
ttk.Label(time_frame, text="开始时间:").grid(row=0, column=0, sticky=tk.W)
|
||||||
|
self.start_time_var = tk.StringVar(value="0.00")
|
||||||
|
ttk.Entry(time_frame, textvariable=self.start_time_var, width=10).grid(row=0, column=1, padx=(5, 0))
|
||||||
|
|
||||||
|
ttk.Label(time_frame, text="结束时间:").grid(row=1, column=0, sticky=tk.W)
|
||||||
|
self.end_time_var = tk.StringVar(value="0.00")
|
||||||
|
ttk.Entry(time_frame, textvariable=self.end_time_var, width=10).grid(row=1, column=1, padx=(5, 0))
|
||||||
|
|
||||||
|
# 标注操作按钮
|
||||||
|
button_frame = ttk.Frame(right_frame)
|
||||||
|
button_frame.pack(fill=tk.X, pady=10)
|
||||||
|
|
||||||
|
ttk.Button(button_frame, text="设置开始", command=self.set_start_time).pack(side=tk.LEFT, padx=(0, 5))
|
||||||
|
ttk.Button(button_frame, text="设置结束", command=self.set_end_time).pack(side=tk.LEFT)
|
||||||
|
|
||||||
|
ttk.Button(right_frame, text="添加标注", command=self.add_annotation).pack(fill=tk.X, pady=5)
|
||||||
|
ttk.Button(right_frame, text="删除选中", command=self.delete_annotation).pack(fill=tk.X, pady=5)
|
||||||
|
|
||||||
|
# 标注列表
|
||||||
|
ttk.Label(right_frame, text="已有标注:").pack(anchor=tk.W, pady=(10, 0))
|
||||||
|
|
||||||
|
# 创建Treeview来显示标注
|
||||||
|
columns = ('时间范围', '文本')
|
||||||
|
self.annotation_tree = ttk.Treeview(right_frame, columns=columns, show='headings', height=8)
|
||||||
|
|
||||||
|
self.annotation_tree.heading('时间范围', text='时间范围')
|
||||||
|
self.annotation_tree.heading('文本', text='语音指令')
|
||||||
|
|
||||||
|
self.annotation_tree.column('时间范围', width=100)
|
||||||
|
self.annotation_tree.column('文本', width=180)
|
||||||
|
|
||||||
|
self.annotation_tree.pack(fill=tk.BOTH, expand=True, pady=5)
|
||||||
|
|
||||||
|
# 绑定选择事件
|
||||||
|
self.annotation_tree.bind('<<TreeviewSelect>>', self.on_annotation_select)
|
||||||
|
|
||||||
|
# 底部状态栏
|
||||||
|
self.status_label = ttk.Label(main_frame, text="请加载数据文件开始标注")
|
||||||
|
self.status_label.pack(fill=tk.X, pady=(10, 0))
|
||||||
|
|
||||||
|
def load_data(self):
|
||||||
|
"""加载数据文件"""
|
||||||
|
session_dir = filedialog.askdirectory(title="选择数据会话目录")
|
||||||
|
if not session_dir:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 检查必要文件
|
||||||
|
robot_data_path = os.path.join(session_dir, "robot_data.h5")
|
||||||
|
metadata_path = os.path.join(session_dir, "metadata.json")
|
||||||
|
|
||||||
|
if not os.path.exists(robot_data_path):
|
||||||
|
messagebox.showerror("错误", "未找到机器人数据文件 (robot_data.h5)")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not os.path.exists(metadata_path):
|
||||||
|
messagebox.showerror("错误", "未找到元数据文件 (metadata.json)")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 加载数据
|
||||||
|
self.session_dir = session_dir
|
||||||
|
|
||||||
|
# 加载HDF5数据
|
||||||
|
self.robot_data = h5py.File(robot_data_path, 'r')
|
||||||
|
|
||||||
|
# 加载元数据
|
||||||
|
with open(metadata_path, 'r') as f:
|
||||||
|
self.metadata = json.load(f)
|
||||||
|
|
||||||
|
# 设置时间范围
|
||||||
|
timestamps = self.robot_data['timestamps'][:]
|
||||||
|
self.total_time = timestamps[-1] - timestamps[0]
|
||||||
|
self.time_scale.configure(to=self.total_time)
|
||||||
|
|
||||||
|
# 加载已有标注(如果存在)
|
||||||
|
self.load_existing_annotations()
|
||||||
|
|
||||||
|
# 更新显示
|
||||||
|
self.update_visualization()
|
||||||
|
self.update_status(f"已加载数据: {os.path.basename(session_dir)}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
messagebox.showerror("错误", f"加载数据失败: {str(e)}")
|
||||||
|
|
||||||
|
def load_existing_annotations(self):
|
||||||
|
"""加载已有的标注数据"""
|
||||||
|
annotations_path = os.path.join(self.session_dir, "annotations.json")
|
||||||
|
if os.path.exists(annotations_path):
|
||||||
|
try:
|
||||||
|
with open(annotations_path, 'r', encoding='utf-8') as f:
|
||||||
|
annotations_data = json.load(f)
|
||||||
|
|
||||||
|
self.annotations = []
|
||||||
|
for ann_data in annotations_data:
|
||||||
|
annotation = Annotation(**ann_data)
|
||||||
|
self.annotations.append(annotation)
|
||||||
|
|
||||||
|
self.update_annotation_list()
|
||||||
|
self.update_status(f"加载了 {len(self.annotations)} 个已有标注")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"加载标注文件失败: {e}")
|
||||||
|
|
||||||
|
def save_annotations(self):
|
||||||
|
"""保存标注数据"""
|
||||||
|
if not self.session_dir:
|
||||||
|
messagebox.showwarning("警告", "请先加载数据")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
annotations_path = os.path.join(self.session_dir, "annotations.json")
|
||||||
|
annotations_data = [asdict(ann) for ann in self.annotations]
|
||||||
|
|
||||||
|
with open(annotations_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(annotations_data, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
self.update_status(f"已保存 {len(self.annotations)} 个标注到 {annotations_path}")
|
||||||
|
messagebox.showinfo("成功", "标注数据已保存")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
messagebox.showerror("错误", f"保存标注失败: {str(e)}")
|
||||||
|
|
||||||
|
def export_data(self):
|
||||||
|
"""导出训练数据"""
|
||||||
|
if not self.session_dir or not self.annotations:
|
||||||
|
messagebox.showwarning("警告", "请先加载数据并添加标注")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
export_path = os.path.join(self.session_dir, "training_data.json")
|
||||||
|
|
||||||
|
# 准备训练数据格式
|
||||||
|
training_data = []
|
||||||
|
timestamps = self.robot_data['timestamps'][:]
|
||||||
|
start_timestamp = timestamps[0]
|
||||||
|
|
||||||
|
for annotation in self.annotations:
|
||||||
|
# 找到对应时间范围的数据索引
|
||||||
|
start_idx = np.searchsorted(timestamps, start_timestamp + annotation.start_time)
|
||||||
|
end_idx = np.searchsorted(timestamps, start_timestamp + annotation.end_time)
|
||||||
|
|
||||||
|
# 提取对应的机器人状态数据
|
||||||
|
segment_data = {
|
||||||
|
"text": annotation.text,
|
||||||
|
"start_time": annotation.start_time,
|
||||||
|
"end_time": annotation.end_time,
|
||||||
|
"joint_positions": self.robot_data['joint_positions'][start_idx:end_idx].tolist(),
|
||||||
|
"joint_velocities": self.robot_data['joint_velocities'][start_idx:end_idx].tolist(),
|
||||||
|
"end_effector_poses": self.robot_data['end_effector_poses'][start_idx:end_idx].tolist(),
|
||||||
|
"gripper_states": self.robot_data['gripper_states'][start_idx:end_idx].tolist(),
|
||||||
|
"timestamps": (timestamps[start_idx:end_idx] - start_timestamp).tolist()
|
||||||
|
}
|
||||||
|
|
||||||
|
training_data.append(segment_data)
|
||||||
|
|
||||||
|
# 保存训练数据
|
||||||
|
with open(export_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(training_data, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
self.update_status(f"已导出 {len(training_data)} 个训练样本到 {export_path}")
|
||||||
|
messagebox.showinfo("成功", f"训练数据已导出到 {export_path}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
messagebox.showerror("错误", f"导出数据失败: {str(e)}")
|
||||||
|
|
||||||
|
def toggle_play(self):
|
||||||
|
"""切换播放/暂停状态"""
|
||||||
|
if not self.robot_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.playing = not self.playing
|
||||||
|
|
||||||
|
if self.playing:
|
||||||
|
self.play_thread = threading.Thread(target=self.play_data)
|
||||||
|
self.play_thread.start()
|
||||||
|
else:
|
||||||
|
if self.play_thread:
|
||||||
|
self.play_thread = None
|
||||||
|
|
||||||
|
def play_data(self):
|
||||||
|
"""播放数据"""
|
||||||
|
timestamps = self.robot_data['timestamps'][:]
|
||||||
|
start_time = timestamps[0]
|
||||||
|
|
||||||
|
while self.playing and self.current_time < self.total_time:
|
||||||
|
# 更新当前时间
|
||||||
|
self.current_time += 0.1 # 每次增加0.1秒
|
||||||
|
|
||||||
|
# 更新GUI
|
||||||
|
self.root.after(0, self.update_time_display)
|
||||||
|
self.root.after(0, self.update_visualization)
|
||||||
|
|
||||||
|
time.sleep(0.1) # 控制播放速度
|
||||||
|
|
||||||
|
self.playing = False
|
||||||
|
|
||||||
|
def reset_playback(self):
|
||||||
|
"""重置播放"""
|
||||||
|
self.playing = False
|
||||||
|
self.current_time = 0.0
|
||||||
|
self.update_time_display()
|
||||||
|
self.update_visualization()
|
||||||
|
|
||||||
|
def on_time_change(self, value):
|
||||||
|
"""时间滑块变化事件"""
|
||||||
|
if not self.playing: # 只在非播放状态下响应滑块变化
|
||||||
|
self.current_time = float(value)
|
||||||
|
self.update_time_display()
|
||||||
|
self.update_visualization()
|
||||||
|
|
||||||
|
def update_time_display(self):
|
||||||
|
"""更新时间显示"""
|
||||||
|
self.time_label.config(text=f"时间: {self.current_time:.2f} / {self.total_time:.2f} 秒")
|
||||||
|
self.time_scale.set(self.current_time)
|
||||||
|
|
||||||
|
def update_visualization(self):
|
||||||
|
"""更新数据可视化"""
|
||||||
|
if not self.robot_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.fig.clear()
|
||||||
|
|
||||||
|
# 获取当前时间对应的数据索引
|
||||||
|
timestamps = self.robot_data['timestamps'][:]
|
||||||
|
start_timestamp = timestamps[0]
|
||||||
|
current_timestamp = start_timestamp + self.current_time
|
||||||
|
current_idx = np.searchsorted(timestamps, current_timestamp)
|
||||||
|
|
||||||
|
# 绘制关节位置
|
||||||
|
ax1 = self.fig.add_subplot(2, 1, 1)
|
||||||
|
joint_positions = self.robot_data['joint_positions'][:]
|
||||||
|
|
||||||
|
# 显示一段时间窗口的数据
|
||||||
|
window_size = min(100, len(joint_positions))
|
||||||
|
start_idx = max(0, current_idx - window_size // 2)
|
||||||
|
end_idx = min(len(joint_positions), start_idx + window_size)
|
||||||
|
|
||||||
|
time_window = (timestamps[start_idx:end_idx] - start_timestamp)
|
||||||
|
|
||||||
|
for i in range(joint_positions.shape[1]):
|
||||||
|
ax1.plot(time_window, joint_positions[start_idx:end_idx, i], label=f'Joint {i+1}')
|
||||||
|
|
||||||
|
# 标记当前时间
|
||||||
|
ax1.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
|
||||||
|
ax1.set_xlabel('Time (s)')
|
||||||
|
ax1.set_ylabel('Joint Position (rad)')
|
||||||
|
ax1.set_title('Joint Positions')
|
||||||
|
ax1.legend()
|
||||||
|
ax1.grid(True)
|
||||||
|
|
||||||
|
# 绘制末端执行器位置
|
||||||
|
ax2 = self.fig.add_subplot(2, 1, 2)
|
||||||
|
end_effector_poses = self.robot_data['end_effector_poses'][:]
|
||||||
|
|
||||||
|
for i in range(3): # 只显示位置信息 (x, y, z)
|
||||||
|
ax2.plot(time_window, end_effector_poses[start_idx:end_idx, i], label=f'{"XYZ"[i]}')
|
||||||
|
|
||||||
|
ax2.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
|
||||||
|
ax2.set_xlabel('Time (s)')
|
||||||
|
ax2.set_ylabel('Position (m)')
|
||||||
|
ax2.set_title('End Effector Position')
|
||||||
|
ax2.legend()
|
||||||
|
ax2.grid(True)
|
||||||
|
|
||||||
|
# 绘制标注区间
|
||||||
|
for annotation in self.annotations:
|
||||||
|
ax1.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
|
||||||
|
ax2.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
|
||||||
|
|
||||||
|
self.fig.tight_layout()
|
||||||
|
self.canvas.draw()
|
||||||
|
|
||||||
|
def set_start_time(self):
|
||||||
|
"""设置标注开始时间"""
|
||||||
|
self.start_time_var.set(f"{self.current_time:.2f}")
|
||||||
|
|
||||||
|
def set_end_time(self):
|
||||||
|
"""设置标注结束时间"""
|
||||||
|
self.end_time_var.set(f"{self.current_time:.2f}")
|
||||||
|
|
||||||
|
def add_annotation(self):
|
||||||
|
"""添加标注"""
|
||||||
|
try:
|
||||||
|
start_time = float(self.start_time_var.get())
|
||||||
|
end_time = float(self.end_time_var.get())
|
||||||
|
text = self.text_entry.get("1.0", tk.END).strip()
|
||||||
|
|
||||||
|
if not text:
|
||||||
|
messagebox.showwarning("警告", "请输入语音指令文本")
|
||||||
|
return
|
||||||
|
|
||||||
|
if start_time >= end_time:
|
||||||
|
messagebox.showwarning("警告", "开始时间必须小于结束时间")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 创建新标注
|
||||||
|
annotation = Annotation(
|
||||||
|
start_time=start_time,
|
||||||
|
end_time=end_time,
|
||||||
|
text=text
|
||||||
|
)
|
||||||
|
|
||||||
|
self.annotations.append(annotation)
|
||||||
|
self.update_annotation_list()
|
||||||
|
self.update_visualization()
|
||||||
|
|
||||||
|
# 清空输入框
|
||||||
|
self.text_entry.delete("1.0", tk.END)
|
||||||
|
|
||||||
|
self.update_status(f"已添加标注: {text[:30]}...")
|
||||||
|
|
||||||
|
except ValueError:
|
||||||
|
messagebox.showerror("错误", "时间格式不正确")
|
||||||
|
except Exception as e:
|
||||||
|
messagebox.showerror("错误", f"添加标注失败: {str(e)}")
|
||||||
|
|
||||||
|
def delete_annotation(self):
|
||||||
|
"""删除选中的标注"""
|
||||||
|
selection = self.annotation_tree.selection()
|
||||||
|
if not selection:
|
||||||
|
messagebox.showwarning("警告", "请选择要删除的标注")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 获取选中项的索引
|
||||||
|
item = selection[0]
|
||||||
|
index = self.annotation_tree.index(item)
|
||||||
|
|
||||||
|
# 删除标注
|
||||||
|
deleted_annotation = self.annotations.pop(index)
|
||||||
|
self.update_annotation_list()
|
||||||
|
self.update_visualization()
|
||||||
|
|
||||||
|
self.update_status(f"已删除标注: {deleted_annotation.text[:30]}...")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
messagebox.showerror("错误", f"删除标注失败: {str(e)}")
|
||||||
|
|
||||||
|
def update_annotation_list(self):
|
||||||
|
"""更新标注列表显示"""
|
||||||
|
# 清空现有项目
|
||||||
|
for item in self.annotation_tree.get_children():
|
||||||
|
self.annotation_tree.delete(item)
|
||||||
|
|
||||||
|
# 添加标注项目
|
||||||
|
for annotation in self.annotations:
|
||||||
|
time_range = f"{annotation.start_time:.2f}-{annotation.end_time:.2f}s"
|
||||||
|
text_preview = annotation.text[:50] + "..." if len(annotation.text) > 50 else annotation.text
|
||||||
|
|
||||||
|
self.annotation_tree.insert('', tk.END, values=(time_range, text_preview))
|
||||||
|
|
||||||
|
def on_annotation_select(self, event):
|
||||||
|
"""标注选择事件"""
|
||||||
|
selection = self.annotation_tree.selection()
|
||||||
|
if selection:
|
||||||
|
item = selection[0]
|
||||||
|
index = self.annotation_tree.index(item)
|
||||||
|
annotation = self.annotations[index]
|
||||||
|
|
||||||
|
# 更新输入框
|
||||||
|
self.start_time_var.set(f"{annotation.start_time:.2f}")
|
||||||
|
self.end_time_var.set(f"{annotation.end_time:.2f}")
|
||||||
|
self.text_entry.delete("1.0", tk.END)
|
||||||
|
self.text_entry.insert("1.0", annotation.text)
|
||||||
|
|
||||||
|
def on_key_press(self, event):
|
||||||
|
"""键盘事件处理"""
|
||||||
|
if event.keysym == 'space':
|
||||||
|
self.toggle_play()
|
||||||
|
elif event.keysym == 'Left':
|
||||||
|
self.current_time = max(0, self.current_time - 1.0)
|
||||||
|
self.update_time_display()
|
||||||
|
self.update_visualization()
|
||||||
|
elif event.keysym == 'Right':
|
||||||
|
self.current_time = min(self.total_time, self.current_time + 1.0)
|
||||||
|
self.update_time_display()
|
||||||
|
self.update_visualization()
|
||||||
|
|
||||||
|
def update_status(self, message):
|
||||||
|
"""更新状态栏"""
|
||||||
|
self.status_label.config(text=message)
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""运行GUI"""
|
||||||
|
self.root.mainloop()
|
||||||
|
|
||||||
|
# 清理资源
|
||||||
|
if self.robot_data:
|
||||||
|
self.robot_data.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""主函数"""
|
||||||
|
app = AnnotationTool()
|
||||||
|
app.run()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
280
data_collector.py
普通文件
280
data_collector.py
普通文件
@ -0,0 +1,280 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
数据采集模块 - 用于录制机器人动作和传感器数据
|
||||||
|
基于LeRobot框架的语音输入法训练系统
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Dict, List, Optional, Any
|
||||||
|
import numpy as np
|
||||||
|
import h5py
|
||||||
|
import pygame
|
||||||
|
from dataclasses import dataclass, asdict
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RobotState:
|
||||||
|
"""机器人状态数据结构"""
|
||||||
|
timestamp: float
|
||||||
|
joint_positions: List[float]
|
||||||
|
joint_velocities: List[float]
|
||||||
|
joint_torques: List[float]
|
||||||
|
end_effector_pose: List[float] # [x, y, z, roll, pitch, yaw]
|
||||||
|
gripper_state: float
|
||||||
|
camera_image: Optional[np.ndarray] = None
|
||||||
|
depth_image: Optional[np.ndarray] = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ControlCommand:
|
||||||
|
"""控制指令数据结构"""
|
||||||
|
timestamp: float
|
||||||
|
command_type: str # 'joystick', 'button', 'voice'
|
||||||
|
command_data: Dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class DataCollector:
|
||||||
|
"""数据采集器类"""
|
||||||
|
|
||||||
|
def __init__(self, output_dir: str = "./collected_data"):
|
||||||
|
self.output_dir = output_dir
|
||||||
|
self.is_recording = False
|
||||||
|
self.recording_thread = None
|
||||||
|
self.data_buffer = []
|
||||||
|
self.command_buffer = []
|
||||||
|
|
||||||
|
# 创建输出目录
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 初始化pygame用于手柄控制
|
||||||
|
pygame.init()
|
||||||
|
pygame.joystick.init()
|
||||||
|
|
||||||
|
# 检查手柄连接
|
||||||
|
self.joystick = None
|
||||||
|
if pygame.joystick.get_count() > 0:
|
||||||
|
self.joystick = pygame.joystick.Joystick(0)
|
||||||
|
self.joystick.init()
|
||||||
|
print(f"检测到手柄: {self.joystick.get_name()}")
|
||||||
|
else:
|
||||||
|
print("警告: 未检测到手柄,将使用键盘模拟")
|
||||||
|
|
||||||
|
def start_recording(self, session_name: str = None) -> str:
|
||||||
|
"""开始录制数据"""
|
||||||
|
if self.is_recording:
|
||||||
|
print("已经在录制中...")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if session_name is None:
|
||||||
|
session_name = f"session_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||||
|
|
||||||
|
self.session_name = session_name
|
||||||
|
self.session_dir = os.path.join(self.output_dir, session_name)
|
||||||
|
os.makedirs(self.session_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 清空缓冲区
|
||||||
|
self.data_buffer.clear()
|
||||||
|
self.command_buffer.clear()
|
||||||
|
|
||||||
|
self.is_recording = True
|
||||||
|
self.recording_thread = threading.Thread(target=self._recording_loop)
|
||||||
|
self.recording_thread.start()
|
||||||
|
|
||||||
|
print(f"开始录制会话: {session_name}")
|
||||||
|
return session_name
|
||||||
|
|
||||||
|
def stop_recording(self):
|
||||||
|
"""停止录制并保存数据"""
|
||||||
|
if not self.is_recording:
|
||||||
|
print("当前没有在录制...")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.is_recording = False
|
||||||
|
if self.recording_thread:
|
||||||
|
self.recording_thread.join()
|
||||||
|
|
||||||
|
# 保存数据
|
||||||
|
self._save_data()
|
||||||
|
print(f"录制完成,数据已保存到: {self.session_dir}")
|
||||||
|
|
||||||
|
def _recording_loop(self):
|
||||||
|
"""录制循环"""
|
||||||
|
print("录制循环开始...")
|
||||||
|
print("使用手柄控制机器人,按 'q' 键停止录制")
|
||||||
|
|
||||||
|
while self.is_recording:
|
||||||
|
try:
|
||||||
|
# 处理pygame事件
|
||||||
|
pygame.event.pump()
|
||||||
|
|
||||||
|
# 获取当前时间戳
|
||||||
|
timestamp = time.time()
|
||||||
|
|
||||||
|
# 模拟获取机器人状态数据
|
||||||
|
robot_state = self._get_robot_state(timestamp)
|
||||||
|
self.data_buffer.append(robot_state)
|
||||||
|
|
||||||
|
# 获取控制指令
|
||||||
|
control_command = self._get_control_command(timestamp)
|
||||||
|
if control_command:
|
||||||
|
self.command_buffer.append(control_command)
|
||||||
|
|
||||||
|
# 控制录制频率 (50Hz)
|
||||||
|
time.sleep(0.02)
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("接收到中断信号,停止录制...")
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
print(f"录制过程中出现错误: {e}")
|
||||||
|
break
|
||||||
|
|
||||||
|
def _get_robot_state(self, timestamp: float) -> RobotState:
|
||||||
|
"""获取机器人状态(模拟数据)"""
|
||||||
|
# 这里应该连接到真实的机器人API获取数据
|
||||||
|
# 目前使用模拟数据
|
||||||
|
return RobotState(
|
||||||
|
timestamp=timestamp,
|
||||||
|
joint_positions=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
|
||||||
|
joint_velocities=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06],
|
||||||
|
joint_torques=[1.0, 1.1, 1.2, 1.3, 1.4, 1.5],
|
||||||
|
end_effector_pose=[0.5, 0.0, 0.3, 0.0, 0.0, 0.0],
|
||||||
|
gripper_state=0.5,
|
||||||
|
camera_image=np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8),
|
||||||
|
depth_image=np.random.rand(480, 640).astype(np.float32)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_control_command(self, timestamp: float) -> Optional[ControlCommand]:
|
||||||
|
"""获取控制指令"""
|
||||||
|
if self.joystick:
|
||||||
|
# 读取手柄输入
|
||||||
|
axes = [self.joystick.get_axis(i) for i in range(self.joystick.get_numaxes())]
|
||||||
|
buttons = [self.joystick.get_button(i) for i in range(self.joystick.get_numbuttons())]
|
||||||
|
|
||||||
|
# 检查是否有显著的输入变化
|
||||||
|
if any(abs(axis) > 0.1 for axis in axes) or any(buttons):
|
||||||
|
return ControlCommand(
|
||||||
|
timestamp=timestamp,
|
||||||
|
command_type="joystick",
|
||||||
|
command_data={
|
||||||
|
"axes": axes,
|
||||||
|
"buttons": buttons
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _save_data(self):
|
||||||
|
"""保存录制的数据"""
|
||||||
|
# 保存机器人状态数据到HDF5文件
|
||||||
|
hdf5_path = os.path.join(self.session_dir, "robot_data.h5")
|
||||||
|
with h5py.File(hdf5_path, 'w') as f:
|
||||||
|
# 创建数据集
|
||||||
|
num_samples = len(self.data_buffer)
|
||||||
|
|
||||||
|
# 时间戳
|
||||||
|
f.create_dataset('timestamps', data=[state.timestamp for state in self.data_buffer])
|
||||||
|
|
||||||
|
# 关节数据
|
||||||
|
joint_positions = np.array([state.joint_positions for state in self.data_buffer])
|
||||||
|
joint_velocities = np.array([state.joint_velocities for state in self.data_buffer])
|
||||||
|
joint_torques = np.array([state.joint_torques for state in self.data_buffer])
|
||||||
|
|
||||||
|
f.create_dataset('joint_positions', data=joint_positions)
|
||||||
|
f.create_dataset('joint_velocities', data=joint_velocities)
|
||||||
|
f.create_dataset('joint_torques', data=joint_torques)
|
||||||
|
|
||||||
|
# 末端执行器位姿
|
||||||
|
end_effector_poses = np.array([state.end_effector_pose for state in self.data_buffer])
|
||||||
|
f.create_dataset('end_effector_poses', data=end_effector_poses)
|
||||||
|
|
||||||
|
# 夹爪状态
|
||||||
|
gripper_states = np.array([state.gripper_state for state in self.data_buffer])
|
||||||
|
f.create_dataset('gripper_states', data=gripper_states)
|
||||||
|
|
||||||
|
# 图像数据
|
||||||
|
if self.data_buffer and self.data_buffer[0].camera_image is not None:
|
||||||
|
camera_images = np.array([state.camera_image for state in self.data_buffer])
|
||||||
|
f.create_dataset('camera_images', data=camera_images, compression='gzip')
|
||||||
|
|
||||||
|
if self.data_buffer and self.data_buffer[0].depth_image is not None:
|
||||||
|
depth_images = np.array([state.depth_image for state in self.data_buffer])
|
||||||
|
f.create_dataset('depth_images', data=depth_images, compression='gzip')
|
||||||
|
|
||||||
|
# 保存控制指令到JSON文件
|
||||||
|
commands_path = os.path.join(self.session_dir, "commands.json")
|
||||||
|
with open(commands_path, 'w') as f:
|
||||||
|
commands_data = [asdict(cmd) for cmd in self.command_buffer]
|
||||||
|
json.dump(commands_data, f, indent=2)
|
||||||
|
|
||||||
|
# 保存会话元数据
|
||||||
|
metadata = {
|
||||||
|
"session_name": self.session_name,
|
||||||
|
"start_time": self.data_buffer[0].timestamp if self.data_buffer else None,
|
||||||
|
"end_time": self.data_buffer[-1].timestamp if self.data_buffer else None,
|
||||||
|
"duration": (self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 0,
|
||||||
|
"num_samples": len(self.data_buffer),
|
||||||
|
"num_commands": len(self.command_buffer),
|
||||||
|
"recording_frequency": len(self.data_buffer) / ((self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata_path = os.path.join(self.session_dir, "metadata.json")
|
||||||
|
with open(metadata_path, 'w') as f:
|
||||||
|
json.dump(metadata, f, indent=2)
|
||||||
|
|
||||||
|
print(f"数据已保存:")
|
||||||
|
print(f" - 机器人数据: {hdf5_path}")
|
||||||
|
print(f" - 控制指令: {commands_path}")
|
||||||
|
print(f" - 元数据: {metadata_path}")
|
||||||
|
print(f" - 总样本数: {len(self.data_buffer)}")
|
||||||
|
print(f" - 总指令数: {len(self.command_buffer)}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""主函数 - 演示数据采集器的使用"""
|
||||||
|
collector = DataCollector()
|
||||||
|
|
||||||
|
print("=== 语音机器人训练系统 - 数据采集器 ===")
|
||||||
|
print("指令:")
|
||||||
|
print(" start <session_name> - 开始录制")
|
||||||
|
print(" stop - 停止录制")
|
||||||
|
print(" quit - 退出程序")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
command = input("\n请输入指令: ").strip().split()
|
||||||
|
|
||||||
|
if not command:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if command[0] == "start":
|
||||||
|
session_name = command[1] if len(command) > 1 else None
|
||||||
|
collector.start_recording(session_name)
|
||||||
|
|
||||||
|
elif command[0] == "stop":
|
||||||
|
collector.stop_recording()
|
||||||
|
|
||||||
|
elif command[0] == "quit":
|
||||||
|
if collector.is_recording:
|
||||||
|
collector.stop_recording()
|
||||||
|
break
|
||||||
|
|
||||||
|
else:
|
||||||
|
print("未知指令,请重新输入")
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\n程序被中断")
|
||||||
|
if collector.is_recording:
|
||||||
|
collector.stop_recording()
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
print(f"发生错误: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
378
example_usage.py
普通文件
378
example_usage.py
普通文件
@ -0,0 +1,378 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
使用示例 - 演示语音机器人训练系统的完整使用流程
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
import numpy as np
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# 添加项目根目录到路径
|
||||||
|
sys.path.append(str(Path(__file__).parent.parent))
|
||||||
|
|
||||||
|
from data_collector import DataCollector, RobotState, ControlCommand
|
||||||
|
from model_trainer import VoiceRobotTrainer, TrainingConfig
|
||||||
|
from inference_engine import VoiceRobotSystem
|
||||||
|
|
||||||
|
|
||||||
|
def example_data_collection():
|
||||||
|
"""示例:数据采集流程"""
|
||||||
|
print("=== 数据采集示例 ===")
|
||||||
|
|
||||||
|
# 创建数据采集器
|
||||||
|
collector = DataCollector(output_dir="./example_data")
|
||||||
|
|
||||||
|
# 模拟数据采集过程
|
||||||
|
print("开始模拟数据采集...")
|
||||||
|
session_name = collector.start_recording("example_session")
|
||||||
|
|
||||||
|
# 模拟录制5秒钟的数据
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
collector.stop_recording()
|
||||||
|
print(f"数据采集完成,会话: {session_name}")
|
||||||
|
|
||||||
|
return session_name
|
||||||
|
|
||||||
|
|
||||||
|
def example_manual_annotation(session_name):
|
||||||
|
"""示例:手动创建标注数据"""
|
||||||
|
print("=== 手动标注示例 ===")
|
||||||
|
|
||||||
|
session_dir = f"./example_data/{session_name}"
|
||||||
|
|
||||||
|
# 创建示例标注数据
|
||||||
|
annotations = [
|
||||||
|
{
|
||||||
|
"start_time": 0.0,
|
||||||
|
"end_time": 2.0,
|
||||||
|
"text": "向前移动",
|
||||||
|
"confidence": 1.0,
|
||||||
|
"annotator": "example",
|
||||||
|
"timestamp": time.time()
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"start_time": 2.0,
|
||||||
|
"end_time": 4.0,
|
||||||
|
"text": "抓取物体",
|
||||||
|
"confidence": 1.0,
|
||||||
|
"annotator": "example",
|
||||||
|
"timestamp": time.time()
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
# 保存标注数据
|
||||||
|
annotations_path = os.path.join(session_dir, "annotations.json")
|
||||||
|
with open(annotations_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(annotations, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
print(f"标注数据已保存到: {annotations_path}")
|
||||||
|
|
||||||
|
# 创建训练数据
|
||||||
|
create_example_training_data(session_dir, annotations)
|
||||||
|
|
||||||
|
|
||||||
|
def create_example_training_data(session_dir, annotations):
|
||||||
|
"""创建示例训练数据"""
|
||||||
|
print("创建训练数据...")
|
||||||
|
|
||||||
|
training_data = []
|
||||||
|
|
||||||
|
for annotation in annotations:
|
||||||
|
# 模拟动作序列数据
|
||||||
|
duration = annotation["end_time"] - annotation["start_time"]
|
||||||
|
num_steps = int(duration * 50) # 50Hz采样率
|
||||||
|
|
||||||
|
# 生成模拟的机器人动作数据
|
||||||
|
joint_positions = []
|
||||||
|
end_effector_poses = []
|
||||||
|
gripper_states = []
|
||||||
|
timestamps = []
|
||||||
|
|
||||||
|
for i in range(num_steps):
|
||||||
|
t = i / 50.0
|
||||||
|
|
||||||
|
# 模拟关节位置(正弦波变化)
|
||||||
|
joint_pos = [0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
|
||||||
|
joint_positions.append(joint_pos)
|
||||||
|
|
||||||
|
# 模拟末端执行器位姿
|
||||||
|
pose = [0.5 + 0.1 * np.sin(2 * np.pi * t),
|
||||||
|
0.0 + 0.1 * np.cos(2 * np.pi * t),
|
||||||
|
0.3, 0.0, 0.0, 0.0]
|
||||||
|
end_effector_poses.append(pose)
|
||||||
|
|
||||||
|
# 模拟夹爪状态
|
||||||
|
if "抓取" in annotation["text"]:
|
||||||
|
gripper_state = min(1.0, t / duration) # 逐渐闭合
|
||||||
|
else:
|
||||||
|
gripper_state = 0.5 # 保持中间位置
|
||||||
|
gripper_states.append(gripper_state)
|
||||||
|
|
||||||
|
timestamps.append(t)
|
||||||
|
|
||||||
|
# 创建训练样本
|
||||||
|
sample = {
|
||||||
|
"text": annotation["text"],
|
||||||
|
"start_time": annotation["start_time"],
|
||||||
|
"end_time": annotation["end_time"],
|
||||||
|
"joint_positions": joint_positions,
|
||||||
|
"joint_velocities": [[0.0] * 6] * num_steps, # 简化为零速度
|
||||||
|
"end_effector_poses": end_effector_poses,
|
||||||
|
"gripper_states": gripper_states,
|
||||||
|
"timestamps": timestamps
|
||||||
|
}
|
||||||
|
|
||||||
|
training_data.append(sample)
|
||||||
|
|
||||||
|
# 保存训练数据
|
||||||
|
training_data_path = os.path.join(session_dir, "training_data.json")
|
||||||
|
with open(training_data_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(training_data, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
print(f"训练数据已保存到: {training_data_path}")
|
||||||
|
|
||||||
|
|
||||||
|
def example_model_training():
|
||||||
|
"""示例:模型训练流程"""
|
||||||
|
print("=== 模型训练示例 ===")
|
||||||
|
|
||||||
|
# 创建训练配置
|
||||||
|
config = TrainingConfig(
|
||||||
|
data_dir="./example_data",
|
||||||
|
output_dir="./example_models",
|
||||||
|
batch_size=2, # 小批次用于示例
|
||||||
|
num_epochs=10, # 少量epoch用于示例
|
||||||
|
learning_rate=1e-3,
|
||||||
|
early_stopping_patience=5
|
||||||
|
)
|
||||||
|
|
||||||
|
# 创建训练器
|
||||||
|
trainer = VoiceRobotTrainer(config)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 开始训练
|
||||||
|
trainer.train()
|
||||||
|
print("模型训练完成!")
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
print(f"训练过程中出现错误: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def example_inference():
|
||||||
|
"""示例:模型推理流程"""
|
||||||
|
print("=== 模型推理示例 ===")
|
||||||
|
|
||||||
|
model_path = "./example_models/best_model.pth"
|
||||||
|
|
||||||
|
if not os.path.exists(model_path):
|
||||||
|
print("模型文件不存在,请先完成训练")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 创建推理系统
|
||||||
|
system = VoiceRobotSystem(model_path)
|
||||||
|
|
||||||
|
# 测试文本指令
|
||||||
|
test_commands = [
|
||||||
|
"向前移动",
|
||||||
|
"抓取物体",
|
||||||
|
"向左转动",
|
||||||
|
"放下物体"
|
||||||
|
]
|
||||||
|
|
||||||
|
print("测试文本指令:")
|
||||||
|
for command in test_commands:
|
||||||
|
print(f"\n指令: {command}")
|
||||||
|
try:
|
||||||
|
system.process_text_command(command)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"处理指令失败: {e}")
|
||||||
|
|
||||||
|
print("推理测试完成!")
|
||||||
|
|
||||||
|
|
||||||
|
def example_complete_workflow():
|
||||||
|
"""示例:完整的工作流程"""
|
||||||
|
print("=== 完整工作流程示例 ===")
|
||||||
|
print("这个示例将演示从数据采集到模型部署的完整流程")
|
||||||
|
|
||||||
|
# 步骤1:数据采集
|
||||||
|
print("\n步骤1: 数据采集")
|
||||||
|
session_name = example_data_collection()
|
||||||
|
|
||||||
|
# 步骤2:数据标注
|
||||||
|
print("\n步骤2: 数据标注")
|
||||||
|
example_manual_annotation(session_name)
|
||||||
|
|
||||||
|
# 步骤3:模型训练
|
||||||
|
print("\n步骤3: 模型训练")
|
||||||
|
training_success = example_model_training()
|
||||||
|
|
||||||
|
if training_success:
|
||||||
|
# 步骤4:模型推理
|
||||||
|
print("\n步骤4: 模型推理")
|
||||||
|
example_inference()
|
||||||
|
|
||||||
|
print("\n=== 完整工作流程示例完成 ===")
|
||||||
|
|
||||||
|
|
||||||
|
def example_custom_robot_integration():
|
||||||
|
"""示例:自定义机器人集成"""
|
||||||
|
print("=== 自定义机器人集成示例 ===")
|
||||||
|
|
||||||
|
class CustomRobotController:
|
||||||
|
"""自定义机器人控制器示例"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
print("初始化自定义机器人控制器")
|
||||||
|
# 这里可以初始化机器人SDK、ROS节点等
|
||||||
|
|
||||||
|
def execute_action(self, prediction):
|
||||||
|
"""执行机器人动作"""
|
||||||
|
joint_positions = prediction['joint_positions']
|
||||||
|
end_effector_poses = prediction['end_effector_poses']
|
||||||
|
gripper_states = prediction['gripper_states']
|
||||||
|
text = prediction['text']
|
||||||
|
|
||||||
|
print(f"执行自定义机器人动作: {text}")
|
||||||
|
|
||||||
|
# 这里实现具体的机器人控制逻辑
|
||||||
|
# 例如:
|
||||||
|
# self.robot_arm.move_joints(joint_positions)
|
||||||
|
# self.robot_arm.move_to_pose(end_effector_poses)
|
||||||
|
# self.gripper.set_position(gripper_states[0])
|
||||||
|
|
||||||
|
print(f"关节目标位置: {joint_positions}")
|
||||||
|
print(f"末端执行器目标位姿: {end_effector_poses}")
|
||||||
|
print(f"夹爪目标状态: {gripper_states}")
|
||||||
|
|
||||||
|
# 使用自定义控制器
|
||||||
|
custom_controller = CustomRobotController()
|
||||||
|
|
||||||
|
# 模拟预测结果
|
||||||
|
mock_prediction = {
|
||||||
|
'joint_positions': np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6]),
|
||||||
|
'end_effector_poses': np.array([0.5, 0.0, 0.3, 0.0, 0.0, 0.0]),
|
||||||
|
'gripper_states': np.array([0.8]),
|
||||||
|
'text': '抓取红色方块'
|
||||||
|
}
|
||||||
|
|
||||||
|
custom_controller.execute_action(mock_prediction)
|
||||||
|
|
||||||
|
|
||||||
|
def example_data_analysis():
|
||||||
|
"""示例:数据分析和可视化"""
|
||||||
|
print("=== 数据分析示例 ===")
|
||||||
|
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
# 设置中文字体
|
||||||
|
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
|
||||||
|
plt.rcParams['axes.unicode_minus'] = False
|
||||||
|
|
||||||
|
# 生成示例数据
|
||||||
|
time_steps = np.linspace(0, 5, 250) # 5秒,50Hz
|
||||||
|
joint_positions = np.array([
|
||||||
|
[0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
|
||||||
|
for t in time_steps
|
||||||
|
])
|
||||||
|
|
||||||
|
# 创建可视化
|
||||||
|
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||||
|
|
||||||
|
# 关节位置
|
||||||
|
axes[0, 0].plot(time_steps, joint_positions)
|
||||||
|
axes[0, 0].set_title('关节位置轨迹')
|
||||||
|
axes[0, 0].set_xlabel('时间 (秒)')
|
||||||
|
axes[0, 0].set_ylabel('位置 (弧度)')
|
||||||
|
axes[0, 0].grid(True)
|
||||||
|
axes[0, 0].legend([f'关节{i+1}' for i in range(6)])
|
||||||
|
|
||||||
|
# 末端执行器轨迹
|
||||||
|
end_effector_x = 0.5 + 0.1 * np.sin(2 * np.pi * time_steps)
|
||||||
|
end_effector_y = 0.0 + 0.1 * np.cos(2 * np.pi * time_steps)
|
||||||
|
|
||||||
|
axes[0, 1].plot(end_effector_x, end_effector_y)
|
||||||
|
axes[0, 1].set_title('末端执行器轨迹 (XY平面)')
|
||||||
|
axes[0, 1].set_xlabel('X位置 (米)')
|
||||||
|
axes[0, 1].set_ylabel('Y位置 (米)')
|
||||||
|
axes[0, 1].grid(True)
|
||||||
|
axes[0, 1].axis('equal')
|
||||||
|
|
||||||
|
# 夹爪状态
|
||||||
|
gripper_states = np.minimum(1.0, time_steps / 5.0)
|
||||||
|
axes[1, 0].plot(time_steps, gripper_states)
|
||||||
|
axes[1, 0].set_title('夹爪状态')
|
||||||
|
axes[1, 0].set_xlabel('时间 (秒)')
|
||||||
|
axes[1, 0].set_ylabel('开合程度')
|
||||||
|
axes[1, 0].grid(True)
|
||||||
|
|
||||||
|
# 速度分析
|
||||||
|
joint_velocities = np.gradient(joint_positions, axis=0) / (time_steps[1] - time_steps[0])
|
||||||
|
axes[1, 1].plot(time_steps, np.linalg.norm(joint_velocities, axis=1))
|
||||||
|
axes[1, 1].set_title('关节速度幅值')
|
||||||
|
axes[1, 1].set_xlabel('时间 (秒)')
|
||||||
|
axes[1, 1].set_ylabel('速度 (弧度/秒)')
|
||||||
|
axes[1, 1].grid(True)
|
||||||
|
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
# 保存图片
|
||||||
|
os.makedirs('./example_output', exist_ok=True)
|
||||||
|
plt.savefig('./example_output/data_analysis.png', dpi=300, bbox_inches='tight')
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
print("数据分析图表已保存到: ./example_output/data_analysis.png")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""主函数"""
|
||||||
|
print("语音机器人训练系统 - 使用示例")
|
||||||
|
print("=" * 50)
|
||||||
|
|
||||||
|
examples = {
|
||||||
|
"1": ("完整工作流程", example_complete_workflow),
|
||||||
|
"2": ("数据采集", example_data_collection),
|
||||||
|
"3": ("模型训练", example_model_training),
|
||||||
|
"4": ("模型推理", example_inference),
|
||||||
|
"5": ("自定义机器人集成", example_custom_robot_integration),
|
||||||
|
"6": ("数据分析", example_data_analysis),
|
||||||
|
"0": ("运行所有示例", None)
|
||||||
|
}
|
||||||
|
|
||||||
|
print("可用示例:")
|
||||||
|
for key, (name, _) in examples.items():
|
||||||
|
print(f" {key}. {name}")
|
||||||
|
|
||||||
|
choice = input("\n请选择要运行的示例 (输入数字): ").strip()
|
||||||
|
|
||||||
|
if choice == "0":
|
||||||
|
# 运行所有示例
|
||||||
|
for key, (name, func) in examples.items():
|
||||||
|
if func is not None:
|
||||||
|
print(f"\n{'='*20} {name} {'='*20}")
|
||||||
|
try:
|
||||||
|
func()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"示例 '{name}' 运行失败: {e}")
|
||||||
|
elif choice in examples:
|
||||||
|
name, func = examples[choice]
|
||||||
|
if func is not None:
|
||||||
|
print(f"\n运行示例: {name}")
|
||||||
|
try:
|
||||||
|
func()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"示例运行失败: {e}")
|
||||||
|
else:
|
||||||
|
print("无效选择")
|
||||||
|
else:
|
||||||
|
print("无效选择")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
441
inference_engine.py
普通文件
441
inference_engine.py
普通文件
@ -0,0 +1,441 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
推理引擎 - 用于部署训练好的语音-动作映射模型
|
||||||
|
基于LeRobot框架的语音输入法训练系统
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import torch
|
||||||
|
import numpy as np
|
||||||
|
import pickle
|
||||||
|
import speech_recognition as sr
|
||||||
|
import pyaudio
|
||||||
|
from typing import Dict, List, Optional, Any
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
from queue import Queue
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
from model_trainer import VoiceRobotModel, TrainingConfig
|
||||||
|
|
||||||
|
|
||||||
|
# 设置日志
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class VoiceRobotInference:
|
||||||
|
"""语音机器人推理引擎"""
|
||||||
|
|
||||||
|
def __init__(self, model_path: str, device: str = "auto"):
|
||||||
|
"""
|
||||||
|
初始化推理引擎
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model_path: 训练好的模型路径
|
||||||
|
device: 计算设备 ("auto", "cpu", "cuda")
|
||||||
|
"""
|
||||||
|
self.model_path = model_path
|
||||||
|
|
||||||
|
# 设置设备
|
||||||
|
if device == "auto":
|
||||||
|
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||||
|
else:
|
||||||
|
self.device = torch.device(device)
|
||||||
|
|
||||||
|
logger.info(f"使用设备: {self.device}")
|
||||||
|
|
||||||
|
# 加载模型和配置
|
||||||
|
self._load_model()
|
||||||
|
|
||||||
|
# 初始化语音识别
|
||||||
|
self._init_speech_recognition()
|
||||||
|
|
||||||
|
# 语音处理队列
|
||||||
|
self.audio_queue = Queue()
|
||||||
|
self.result_queue = Queue()
|
||||||
|
self.is_listening = False
|
||||||
|
self.listen_thread = None
|
||||||
|
|
||||||
|
logger.info("推理引擎初始化完成")
|
||||||
|
|
||||||
|
def _load_model(self):
|
||||||
|
"""加载训练好的模型"""
|
||||||
|
logger.info(f"加载模型: {self.model_path}")
|
||||||
|
|
||||||
|
# 加载模型检查点
|
||||||
|
checkpoint = torch.load(self.model_path, map_location=self.device)
|
||||||
|
self.config = checkpoint['config']
|
||||||
|
|
||||||
|
# 创建模型
|
||||||
|
self.model = VoiceRobotModel(self.config)
|
||||||
|
self.model.load_state_dict(checkpoint['model_state_dict'])
|
||||||
|
self.model.to(self.device)
|
||||||
|
self.model.eval()
|
||||||
|
|
||||||
|
# 加载tokenizer
|
||||||
|
self.tokenizer = AutoTokenizer.from_pretrained(self.config.text_encoder_model)
|
||||||
|
|
||||||
|
# 加载数据标准化器
|
||||||
|
scaler_path = os.path.join(os.path.dirname(self.model_path), "scalers.pkl")
|
||||||
|
if os.path.exists(scaler_path):
|
||||||
|
with open(scaler_path, 'rb') as f:
|
||||||
|
scalers = pickle.load(f)
|
||||||
|
self.action_scaler = scalers['action_scaler']
|
||||||
|
self.pose_scaler = scalers['pose_scaler']
|
||||||
|
self.gripper_scaler = scalers['gripper_scaler']
|
||||||
|
else:
|
||||||
|
logger.warning("未找到数据标准化器,预测结果可能不准确")
|
||||||
|
self.action_scaler = None
|
||||||
|
self.pose_scaler = None
|
||||||
|
self.gripper_scaler = None
|
||||||
|
|
||||||
|
logger.info("模型加载完成")
|
||||||
|
|
||||||
|
def _init_speech_recognition(self):
|
||||||
|
"""初始化语音识别"""
|
||||||
|
self.recognizer = sr.Recognizer()
|
||||||
|
self.microphone = sr.Microphone()
|
||||||
|
|
||||||
|
# 调整环境噪声
|
||||||
|
logger.info("调整环境噪声...")
|
||||||
|
with self.microphone as source:
|
||||||
|
self.recognizer.adjust_for_ambient_noise(source)
|
||||||
|
|
||||||
|
logger.info("语音识别初始化完成")
|
||||||
|
|
||||||
|
def predict_from_text(self, text: str) -> Dict[str, np.ndarray]:
|
||||||
|
"""
|
||||||
|
从文本预测机器人动作
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: 语音指令文本
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
包含预测动作的字典
|
||||||
|
"""
|
||||||
|
logger.info(f"预测文本: {text}")
|
||||||
|
|
||||||
|
# 文本编码
|
||||||
|
encoding = self.tokenizer(
|
||||||
|
text,
|
||||||
|
truncation=True,
|
||||||
|
padding='max_length',
|
||||||
|
max_length=128,
|
||||||
|
return_tensors='pt'
|
||||||
|
)
|
||||||
|
|
||||||
|
# 移动到设备
|
||||||
|
input_ids = encoding['input_ids'].to(self.device)
|
||||||
|
attention_mask = encoding['attention_mask'].to(self.device)
|
||||||
|
|
||||||
|
# 模型推理
|
||||||
|
with torch.no_grad():
|
||||||
|
predictions = self.model(input_ids, attention_mask)
|
||||||
|
|
||||||
|
# 转换为numpy数组
|
||||||
|
joint_positions = predictions['joint_positions'].cpu().numpy()[0]
|
||||||
|
end_effector_poses = predictions['end_effector_poses'].cpu().numpy()[0]
|
||||||
|
gripper_states = predictions['gripper_states'].cpu().numpy()[0]
|
||||||
|
|
||||||
|
# 反标准化
|
||||||
|
if self.action_scaler is not None:
|
||||||
|
joint_positions = self.action_scaler.inverse_transform(joint_positions.reshape(1, -1))[0]
|
||||||
|
|
||||||
|
if self.pose_scaler is not None:
|
||||||
|
end_effector_poses = self.pose_scaler.inverse_transform(end_effector_poses.reshape(1, -1))[0]
|
||||||
|
|
||||||
|
if self.gripper_scaler is not None:
|
||||||
|
gripper_states = self.gripper_scaler.inverse_transform(gripper_states.reshape(1, -1))[0]
|
||||||
|
|
||||||
|
result = {
|
||||||
|
'joint_positions': joint_positions,
|
||||||
|
'end_effector_poses': end_effector_poses,
|
||||||
|
'gripper_states': gripper_states,
|
||||||
|
'text': text
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(f"预测完成: 关节位置={joint_positions}, 末端位姿={end_effector_poses}, 夹爪状态={gripper_states}")
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def recognize_speech(self, audio_data) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
识别语音
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio_data: 音频数据
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
识别的文本,如果识别失败返回None
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# 使用Google语音识别(需要网络连接)
|
||||||
|
text = self.recognizer.recognize_google(audio_data, language='zh-CN')
|
||||||
|
logger.info(f"语音识别结果: {text}")
|
||||||
|
return text
|
||||||
|
except sr.UnknownValueError:
|
||||||
|
logger.warning("无法识别语音")
|
||||||
|
return None
|
||||||
|
except sr.RequestError as e:
|
||||||
|
logger.error(f"语音识别服务错误: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def start_listening(self):
|
||||||
|
"""开始监听语音"""
|
||||||
|
if self.is_listening:
|
||||||
|
logger.warning("已经在监听中...")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.is_listening = True
|
||||||
|
self.listen_thread = threading.Thread(target=self._listen_loop)
|
||||||
|
self.listen_thread.start()
|
||||||
|
|
||||||
|
logger.info("开始监听语音...")
|
||||||
|
|
||||||
|
def stop_listening(self):
|
||||||
|
"""停止监听语音"""
|
||||||
|
if not self.is_listening:
|
||||||
|
logger.warning("当前没有在监听...")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.is_listening = False
|
||||||
|
if self.listen_thread:
|
||||||
|
self.listen_thread.join()
|
||||||
|
|
||||||
|
logger.info("停止监听语音")
|
||||||
|
|
||||||
|
def _listen_loop(self):
|
||||||
|
"""语音监听循环"""
|
||||||
|
while self.is_listening:
|
||||||
|
try:
|
||||||
|
with self.microphone as source:
|
||||||
|
logger.info("请说话...")
|
||||||
|
# 监听音频,超时时间为1秒
|
||||||
|
audio = self.recognizer.listen(source, timeout=1, phrase_time_limit=5)
|
||||||
|
|
||||||
|
# 识别语音
|
||||||
|
text = self.recognize_speech(audio)
|
||||||
|
|
||||||
|
if text:
|
||||||
|
# 预测动作
|
||||||
|
prediction = self.predict_from_text(text)
|
||||||
|
|
||||||
|
# 将结果放入队列
|
||||||
|
self.result_queue.put(prediction)
|
||||||
|
|
||||||
|
except sr.WaitTimeoutError:
|
||||||
|
# 超时,继续监听
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"监听过程中出现错误: {e}")
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
def get_latest_prediction(self) -> Optional[Dict]:
|
||||||
|
"""获取最新的预测结果"""
|
||||||
|
if not self.result_queue.empty():
|
||||||
|
return self.result_queue.get()
|
||||||
|
return None
|
||||||
|
|
||||||
|
def predict_from_audio_file(self, audio_file_path: str) -> Optional[Dict]:
|
||||||
|
"""
|
||||||
|
从音频文件预测动作
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio_file_path: 音频文件路径
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
预测结果字典
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with sr.AudioFile(audio_file_path) as source:
|
||||||
|
audio = self.recognizer.record(source)
|
||||||
|
|
||||||
|
text = self.recognize_speech(audio)
|
||||||
|
if text:
|
||||||
|
return self.predict_from_text(text)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"处理音频文件失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class RobotController:
|
||||||
|
"""机器人控制器接口"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
"""初始化机器人控制器"""
|
||||||
|
# 这里应该连接到真实的机器人控制系统
|
||||||
|
# 例如:ROS节点、机器人SDK等
|
||||||
|
logger.info("机器人控制器初始化(模拟模式)")
|
||||||
|
|
||||||
|
def execute_action(self, prediction: Dict[str, np.ndarray]):
|
||||||
|
"""
|
||||||
|
执行机器人动作
|
||||||
|
|
||||||
|
Args:
|
||||||
|
prediction: 预测的动作参数
|
||||||
|
"""
|
||||||
|
joint_positions = prediction['joint_positions']
|
||||||
|
end_effector_poses = prediction['end_effector_poses']
|
||||||
|
gripper_states = prediction['gripper_states']
|
||||||
|
text = prediction['text']
|
||||||
|
|
||||||
|
logger.info(f"执行动作指令: {text}")
|
||||||
|
logger.info(f"关节位置: {joint_positions}")
|
||||||
|
logger.info(f"末端位姿: {end_effector_poses}")
|
||||||
|
logger.info(f"夹爪状态: {gripper_states}")
|
||||||
|
|
||||||
|
# 这里应该调用真实的机器人控制API
|
||||||
|
# 例如:
|
||||||
|
# self.robot_arm.move_to_joint_positions(joint_positions)
|
||||||
|
# self.robot_arm.move_to_pose(end_effector_poses)
|
||||||
|
# self.gripper.set_position(gripper_states[0])
|
||||||
|
|
||||||
|
# 模拟执行时间
|
||||||
|
time.sleep(1.0)
|
||||||
|
logger.info("动作执行完成")
|
||||||
|
|
||||||
|
|
||||||
|
class VoiceRobotSystem:
|
||||||
|
"""完整的语音机器人系统"""
|
||||||
|
|
||||||
|
def __init__(self, model_path: str):
|
||||||
|
"""
|
||||||
|
初始化语音机器人系统
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model_path: 训练好的模型路径
|
||||||
|
"""
|
||||||
|
self.inference_engine = VoiceRobotInference(model_path)
|
||||||
|
self.robot_controller = RobotController()
|
||||||
|
|
||||||
|
self.is_running = False
|
||||||
|
self.main_thread = None
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""启动系统"""
|
||||||
|
if self.is_running:
|
||||||
|
logger.warning("系统已经在运行中...")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.is_running = True
|
||||||
|
self.inference_engine.start_listening()
|
||||||
|
self.main_thread = threading.Thread(target=self._main_loop)
|
||||||
|
self.main_thread.start()
|
||||||
|
|
||||||
|
logger.info("语音机器人系统启动")
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""停止系统"""
|
||||||
|
if not self.is_running:
|
||||||
|
logger.warning("系统当前没有运行...")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.is_running = False
|
||||||
|
self.inference_engine.stop_listening()
|
||||||
|
|
||||||
|
if self.main_thread:
|
||||||
|
self.main_thread.join()
|
||||||
|
|
||||||
|
logger.info("语音机器人系统停止")
|
||||||
|
|
||||||
|
def _main_loop(self):
|
||||||
|
"""主循环"""
|
||||||
|
while self.is_running:
|
||||||
|
# 获取最新的预测结果
|
||||||
|
prediction = self.inference_engine.get_latest_prediction()
|
||||||
|
|
||||||
|
if prediction:
|
||||||
|
# 执行机器人动作
|
||||||
|
self.robot_controller.execute_action(prediction)
|
||||||
|
|
||||||
|
time.sleep(0.1) # 控制循环频率
|
||||||
|
|
||||||
|
def process_text_command(self, text: str):
|
||||||
|
"""处理文本指令"""
|
||||||
|
prediction = self.inference_engine.predict_from_text(text)
|
||||||
|
self.robot_controller.execute_action(prediction)
|
||||||
|
|
||||||
|
def process_audio_file(self, audio_file_path: str):
|
||||||
|
"""处理音频文件"""
|
||||||
|
prediction = self.inference_engine.predict_from_audio_file(audio_file_path)
|
||||||
|
if prediction:
|
||||||
|
self.robot_controller.execute_action(prediction)
|
||||||
|
else:
|
||||||
|
logger.warning("无法从音频文件中识别指令")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""主函数 - 演示推理引擎的使用"""
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="语音机器人推理引擎")
|
||||||
|
parser.add_argument("--model", required=True, help="模型文件路径")
|
||||||
|
parser.add_argument("--mode", choices=["interactive", "text", "audio"], default="interactive", help="运行模式")
|
||||||
|
parser.add_argument("--text", help="文本指令(text模式)")
|
||||||
|
parser.add_argument("--audio", help="音频文件路径(audio模式)")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if not os.path.exists(args.model):
|
||||||
|
logger.error(f"模型文件不存在: {args.model}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 创建系统
|
||||||
|
system = VoiceRobotSystem(args.model)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if args.mode == "interactive":
|
||||||
|
# 交互模式
|
||||||
|
system.start()
|
||||||
|
|
||||||
|
print("=== 语音机器人系统 ===")
|
||||||
|
print("系统正在监听语音指令...")
|
||||||
|
print("输入 'quit' 退出程序")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
command = input("\n输入文本指令 (或 'quit' 退出): ").strip()
|
||||||
|
|
||||||
|
if command.lower() == 'quit':
|
||||||
|
break
|
||||||
|
elif command:
|
||||||
|
system.process_text_command(command)
|
||||||
|
|
||||||
|
system.stop()
|
||||||
|
|
||||||
|
elif args.mode == "text":
|
||||||
|
# 文本模式
|
||||||
|
if not args.text:
|
||||||
|
logger.error("文本模式需要提供 --text 参数")
|
||||||
|
return
|
||||||
|
|
||||||
|
system.process_text_command(args.text)
|
||||||
|
|
||||||
|
elif args.mode == "audio":
|
||||||
|
# 音频模式
|
||||||
|
if not args.audio:
|
||||||
|
logger.error("音频模式需要提供 --audio 参数")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not os.path.exists(args.audio):
|
||||||
|
logger.error(f"音频文件不存在: {args.audio}")
|
||||||
|
return
|
||||||
|
|
||||||
|
system.process_audio_file(args.audio)
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logger.info("程序被用户中断")
|
||||||
|
system.stop()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"程序运行出错: {e}")
|
||||||
|
system.stop()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
619
model_trainer.py
普通文件
619
model_trainer.py
普通文件
@ -0,0 +1,619 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
模型训练管道 - 基于LeRobot框架训练语音-动作映射模型
|
||||||
|
基于LeRobot框架的语音输入法训练系统
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
import torch.optim as optim
|
||||||
|
from torch.utils.data import Dataset, DataLoader
|
||||||
|
import numpy as np
|
||||||
|
import h5py
|
||||||
|
from typing import Dict, List, Tuple, Optional, Any
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from sklearn.model_selection import train_test_split
|
||||||
|
from sklearn.preprocessing import StandardScaler
|
||||||
|
import pickle
|
||||||
|
import time
|
||||||
|
from transformers import AutoTokenizer, AutoModel
|
||||||
|
|
||||||
|
|
||||||
|
# 设置日志
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TrainingConfig:
|
||||||
|
"""训练配置"""
|
||||||
|
# 数据相关
|
||||||
|
data_dir: str = "./collected_data"
|
||||||
|
output_dir: str = "./trained_models"
|
||||||
|
|
||||||
|
# 模型相关
|
||||||
|
text_encoder_model: str = "bert-base-chinese" # 中文BERT模型
|
||||||
|
hidden_dim: int = 256
|
||||||
|
action_dim: int = 6 # 关节数量
|
||||||
|
pose_dim: int = 6 # 末端执行器位姿维度
|
||||||
|
gripper_dim: int = 1 # 夹爪状态维度
|
||||||
|
|
||||||
|
# 训练相关
|
||||||
|
batch_size: int = 32
|
||||||
|
learning_rate: float = 1e-4
|
||||||
|
num_epochs: int = 100
|
||||||
|
validation_split: float = 0.2
|
||||||
|
early_stopping_patience: int = 10
|
||||||
|
|
||||||
|
# 设备
|
||||||
|
device: str = "cuda" if torch.cuda.is_available() else "cpu"
|
||||||
|
|
||||||
|
|
||||||
|
class VoiceRobotDataset(Dataset):
|
||||||
|
"""语音机器人数据集"""
|
||||||
|
|
||||||
|
def __init__(self, data_samples: List[Dict], tokenizer, scaler=None, max_length=128):
|
||||||
|
self.data_samples = data_samples
|
||||||
|
self.tokenizer = tokenizer
|
||||||
|
self.scaler = scaler
|
||||||
|
self.max_length = max_length
|
||||||
|
|
||||||
|
# 如果没有提供scaler,创建一个新的
|
||||||
|
if self.scaler is None:
|
||||||
|
self._fit_scaler()
|
||||||
|
|
||||||
|
def _fit_scaler(self):
|
||||||
|
"""拟合数据标准化器"""
|
||||||
|
all_actions = []
|
||||||
|
all_poses = []
|
||||||
|
all_grippers = []
|
||||||
|
|
||||||
|
for sample in self.data_samples:
|
||||||
|
# 合并所有时间步的数据
|
||||||
|
joint_positions = np.array(sample['joint_positions'])
|
||||||
|
end_effector_poses = np.array(sample['end_effector_poses'])
|
||||||
|
gripper_states = np.array(sample['gripper_states'])
|
||||||
|
|
||||||
|
all_actions.extend(joint_positions)
|
||||||
|
all_poses.extend(end_effector_poses)
|
||||||
|
all_grippers.extend(gripper_states.reshape(-1, 1))
|
||||||
|
|
||||||
|
# 创建标准化器
|
||||||
|
self.action_scaler = StandardScaler()
|
||||||
|
self.pose_scaler = StandardScaler()
|
||||||
|
self.gripper_scaler = StandardScaler()
|
||||||
|
|
||||||
|
self.action_scaler.fit(all_actions)
|
||||||
|
self.pose_scaler.fit(all_poses)
|
||||||
|
self.gripper_scaler.fit(all_grippers)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.data_samples)
|
||||||
|
|
||||||
|
def __getitem__(self, idx):
|
||||||
|
sample = self.data_samples[idx]
|
||||||
|
|
||||||
|
# 文本编码
|
||||||
|
text = sample['text']
|
||||||
|
encoding = self.tokenizer(
|
||||||
|
text,
|
||||||
|
truncation=True,
|
||||||
|
padding='max_length',
|
||||||
|
max_length=self.max_length,
|
||||||
|
return_tensors='pt'
|
||||||
|
)
|
||||||
|
|
||||||
|
# 动作数据处理
|
||||||
|
joint_positions = np.array(sample['joint_positions'])
|
||||||
|
end_effector_poses = np.array(sample['end_effector_poses'])
|
||||||
|
gripper_states = np.array(sample['gripper_states']).reshape(-1, 1)
|
||||||
|
|
||||||
|
# 数据标准化
|
||||||
|
joint_positions_norm = self.action_scaler.transform(joint_positions)
|
||||||
|
end_effector_poses_norm = self.pose_scaler.transform(end_effector_poses)
|
||||||
|
gripper_states_norm = self.gripper_scaler.transform(gripper_states)
|
||||||
|
|
||||||
|
# 转换为张量
|
||||||
|
joint_positions_tensor = torch.FloatTensor(joint_positions_norm)
|
||||||
|
end_effector_poses_tensor = torch.FloatTensor(end_effector_poses_norm)
|
||||||
|
gripper_states_tensor = torch.FloatTensor(gripper_states_norm)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'input_ids': encoding['input_ids'].squeeze(),
|
||||||
|
'attention_mask': encoding['attention_mask'].squeeze(),
|
||||||
|
'joint_positions': joint_positions_tensor,
|
||||||
|
'end_effector_poses': end_effector_poses_tensor,
|
||||||
|
'gripper_states': gripper_states_tensor,
|
||||||
|
'text': text
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class VoiceRobotModel(nn.Module):
|
||||||
|
"""语音机器人模型"""
|
||||||
|
|
||||||
|
def __init__(self, config: TrainingConfig):
|
||||||
|
super().__init__()
|
||||||
|
self.config = config
|
||||||
|
|
||||||
|
# 文本编码器
|
||||||
|
self.text_encoder = AutoModel.from_pretrained(config.text_encoder_model)
|
||||||
|
text_hidden_size = self.text_encoder.config.hidden_size
|
||||||
|
|
||||||
|
# 冻结文本编码器的部分层(可选)
|
||||||
|
# for param in self.text_encoder.parameters():
|
||||||
|
# param.requires_grad = False
|
||||||
|
|
||||||
|
# 动作解码器
|
||||||
|
self.action_decoder = nn.Sequential(
|
||||||
|
nn.Linear(text_hidden_size, config.hidden_dim),
|
||||||
|
nn.ReLU(),
|
||||||
|
nn.Dropout(0.1),
|
||||||
|
nn.Linear(config.hidden_dim, config.hidden_dim),
|
||||||
|
nn.ReLU(),
|
||||||
|
nn.Dropout(0.1),
|
||||||
|
nn.Linear(config.hidden_dim, config.action_dim)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 位姿解码器
|
||||||
|
self.pose_decoder = nn.Sequential(
|
||||||
|
nn.Linear(text_hidden_size, config.hidden_dim),
|
||||||
|
nn.ReLU(),
|
||||||
|
nn.Dropout(0.1),
|
||||||
|
nn.Linear(config.hidden_dim, config.hidden_dim),
|
||||||
|
nn.ReLU(),
|
||||||
|
nn.Dropout(0.1),
|
||||||
|
nn.Linear(config.hidden_dim, config.pose_dim)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 夹爪解码器
|
||||||
|
self.gripper_decoder = nn.Sequential(
|
||||||
|
nn.Linear(text_hidden_size, config.hidden_dim // 2),
|
||||||
|
nn.ReLU(),
|
||||||
|
nn.Dropout(0.1),
|
||||||
|
nn.Linear(config.hidden_dim // 2, config.gripper_dim),
|
||||||
|
nn.Sigmoid() # 夹爪状态通常在0-1之间
|
||||||
|
)
|
||||||
|
|
||||||
|
def forward(self, input_ids, attention_mask):
|
||||||
|
# 文本编码
|
||||||
|
text_outputs = self.text_encoder(
|
||||||
|
input_ids=input_ids,
|
||||||
|
attention_mask=attention_mask
|
||||||
|
)
|
||||||
|
|
||||||
|
# 使用[CLS] token的表示
|
||||||
|
text_features = text_outputs.last_hidden_state[:, 0, :] # [batch_size, hidden_size]
|
||||||
|
|
||||||
|
# 解码动作
|
||||||
|
joint_positions = self.action_decoder(text_features)
|
||||||
|
end_effector_poses = self.pose_decoder(text_features)
|
||||||
|
gripper_states = self.gripper_decoder(text_features)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'joint_positions': joint_positions,
|
||||||
|
'end_effector_poses': end_effector_poses,
|
||||||
|
'gripper_states': gripper_states
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class VoiceRobotTrainer:
|
||||||
|
"""语音机器人训练器"""
|
||||||
|
|
||||||
|
def __init__(self, config: TrainingConfig):
|
||||||
|
self.config = config
|
||||||
|
self.device = torch.device(config.device)
|
||||||
|
|
||||||
|
# 创建输出目录
|
||||||
|
os.makedirs(config.output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 初始化tokenizer
|
||||||
|
self.tokenizer = AutoTokenizer.from_pretrained(config.text_encoder_model)
|
||||||
|
|
||||||
|
# 训练历史
|
||||||
|
self.train_history = {
|
||||||
|
'train_loss': [],
|
||||||
|
'val_loss': [],
|
||||||
|
'joint_loss': [],
|
||||||
|
'pose_loss': [],
|
||||||
|
'gripper_loss': []
|
||||||
|
}
|
||||||
|
|
||||||
|
def load_data(self) -> Tuple[List[Dict], List[Dict]]:
|
||||||
|
"""加载训练数据"""
|
||||||
|
logger.info("加载训练数据...")
|
||||||
|
|
||||||
|
all_samples = []
|
||||||
|
|
||||||
|
# 遍历数据目录中的所有会话
|
||||||
|
data_path = Path(self.config.data_dir)
|
||||||
|
for session_dir in data_path.iterdir():
|
||||||
|
if session_dir.is_dir():
|
||||||
|
training_data_path = session_dir / "training_data.json"
|
||||||
|
if training_data_path.exists():
|
||||||
|
with open(training_data_path, 'r', encoding='utf-8') as f:
|
||||||
|
session_data = json.load(f)
|
||||||
|
all_samples.extend(session_data)
|
||||||
|
|
||||||
|
if not all_samples:
|
||||||
|
raise ValueError("未找到训练数据,请先运行数据采集和标注")
|
||||||
|
|
||||||
|
logger.info(f"加载了 {len(all_samples)} 个训练样本")
|
||||||
|
|
||||||
|
# 划分训练集和验证集
|
||||||
|
train_samples, val_samples = train_test_split(
|
||||||
|
all_samples,
|
||||||
|
test_size=self.config.validation_split,
|
||||||
|
random_state=42
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"训练集: {len(train_samples)} 样本")
|
||||||
|
logger.info(f"验证集: {len(val_samples)} 样本")
|
||||||
|
|
||||||
|
return train_samples, val_samples
|
||||||
|
|
||||||
|
def create_datasets(self, train_samples: List[Dict], val_samples: List[Dict]) -> Tuple[VoiceRobotDataset, VoiceRobotDataset]:
|
||||||
|
"""创建数据集"""
|
||||||
|
logger.info("创建数据集...")
|
||||||
|
|
||||||
|
# 创建训练数据集
|
||||||
|
train_dataset = VoiceRobotDataset(train_samples, self.tokenizer)
|
||||||
|
|
||||||
|
# 创建验证数据集(使用训练集的scaler)
|
||||||
|
val_dataset = VoiceRobotDataset(
|
||||||
|
val_samples,
|
||||||
|
self.tokenizer,
|
||||||
|
scaler={
|
||||||
|
'action_scaler': train_dataset.action_scaler,
|
||||||
|
'pose_scaler': train_dataset.pose_scaler,
|
||||||
|
'gripper_scaler': train_dataset.gripper_scaler
|
||||||
|
}
|
||||||
|
)
|
||||||
|
val_dataset.action_scaler = train_dataset.action_scaler
|
||||||
|
val_dataset.pose_scaler = train_dataset.pose_scaler
|
||||||
|
val_dataset.gripper_scaler = train_dataset.gripper_scaler
|
||||||
|
|
||||||
|
# 保存scaler
|
||||||
|
scaler_path = os.path.join(self.config.output_dir, "scalers.pkl")
|
||||||
|
with open(scaler_path, 'wb') as f:
|
||||||
|
pickle.dump({
|
||||||
|
'action_scaler': train_dataset.action_scaler,
|
||||||
|
'pose_scaler': train_dataset.pose_scaler,
|
||||||
|
'gripper_scaler': train_dataset.gripper_scaler
|
||||||
|
}, f)
|
||||||
|
|
||||||
|
return train_dataset, val_dataset
|
||||||
|
|
||||||
|
def create_model(self) -> VoiceRobotModel:
|
||||||
|
"""创建模型"""
|
||||||
|
logger.info("创建模型...")
|
||||||
|
model = VoiceRobotModel(self.config)
|
||||||
|
model.to(self.device)
|
||||||
|
|
||||||
|
# 打印模型信息
|
||||||
|
total_params = sum(p.numel() for p in model.parameters())
|
||||||
|
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||||
|
logger.info(f"模型参数总数: {total_params:,}")
|
||||||
|
logger.info(f"可训练参数: {trainable_params:,}")
|
||||||
|
|
||||||
|
return model
|
||||||
|
|
||||||
|
def compute_loss(self, predictions: Dict, targets: Dict) -> Tuple[torch.Tensor, Dict]:
|
||||||
|
"""计算损失"""
|
||||||
|
# 序列损失函数
|
||||||
|
mse_loss = nn.MSELoss()
|
||||||
|
|
||||||
|
# 计算各部分损失
|
||||||
|
joint_loss = 0
|
||||||
|
pose_loss = 0
|
||||||
|
gripper_loss = 0
|
||||||
|
|
||||||
|
batch_size = predictions['joint_positions'].size(0)
|
||||||
|
|
||||||
|
for i in range(batch_size):
|
||||||
|
# 关节位置损失
|
||||||
|
pred_joints = predictions['joint_positions'][i] # [action_dim]
|
||||||
|
target_joints = targets['joint_positions'][i] # [seq_len, action_dim]
|
||||||
|
|
||||||
|
# 使用序列的平均值作为目标
|
||||||
|
target_joints_mean = target_joints.mean(dim=0)
|
||||||
|
joint_loss += mse_loss(pred_joints, target_joints_mean)
|
||||||
|
|
||||||
|
# 末端执行器位姿损失
|
||||||
|
pred_pose = predictions['end_effector_poses'][i] # [pose_dim]
|
||||||
|
target_pose = targets['end_effector_poses'][i] # [seq_len, pose_dim]
|
||||||
|
target_pose_mean = target_pose.mean(dim=0)
|
||||||
|
pose_loss += mse_loss(pred_pose, target_pose_mean)
|
||||||
|
|
||||||
|
# 夹爪状态损失
|
||||||
|
pred_gripper = predictions['gripper_states'][i] # [gripper_dim]
|
||||||
|
target_gripper = targets['gripper_states'][i] # [seq_len, gripper_dim]
|
||||||
|
target_gripper_mean = target_gripper.mean(dim=0)
|
||||||
|
gripper_loss += mse_loss(pred_gripper, target_gripper_mean)
|
||||||
|
|
||||||
|
# 平均损失
|
||||||
|
joint_loss /= batch_size
|
||||||
|
pose_loss /= batch_size
|
||||||
|
gripper_loss /= batch_size
|
||||||
|
|
||||||
|
# 总损失(加权)
|
||||||
|
total_loss = joint_loss + pose_loss + gripper_loss * 0.5
|
||||||
|
|
||||||
|
loss_dict = {
|
||||||
|
'total_loss': total_loss,
|
||||||
|
'joint_loss': joint_loss,
|
||||||
|
'pose_loss': pose_loss,
|
||||||
|
'gripper_loss': gripper_loss
|
||||||
|
}
|
||||||
|
|
||||||
|
return total_loss, loss_dict
|
||||||
|
|
||||||
|
def train_epoch(self, model: VoiceRobotModel, dataloader: DataLoader, optimizer: optim.Optimizer) -> Dict:
|
||||||
|
"""训练一个epoch"""
|
||||||
|
model.train()
|
||||||
|
total_loss = 0
|
||||||
|
total_joint_loss = 0
|
||||||
|
total_pose_loss = 0
|
||||||
|
total_gripper_loss = 0
|
||||||
|
num_batches = 0
|
||||||
|
|
||||||
|
for batch in dataloader:
|
||||||
|
# 移动数据到设备
|
||||||
|
input_ids = batch['input_ids'].to(self.device)
|
||||||
|
attention_mask = batch['attention_mask'].to(self.device)
|
||||||
|
|
||||||
|
targets = {
|
||||||
|
'joint_positions': batch['joint_positions'].to(self.device),
|
||||||
|
'end_effector_poses': batch['end_effector_poses'].to(self.device),
|
||||||
|
'gripper_states': batch['gripper_states'].to(self.device)
|
||||||
|
}
|
||||||
|
|
||||||
|
# 前向传播
|
||||||
|
optimizer.zero_grad()
|
||||||
|
predictions = model(input_ids, attention_mask)
|
||||||
|
|
||||||
|
# 计算损失
|
||||||
|
loss, loss_dict = self.compute_loss(predictions, targets)
|
||||||
|
|
||||||
|
# 反向传播
|
||||||
|
loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
|
||||||
|
# 累计损失
|
||||||
|
total_loss += loss.item()
|
||||||
|
total_joint_loss += loss_dict['joint_loss'].item()
|
||||||
|
total_pose_loss += loss_dict['pose_loss'].item()
|
||||||
|
total_gripper_loss += loss_dict['gripper_loss'].item()
|
||||||
|
num_batches += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
'train_loss': total_loss / num_batches,
|
||||||
|
'joint_loss': total_joint_loss / num_batches,
|
||||||
|
'pose_loss': total_pose_loss / num_batches,
|
||||||
|
'gripper_loss': total_gripper_loss / num_batches
|
||||||
|
}
|
||||||
|
|
||||||
|
def validate_epoch(self, model: VoiceRobotModel, dataloader: DataLoader) -> Dict:
|
||||||
|
"""验证一个epoch"""
|
||||||
|
model.eval()
|
||||||
|
total_loss = 0
|
||||||
|
total_joint_loss = 0
|
||||||
|
total_pose_loss = 0
|
||||||
|
total_gripper_loss = 0
|
||||||
|
num_batches = 0
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
for batch in dataloader:
|
||||||
|
# 移动数据到设备
|
||||||
|
input_ids = batch['input_ids'].to(self.device)
|
||||||
|
attention_mask = batch['attention_mask'].to(self.device)
|
||||||
|
|
||||||
|
targets = {
|
||||||
|
'joint_positions': batch['joint_positions'].to(self.device),
|
||||||
|
'end_effector_poses': batch['end_effector_poses'].to(self.device),
|
||||||
|
'gripper_states': batch['gripper_states'].to(self.device)
|
||||||
|
}
|
||||||
|
|
||||||
|
# 前向传播
|
||||||
|
predictions = model(input_ids, attention_mask)
|
||||||
|
|
||||||
|
# 计算损失
|
||||||
|
loss, loss_dict = self.compute_loss(predictions, targets)
|
||||||
|
|
||||||
|
# 累计损失
|
||||||
|
total_loss += loss.item()
|
||||||
|
total_joint_loss += loss_dict['joint_loss'].item()
|
||||||
|
total_pose_loss += loss_dict['pose_loss'].item()
|
||||||
|
total_gripper_loss += loss_dict['gripper_loss'].item()
|
||||||
|
num_batches += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
'val_loss': total_loss / num_batches,
|
||||||
|
'joint_loss': total_joint_loss / num_batches,
|
||||||
|
'pose_loss': total_pose_loss / num_batches,
|
||||||
|
'gripper_loss': total_gripper_loss / num_batches
|
||||||
|
}
|
||||||
|
|
||||||
|
def train(self):
|
||||||
|
"""主训练函数"""
|
||||||
|
logger.info("开始训练...")
|
||||||
|
|
||||||
|
# 加载数据
|
||||||
|
train_samples, val_samples = self.load_data()
|
||||||
|
train_dataset, val_dataset = self.create_datasets(train_samples, val_samples)
|
||||||
|
|
||||||
|
# 创建数据加载器
|
||||||
|
train_dataloader = DataLoader(
|
||||||
|
train_dataset,
|
||||||
|
batch_size=self.config.batch_size,
|
||||||
|
shuffle=True,
|
||||||
|
num_workers=0 # 在Windows上设置为0
|
||||||
|
)
|
||||||
|
val_dataloader = DataLoader(
|
||||||
|
val_dataset,
|
||||||
|
batch_size=self.config.batch_size,
|
||||||
|
shuffle=False,
|
||||||
|
num_workers=0
|
||||||
|
)
|
||||||
|
|
||||||
|
# 创建模型
|
||||||
|
model = self.create_model()
|
||||||
|
|
||||||
|
# 创建优化器
|
||||||
|
optimizer = optim.Adam(model.parameters(), lr=self.config.learning_rate)
|
||||||
|
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5)
|
||||||
|
|
||||||
|
# 早停机制
|
||||||
|
best_val_loss = float('inf')
|
||||||
|
patience_counter = 0
|
||||||
|
|
||||||
|
# 训练循环
|
||||||
|
for epoch in range(self.config.num_epochs):
|
||||||
|
start_time = time.time()
|
||||||
|
|
||||||
|
# 训练
|
||||||
|
train_metrics = self.train_epoch(model, train_dataloader, optimizer)
|
||||||
|
|
||||||
|
# 验证
|
||||||
|
val_metrics = self.validate_epoch(model, val_dataloader)
|
||||||
|
|
||||||
|
# 更新学习率
|
||||||
|
scheduler.step(val_metrics['val_loss'])
|
||||||
|
|
||||||
|
# 记录历史
|
||||||
|
self.train_history['train_loss'].append(train_metrics['train_loss'])
|
||||||
|
self.train_history['val_loss'].append(val_metrics['val_loss'])
|
||||||
|
self.train_history['joint_loss'].append(train_metrics['joint_loss'])
|
||||||
|
self.train_history['pose_loss'].append(train_metrics['pose_loss'])
|
||||||
|
self.train_history['gripper_loss'].append(train_metrics['gripper_loss'])
|
||||||
|
|
||||||
|
# 打印进度
|
||||||
|
epoch_time = time.time() - start_time
|
||||||
|
logger.info(
|
||||||
|
f"Epoch {epoch+1}/{self.config.num_epochs} - "
|
||||||
|
f"Train Loss: {train_metrics['train_loss']:.4f}, "
|
||||||
|
f"Val Loss: {val_metrics['val_loss']:.4f}, "
|
||||||
|
f"Time: {epoch_time:.2f}s"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 保存最佳模型
|
||||||
|
if val_metrics['val_loss'] < best_val_loss:
|
||||||
|
best_val_loss = val_metrics['val_loss']
|
||||||
|
patience_counter = 0
|
||||||
|
|
||||||
|
# 保存模型
|
||||||
|
model_path = os.path.join(self.config.output_dir, "best_model.pth")
|
||||||
|
torch.save({
|
||||||
|
'model_state_dict': model.state_dict(),
|
||||||
|
'config': self.config,
|
||||||
|
'train_history': self.train_history,
|
||||||
|
'epoch': epoch
|
||||||
|
}, model_path)
|
||||||
|
|
||||||
|
logger.info(f"保存最佳模型到 {model_path}")
|
||||||
|
else:
|
||||||
|
patience_counter += 1
|
||||||
|
|
||||||
|
# 早停检查
|
||||||
|
if patience_counter >= self.config.early_stopping_patience:
|
||||||
|
logger.info(f"早停触发,在第 {epoch+1} 轮停止训练")
|
||||||
|
break
|
||||||
|
|
||||||
|
# 保存最终模型
|
||||||
|
final_model_path = os.path.join(self.config.output_dir, "final_model.pth")
|
||||||
|
torch.save({
|
||||||
|
'model_state_dict': model.state_dict(),
|
||||||
|
'config': self.config,
|
||||||
|
'train_history': self.train_history,
|
||||||
|
'epoch': epoch
|
||||||
|
}, final_model_path)
|
||||||
|
|
||||||
|
# 保存训练历史
|
||||||
|
history_path = os.path.join(self.config.output_dir, "training_history.json")
|
||||||
|
with open(history_path, 'w') as f:
|
||||||
|
json.dump(self.train_history, f, indent=2)
|
||||||
|
|
||||||
|
# 绘制训练曲线
|
||||||
|
self.plot_training_curves()
|
||||||
|
|
||||||
|
logger.info("训练完成!")
|
||||||
|
|
||||||
|
def plot_training_curves(self):
|
||||||
|
"""绘制训练曲线"""
|
||||||
|
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans'] # 支持中文显示
|
||||||
|
plt.rcParams['axes.unicode_minus'] = False
|
||||||
|
|
||||||
|
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||||
|
|
||||||
|
# 总损失
|
||||||
|
axes[0, 0].plot(self.train_history['train_loss'], label='训练损失')
|
||||||
|
axes[0, 0].plot(self.train_history['val_loss'], label='验证损失')
|
||||||
|
axes[0, 0].set_title('总损失')
|
||||||
|
axes[0, 0].set_xlabel('Epoch')
|
||||||
|
axes[0, 0].set_ylabel('Loss')
|
||||||
|
axes[0, 0].legend()
|
||||||
|
axes[0, 0].grid(True)
|
||||||
|
|
||||||
|
# 关节损失
|
||||||
|
axes[0, 1].plot(self.train_history['joint_loss'], label='关节损失')
|
||||||
|
axes[0, 1].set_title('关节位置损失')
|
||||||
|
axes[0, 1].set_xlabel('Epoch')
|
||||||
|
axes[0, 1].set_ylabel('Loss')
|
||||||
|
axes[0, 1].legend()
|
||||||
|
axes[0, 1].grid(True)
|
||||||
|
|
||||||
|
# 位姿损失
|
||||||
|
axes[1, 0].plot(self.train_history['pose_loss'], label='位姿损失')
|
||||||
|
axes[1, 0].set_title('末端执行器位姿损失')
|
||||||
|
axes[1, 0].set_xlabel('Epoch')
|
||||||
|
axes[1, 0].set_ylabel('Loss')
|
||||||
|
axes[1, 0].legend()
|
||||||
|
axes[1, 0].grid(True)
|
||||||
|
|
||||||
|
# 夹爪损失
|
||||||
|
axes[1, 1].plot(self.train_history['gripper_loss'], label='夹爪损失')
|
||||||
|
axes[1, 1].set_title('夹爪状态损失')
|
||||||
|
axes[1, 1].set_xlabel('Epoch')
|
||||||
|
axes[1, 1].set_ylabel('Loss')
|
||||||
|
axes[1, 1].legend()
|
||||||
|
axes[1, 1].grid(True)
|
||||||
|
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
# 保存图片
|
||||||
|
plot_path = os.path.join(self.config.output_dir, "training_curves.png")
|
||||||
|
plt.savefig(plot_path, dpi=300, bbox_inches='tight')
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
logger.info(f"训练曲线已保存到 {plot_path}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""主函数"""
|
||||||
|
# 创建配置
|
||||||
|
config = TrainingConfig(
|
||||||
|
data_dir="./collected_data",
|
||||||
|
output_dir="./trained_models",
|
||||||
|
batch_size=16, # 根据GPU内存调整
|
||||||
|
num_epochs=50,
|
||||||
|
learning_rate=1e-4
|
||||||
|
)
|
||||||
|
|
||||||
|
# 创建训练器
|
||||||
|
trainer = VoiceRobotTrainer(config)
|
||||||
|
|
||||||
|
# 开始训练
|
||||||
|
try:
|
||||||
|
trainer.train()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"训练过程中出现错误: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
43
requirements.txt
普通文件
43
requirements.txt
普通文件
@ -0,0 +1,43 @@
|
|||||||
|
# 语音机器人训练系统依赖包
|
||||||
|
|
||||||
|
# 核心框架
|
||||||
|
torch>=1.9.0
|
||||||
|
numpy>=1.21.0
|
||||||
|
h5py>=3.1.0
|
||||||
|
|
||||||
|
# LeRobot框架
|
||||||
|
lerobot
|
||||||
|
|
||||||
|
# 数据处理
|
||||||
|
pandas>=1.3.0
|
||||||
|
scipy>=1.7.0
|
||||||
|
opencv-python>=4.5.0
|
||||||
|
pillow>=8.3.0
|
||||||
|
|
||||||
|
# 语音识别
|
||||||
|
SpeechRecognition>=3.8.1
|
||||||
|
pyaudio>=0.2.11
|
||||||
|
pydub>=0.25.1
|
||||||
|
|
||||||
|
# 游戏手柄控制
|
||||||
|
pygame>=2.0.0
|
||||||
|
|
||||||
|
# GUI和可视化
|
||||||
|
matplotlib>=3.4.0
|
||||||
|
|
||||||
|
# Web框架(用于标注工具的Web版本)
|
||||||
|
flask>=2.0.0
|
||||||
|
flask-cors>=3.0.10
|
||||||
|
|
||||||
|
# 机器学习相关
|
||||||
|
scikit-learn>=1.0.0
|
||||||
|
transformers>=4.12.0
|
||||||
|
|
||||||
|
# 数据序列化
|
||||||
|
jsonschema>=3.2.0
|
||||||
|
|
||||||
|
# 开发工具
|
||||||
|
pytest>=6.2.0
|
||||||
|
black>=21.0.0
|
||||||
|
flake8>=3.9.0
|
||||||
|
|
||||||
93
setup.py
普通文件
93
setup.py
普通文件
@ -0,0 +1,93 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
语音输入法机器人训练系统安装脚本
|
||||||
|
"""
|
||||||
|
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
import os
|
||||||
|
|
||||||
|
# 读取README文件
|
||||||
|
def read_readme():
|
||||||
|
with open("README.md", "r", encoding="utf-8") as fh:
|
||||||
|
return fh.read()
|
||||||
|
|
||||||
|
# 读取requirements文件
|
||||||
|
def read_requirements():
|
||||||
|
with open("requirements.txt", "r", encoding="utf-8") as fh:
|
||||||
|
return [line.strip() for line in fh if line.strip() and not line.startswith("#")]
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="voice-robot-training-system",
|
||||||
|
version="1.0.0",
|
||||||
|
author="Manus AI",
|
||||||
|
author_email="contact@manus.ai",
|
||||||
|
description="基于LeRobot框架的语音输入法机器人训练系统",
|
||||||
|
long_description=read_readme(),
|
||||||
|
long_description_content_type="text/markdown",
|
||||||
|
url="https://github.com/manus-ai/voice-robot-training-system",
|
||||||
|
packages=find_packages(),
|
||||||
|
classifiers=[
|
||||||
|
"Development Status :: 4 - Beta",
|
||||||
|
"Intended Audience :: Developers",
|
||||||
|
"Intended Audience :: Science/Research",
|
||||||
|
"License :: OSI Approved :: Apache Software License",
|
||||||
|
"Operating System :: OS Independent",
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.8",
|
||||||
|
"Programming Language :: Python :: 3.9",
|
||||||
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||||
|
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
|
||||||
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||||
|
],
|
||||||
|
python_requires=">=3.8",
|
||||||
|
install_requires=read_requirements(),
|
||||||
|
extras_require={
|
||||||
|
"dev": [
|
||||||
|
"pytest>=6.2.0",
|
||||||
|
"black>=21.0.0",
|
||||||
|
"flake8>=3.9.0",
|
||||||
|
"mypy>=0.910",
|
||||||
|
"pre-commit>=2.15.0",
|
||||||
|
],
|
||||||
|
"docs": [
|
||||||
|
"sphinx>=4.0.0",
|
||||||
|
"sphinx-rtd-theme>=0.5.0",
|
||||||
|
"myst-parser>=0.15.0",
|
||||||
|
],
|
||||||
|
"gpu": [
|
||||||
|
"torch[cuda]>=1.9.0",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
entry_points={
|
||||||
|
"console_scripts": [
|
||||||
|
"voice-robot-collect=data_collector:main",
|
||||||
|
"voice-robot-annotate=annotation_tool:main",
|
||||||
|
"voice-robot-train=model_trainer:main",
|
||||||
|
"voice-robot-infer=inference_engine:main",
|
||||||
|
"voice-robot-example=examples.example_usage:main",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
include_package_data=True,
|
||||||
|
package_data={
|
||||||
|
"": ["*.md", "*.txt", "*.yml", "*.yaml", "*.json"],
|
||||||
|
},
|
||||||
|
zip_safe=False,
|
||||||
|
keywords=[
|
||||||
|
"robotics",
|
||||||
|
"voice-control",
|
||||||
|
"machine-learning",
|
||||||
|
"deep-learning",
|
||||||
|
"lerobot",
|
||||||
|
"speech-recognition",
|
||||||
|
"human-robot-interaction",
|
||||||
|
"imitation-learning",
|
||||||
|
],
|
||||||
|
project_urls={
|
||||||
|
"Bug Reports": "https://github.com/manus-ai/voice-robot-training-system/issues",
|
||||||
|
"Source": "https://github.com/manus-ai/voice-robot-training-system",
|
||||||
|
"Documentation": "https://voice-robot-training-system.readthedocs.io/",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
在新工单中引用
屏蔽一个用户