这个提交包含在:
2025-09-05 16:48:26 +08:00
当前提交 2a16646b8e
共有 8 个文件被更改,包括 2382 次插入 和 0 次删除

10
TODO.MD 普通文件
查看文件

@ -0,0 +1,10 @@
# ToDos
## data_controller
- 给手臂驱动加上真实的控制机器人的部分
- _get_robot_state 按照机器人实际情况改
## trainer
- 不能训练平均值,需要生成完整的动作序列
## microphone
- 移除语言识别部分

518
annotation_tool.py 普通文件
查看文件

@ -0,0 +1,518 @@
#!/usr/bin/env python3
"""
数据标注工具 - 用于为录制的机器人动作数据添加语音指令标注
基于LeRobot框架的语音输入法训练系统
"""
import os
import json
import tkinter as tk
from tkinter import ttk, filedialog, messagebox
import h5py
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import cv2
from PIL import Image, ImageTk
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, asdict
import threading
import time
@dataclass
class Annotation:
"""标注数据结构"""
start_time: float
end_time: float
text: str
confidence: float = 1.0
annotator: str = "default"
timestamp: float = None
def __post_init__(self):
if self.timestamp is None:
self.timestamp = time.time()
class AnnotationTool:
"""数据标注工具GUI"""
def __init__(self):
self.root = tk.Tk()
self.root.title("语音机器人训练系统 - 数据标注工具")
self.root.geometry("1200x800")
# 数据相关变量
self.session_dir = None
self.robot_data = None
self.metadata = None
self.annotations = []
self.current_time = 0.0
self.total_time = 0.0
self.playing = False
self.play_thread = None
# GUI组件
self.setup_gui()
# 绑定键盘事件
self.root.bind('<Key>', self.on_key_press)
self.root.focus_set()
def setup_gui(self):
"""设置GUI界面"""
# 主框架
main_frame = ttk.Frame(self.root)
main_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
# 顶部控制面板
control_frame = ttk.Frame(main_frame)
control_frame.pack(fill=tk.X, pady=(0, 10))
# 文件操作按钮
ttk.Button(control_frame, text="加载数据", command=self.load_data).pack(side=tk.LEFT, padx=(0, 5))
ttk.Button(control_frame, text="保存标注", command=self.save_annotations).pack(side=tk.LEFT, padx=(0, 5))
ttk.Button(control_frame, text="导出数据", command=self.export_data).pack(side=tk.LEFT, padx=(0, 5))
# 播放控制按钮
ttk.Separator(control_frame, orient=tk.VERTICAL).pack(side=tk.LEFT, padx=10, fill=tk.Y)
ttk.Button(control_frame, text="播放/暂停", command=self.toggle_play).pack(side=tk.LEFT, padx=(0, 5))
ttk.Button(control_frame, text="重置", command=self.reset_playback).pack(side=tk.LEFT, padx=(0, 5))
# 时间显示
self.time_label = ttk.Label(control_frame, text="时间: 0.00 / 0.00 秒")
self.time_label.pack(side=tk.RIGHT)
# 中间内容区域
content_frame = ttk.Frame(main_frame)
content_frame.pack(fill=tk.BOTH, expand=True)
# 左侧:数据可视化
left_frame = ttk.LabelFrame(content_frame, text="数据可视化")
left_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=(0, 5))
# 创建matplotlib图表
self.fig = Figure(figsize=(8, 6), dpi=100)
self.canvas = FigureCanvasTkAgg(self.fig, left_frame)
self.canvas.get_tk_widget().pack(fill=tk.BOTH, expand=True)
# 时间滑块
self.time_scale = ttk.Scale(left_frame, from_=0, to=100, orient=tk.HORIZONTAL, command=self.on_time_change)
self.time_scale.pack(fill=tk.X, pady=5)
# 右侧:标注面板
right_frame = ttk.LabelFrame(content_frame, text="标注面板", width=300)
right_frame.pack(side=tk.RIGHT, fill=tk.Y, padx=(5, 0))
right_frame.pack_propagate(False)
# 当前标注输入
ttk.Label(right_frame, text="语音指令:").pack(anchor=tk.W, pady=(10, 0))
self.text_entry = tk.Text(right_frame, height=3, wrap=tk.WORD)
self.text_entry.pack(fill=tk.X, pady=5)
# 时间范围设置
time_frame = ttk.Frame(right_frame)
time_frame.pack(fill=tk.X, pady=5)
ttk.Label(time_frame, text="开始时间:").grid(row=0, column=0, sticky=tk.W)
self.start_time_var = tk.StringVar(value="0.00")
ttk.Entry(time_frame, textvariable=self.start_time_var, width=10).grid(row=0, column=1, padx=(5, 0))
ttk.Label(time_frame, text="结束时间:").grid(row=1, column=0, sticky=tk.W)
self.end_time_var = tk.StringVar(value="0.00")
ttk.Entry(time_frame, textvariable=self.end_time_var, width=10).grid(row=1, column=1, padx=(5, 0))
# 标注操作按钮
button_frame = ttk.Frame(right_frame)
button_frame.pack(fill=tk.X, pady=10)
ttk.Button(button_frame, text="设置开始", command=self.set_start_time).pack(side=tk.LEFT, padx=(0, 5))
ttk.Button(button_frame, text="设置结束", command=self.set_end_time).pack(side=tk.LEFT)
ttk.Button(right_frame, text="添加标注", command=self.add_annotation).pack(fill=tk.X, pady=5)
ttk.Button(right_frame, text="删除选中", command=self.delete_annotation).pack(fill=tk.X, pady=5)
# 标注列表
ttk.Label(right_frame, text="已有标注:").pack(anchor=tk.W, pady=(10, 0))
# 创建Treeview来显示标注
columns = ('时间范围', '文本')
self.annotation_tree = ttk.Treeview(right_frame, columns=columns, show='headings', height=8)
self.annotation_tree.heading('时间范围', text='时间范围')
self.annotation_tree.heading('文本', text='语音指令')
self.annotation_tree.column('时间范围', width=100)
self.annotation_tree.column('文本', width=180)
self.annotation_tree.pack(fill=tk.BOTH, expand=True, pady=5)
# 绑定选择事件
self.annotation_tree.bind('<<TreeviewSelect>>', self.on_annotation_select)
# 底部状态栏
self.status_label = ttk.Label(main_frame, text="请加载数据文件开始标注")
self.status_label.pack(fill=tk.X, pady=(10, 0))
def load_data(self):
"""加载数据文件"""
session_dir = filedialog.askdirectory(title="选择数据会话目录")
if not session_dir:
return
try:
# 检查必要文件
robot_data_path = os.path.join(session_dir, "robot_data.h5")
metadata_path = os.path.join(session_dir, "metadata.json")
if not os.path.exists(robot_data_path):
messagebox.showerror("错误", "未找到机器人数据文件 (robot_data.h5)")
return
if not os.path.exists(metadata_path):
messagebox.showerror("错误", "未找到元数据文件 (metadata.json)")
return
# 加载数据
self.session_dir = session_dir
# 加载HDF5数据
self.robot_data = h5py.File(robot_data_path, 'r')
# 加载元数据
with open(metadata_path, 'r') as f:
self.metadata = json.load(f)
# 设置时间范围
timestamps = self.robot_data['timestamps'][:]
self.total_time = timestamps[-1] - timestamps[0]
self.time_scale.configure(to=self.total_time)
# 加载已有标注(如果存在)
self.load_existing_annotations()
# 更新显示
self.update_visualization()
self.update_status(f"已加载数据: {os.path.basename(session_dir)}")
except Exception as e:
messagebox.showerror("错误", f"加载数据失败: {str(e)}")
def load_existing_annotations(self):
"""加载已有的标注数据"""
annotations_path = os.path.join(self.session_dir, "annotations.json")
if os.path.exists(annotations_path):
try:
with open(annotations_path, 'r', encoding='utf-8') as f:
annotations_data = json.load(f)
self.annotations = []
for ann_data in annotations_data:
annotation = Annotation(**ann_data)
self.annotations.append(annotation)
self.update_annotation_list()
self.update_status(f"加载了 {len(self.annotations)} 个已有标注")
except Exception as e:
print(f"加载标注文件失败: {e}")
def save_annotations(self):
"""保存标注数据"""
if not self.session_dir:
messagebox.showwarning("警告", "请先加载数据")
return
try:
annotations_path = os.path.join(self.session_dir, "annotations.json")
annotations_data = [asdict(ann) for ann in self.annotations]
with open(annotations_path, 'w', encoding='utf-8') as f:
json.dump(annotations_data, f, indent=2, ensure_ascii=False)
self.update_status(f"已保存 {len(self.annotations)} 个标注到 {annotations_path}")
messagebox.showinfo("成功", "标注数据已保存")
except Exception as e:
messagebox.showerror("错误", f"保存标注失败: {str(e)}")
def export_data(self):
"""导出训练数据"""
if not self.session_dir or not self.annotations:
messagebox.showwarning("警告", "请先加载数据并添加标注")
return
try:
export_path = os.path.join(self.session_dir, "training_data.json")
# 准备训练数据格式
training_data = []
timestamps = self.robot_data['timestamps'][:]
start_timestamp = timestamps[0]
for annotation in self.annotations:
# 找到对应时间范围的数据索引
start_idx = np.searchsorted(timestamps, start_timestamp + annotation.start_time)
end_idx = np.searchsorted(timestamps, start_timestamp + annotation.end_time)
# 提取对应的机器人状态数据
segment_data = {
"text": annotation.text,
"start_time": annotation.start_time,
"end_time": annotation.end_time,
"joint_positions": self.robot_data['joint_positions'][start_idx:end_idx].tolist(),
"joint_velocities": self.robot_data['joint_velocities'][start_idx:end_idx].tolist(),
"end_effector_poses": self.robot_data['end_effector_poses'][start_idx:end_idx].tolist(),
"gripper_states": self.robot_data['gripper_states'][start_idx:end_idx].tolist(),
"timestamps": (timestamps[start_idx:end_idx] - start_timestamp).tolist()
}
training_data.append(segment_data)
# 保存训练数据
with open(export_path, 'w', encoding='utf-8') as f:
json.dump(training_data, f, indent=2, ensure_ascii=False)
self.update_status(f"已导出 {len(training_data)} 个训练样本到 {export_path}")
messagebox.showinfo("成功", f"训练数据已导出到 {export_path}")
except Exception as e:
messagebox.showerror("错误", f"导出数据失败: {str(e)}")
def toggle_play(self):
"""切换播放/暂停状态"""
if not self.robot_data:
return
self.playing = not self.playing
if self.playing:
self.play_thread = threading.Thread(target=self.play_data)
self.play_thread.start()
else:
if self.play_thread:
self.play_thread = None
def play_data(self):
"""播放数据"""
timestamps = self.robot_data['timestamps'][:]
start_time = timestamps[0]
while self.playing and self.current_time < self.total_time:
# 更新当前时间
self.current_time += 0.1 # 每次增加0.1秒
# 更新GUI
self.root.after(0, self.update_time_display)
self.root.after(0, self.update_visualization)
time.sleep(0.1) # 控制播放速度
self.playing = False
def reset_playback(self):
"""重置播放"""
self.playing = False
self.current_time = 0.0
self.update_time_display()
self.update_visualization()
def on_time_change(self, value):
"""时间滑块变化事件"""
if not self.playing: # 只在非播放状态下响应滑块变化
self.current_time = float(value)
self.update_time_display()
self.update_visualization()
def update_time_display(self):
"""更新时间显示"""
self.time_label.config(text=f"时间: {self.current_time:.2f} / {self.total_time:.2f} 秒")
self.time_scale.set(self.current_time)
def update_visualization(self):
"""更新数据可视化"""
if not self.robot_data:
return
self.fig.clear()
# 获取当前时间对应的数据索引
timestamps = self.robot_data['timestamps'][:]
start_timestamp = timestamps[0]
current_timestamp = start_timestamp + self.current_time
current_idx = np.searchsorted(timestamps, current_timestamp)
# 绘制关节位置
ax1 = self.fig.add_subplot(2, 1, 1)
joint_positions = self.robot_data['joint_positions'][:]
# 显示一段时间窗口的数据
window_size = min(100, len(joint_positions))
start_idx = max(0, current_idx - window_size // 2)
end_idx = min(len(joint_positions), start_idx + window_size)
time_window = (timestamps[start_idx:end_idx] - start_timestamp)
for i in range(joint_positions.shape[1]):
ax1.plot(time_window, joint_positions[start_idx:end_idx, i], label=f'Joint {i+1}')
# 标记当前时间
ax1.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
ax1.set_xlabel('Time (s)')
ax1.set_ylabel('Joint Position (rad)')
ax1.set_title('Joint Positions')
ax1.legend()
ax1.grid(True)
# 绘制末端执行器位置
ax2 = self.fig.add_subplot(2, 1, 2)
end_effector_poses = self.robot_data['end_effector_poses'][:]
for i in range(3): # 只显示位置信息 (x, y, z)
ax2.plot(time_window, end_effector_poses[start_idx:end_idx, i], label=f'{"XYZ"[i]}')
ax2.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
ax2.set_xlabel('Time (s)')
ax2.set_ylabel('Position (m)')
ax2.set_title('End Effector Position')
ax2.legend()
ax2.grid(True)
# 绘制标注区间
for annotation in self.annotations:
ax1.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
ax2.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
self.fig.tight_layout()
self.canvas.draw()
def set_start_time(self):
"""设置标注开始时间"""
self.start_time_var.set(f"{self.current_time:.2f}")
def set_end_time(self):
"""设置标注结束时间"""
self.end_time_var.set(f"{self.current_time:.2f}")
def add_annotation(self):
"""添加标注"""
try:
start_time = float(self.start_time_var.get())
end_time = float(self.end_time_var.get())
text = self.text_entry.get("1.0", tk.END).strip()
if not text:
messagebox.showwarning("警告", "请输入语音指令文本")
return
if start_time >= end_time:
messagebox.showwarning("警告", "开始时间必须小于结束时间")
return
# 创建新标注
annotation = Annotation(
start_time=start_time,
end_time=end_time,
text=text
)
self.annotations.append(annotation)
self.update_annotation_list()
self.update_visualization()
# 清空输入框
self.text_entry.delete("1.0", tk.END)
self.update_status(f"已添加标注: {text[:30]}...")
except ValueError:
messagebox.showerror("错误", "时间格式不正确")
except Exception as e:
messagebox.showerror("错误", f"添加标注失败: {str(e)}")
def delete_annotation(self):
"""删除选中的标注"""
selection = self.annotation_tree.selection()
if not selection:
messagebox.showwarning("警告", "请选择要删除的标注")
return
try:
# 获取选中项的索引
item = selection[0]
index = self.annotation_tree.index(item)
# 删除标注
deleted_annotation = self.annotations.pop(index)
self.update_annotation_list()
self.update_visualization()
self.update_status(f"已删除标注: {deleted_annotation.text[:30]}...")
except Exception as e:
messagebox.showerror("错误", f"删除标注失败: {str(e)}")
def update_annotation_list(self):
"""更新标注列表显示"""
# 清空现有项目
for item in self.annotation_tree.get_children():
self.annotation_tree.delete(item)
# 添加标注项目
for annotation in self.annotations:
time_range = f"{annotation.start_time:.2f}-{annotation.end_time:.2f}s"
text_preview = annotation.text[:50] + "..." if len(annotation.text) > 50 else annotation.text
self.annotation_tree.insert('', tk.END, values=(time_range, text_preview))
def on_annotation_select(self, event):
"""标注选择事件"""
selection = self.annotation_tree.selection()
if selection:
item = selection[0]
index = self.annotation_tree.index(item)
annotation = self.annotations[index]
# 更新输入框
self.start_time_var.set(f"{annotation.start_time:.2f}")
self.end_time_var.set(f"{annotation.end_time:.2f}")
self.text_entry.delete("1.0", tk.END)
self.text_entry.insert("1.0", annotation.text)
def on_key_press(self, event):
"""键盘事件处理"""
if event.keysym == 'space':
self.toggle_play()
elif event.keysym == 'Left':
self.current_time = max(0, self.current_time - 1.0)
self.update_time_display()
self.update_visualization()
elif event.keysym == 'Right':
self.current_time = min(self.total_time, self.current_time + 1.0)
self.update_time_display()
self.update_visualization()
def update_status(self, message):
"""更新状态栏"""
self.status_label.config(text=message)
def run(self):
"""运行GUI"""
self.root.mainloop()
# 清理资源
if self.robot_data:
self.robot_data.close()
def main():
"""主函数"""
app = AnnotationTool()
app.run()
if __name__ == "__main__":
main()

280
data_collector.py 普通文件
查看文件

@ -0,0 +1,280 @@
#!/usr/bin/env python3
"""
数据采集模块 - 用于录制机器人动作和传感器数据
基于LeRobot框架的语音输入法训练系统
"""
import os
import json
import time
import threading
from datetime import datetime
from typing import Dict, List, Optional, Any
import numpy as np
import h5py
import pygame
from dataclasses import dataclass, asdict
@dataclass
class RobotState:
"""机器人状态数据结构"""
timestamp: float
joint_positions: List[float]
joint_velocities: List[float]
joint_torques: List[float]
end_effector_pose: List[float] # [x, y, z, roll, pitch, yaw]
gripper_state: float
camera_image: Optional[np.ndarray] = None
depth_image: Optional[np.ndarray] = None
@dataclass
class ControlCommand:
"""控制指令数据结构"""
timestamp: float
command_type: str # 'joystick', 'button', 'voice'
command_data: Dict[str, Any]
class DataCollector:
"""数据采集器类"""
def __init__(self, output_dir: str = "./collected_data"):
self.output_dir = output_dir
self.is_recording = False
self.recording_thread = None
self.data_buffer = []
self.command_buffer = []
# 创建输出目录
os.makedirs(output_dir, exist_ok=True)
# 初始化pygame用于手柄控制
pygame.init()
pygame.joystick.init()
# 检查手柄连接
self.joystick = None
if pygame.joystick.get_count() > 0:
self.joystick = pygame.joystick.Joystick(0)
self.joystick.init()
print(f"检测到手柄: {self.joystick.get_name()}")
else:
print("警告: 未检测到手柄,将使用键盘模拟")
def start_recording(self, session_name: str = None) -> str:
"""开始录制数据"""
if self.is_recording:
print("已经在录制中...")
return None
if session_name is None:
session_name = f"session_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
self.session_name = session_name
self.session_dir = os.path.join(self.output_dir, session_name)
os.makedirs(self.session_dir, exist_ok=True)
# 清空缓冲区
self.data_buffer.clear()
self.command_buffer.clear()
self.is_recording = True
self.recording_thread = threading.Thread(target=self._recording_loop)
self.recording_thread.start()
print(f"开始录制会话: {session_name}")
return session_name
def stop_recording(self):
"""停止录制并保存数据"""
if not self.is_recording:
print("当前没有在录制...")
return
self.is_recording = False
if self.recording_thread:
self.recording_thread.join()
# 保存数据
self._save_data()
print(f"录制完成,数据已保存到: {self.session_dir}")
def _recording_loop(self):
"""录制循环"""
print("录制循环开始...")
print("使用手柄控制机器人,按 'q' 键停止录制")
while self.is_recording:
try:
# 处理pygame事件
pygame.event.pump()
# 获取当前时间戳
timestamp = time.time()
# 模拟获取机器人状态数据
robot_state = self._get_robot_state(timestamp)
self.data_buffer.append(robot_state)
# 获取控制指令
control_command = self._get_control_command(timestamp)
if control_command:
self.command_buffer.append(control_command)
# 控制录制频率 (50Hz)
time.sleep(0.02)
except KeyboardInterrupt:
print("接收到中断信号,停止录制...")
break
except Exception as e:
print(f"录制过程中出现错误: {e}")
break
def _get_robot_state(self, timestamp: float) -> RobotState:
"""获取机器人状态(模拟数据)"""
# 这里应该连接到真实的机器人API获取数据
# 目前使用模拟数据
return RobotState(
timestamp=timestamp,
joint_positions=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
joint_velocities=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06],
joint_torques=[1.0, 1.1, 1.2, 1.3, 1.4, 1.5],
end_effector_pose=[0.5, 0.0, 0.3, 0.0, 0.0, 0.0],
gripper_state=0.5,
camera_image=np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8),
depth_image=np.random.rand(480, 640).astype(np.float32)
)
def _get_control_command(self, timestamp: float) -> Optional[ControlCommand]:
"""获取控制指令"""
if self.joystick:
# 读取手柄输入
axes = [self.joystick.get_axis(i) for i in range(self.joystick.get_numaxes())]
buttons = [self.joystick.get_button(i) for i in range(self.joystick.get_numbuttons())]
# 检查是否有显著的输入变化
if any(abs(axis) > 0.1 for axis in axes) or any(buttons):
return ControlCommand(
timestamp=timestamp,
command_type="joystick",
command_data={
"axes": axes,
"buttons": buttons
}
)
return None
def _save_data(self):
"""保存录制的数据"""
# 保存机器人状态数据到HDF5文件
hdf5_path = os.path.join(self.session_dir, "robot_data.h5")
with h5py.File(hdf5_path, 'w') as f:
# 创建数据集
num_samples = len(self.data_buffer)
# 时间戳
f.create_dataset('timestamps', data=[state.timestamp for state in self.data_buffer])
# 关节数据
joint_positions = np.array([state.joint_positions for state in self.data_buffer])
joint_velocities = np.array([state.joint_velocities for state in self.data_buffer])
joint_torques = np.array([state.joint_torques for state in self.data_buffer])
f.create_dataset('joint_positions', data=joint_positions)
f.create_dataset('joint_velocities', data=joint_velocities)
f.create_dataset('joint_torques', data=joint_torques)
# 末端执行器位姿
end_effector_poses = np.array([state.end_effector_pose for state in self.data_buffer])
f.create_dataset('end_effector_poses', data=end_effector_poses)
# 夹爪状态
gripper_states = np.array([state.gripper_state for state in self.data_buffer])
f.create_dataset('gripper_states', data=gripper_states)
# 图像数据
if self.data_buffer and self.data_buffer[0].camera_image is not None:
camera_images = np.array([state.camera_image for state in self.data_buffer])
f.create_dataset('camera_images', data=camera_images, compression='gzip')
if self.data_buffer and self.data_buffer[0].depth_image is not None:
depth_images = np.array([state.depth_image for state in self.data_buffer])
f.create_dataset('depth_images', data=depth_images, compression='gzip')
# 保存控制指令到JSON文件
commands_path = os.path.join(self.session_dir, "commands.json")
with open(commands_path, 'w') as f:
commands_data = [asdict(cmd) for cmd in self.command_buffer]
json.dump(commands_data, f, indent=2)
# 保存会话元数据
metadata = {
"session_name": self.session_name,
"start_time": self.data_buffer[0].timestamp if self.data_buffer else None,
"end_time": self.data_buffer[-1].timestamp if self.data_buffer else None,
"duration": (self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 0,
"num_samples": len(self.data_buffer),
"num_commands": len(self.command_buffer),
"recording_frequency": len(self.data_buffer) / ((self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 1)
}
metadata_path = os.path.join(self.session_dir, "metadata.json")
with open(metadata_path, 'w') as f:
json.dump(metadata, f, indent=2)
print(f"数据已保存:")
print(f" - 机器人数据: {hdf5_path}")
print(f" - 控制指令: {commands_path}")
print(f" - 元数据: {metadata_path}")
print(f" - 总样本数: {len(self.data_buffer)}")
print(f" - 总指令数: {len(self.command_buffer)}")
def main():
"""主函数 - 演示数据采集器的使用"""
collector = DataCollector()
print("=== 语音机器人训练系统 - 数据采集器 ===")
print("指令:")
print(" start <session_name> - 开始录制")
print(" stop - 停止录制")
print(" quit - 退出程序")
while True:
try:
command = input("\n请输入指令: ").strip().split()
if not command:
continue
if command[0] == "start":
session_name = command[1] if len(command) > 1 else None
collector.start_recording(session_name)
elif command[0] == "stop":
collector.stop_recording()
elif command[0] == "quit":
if collector.is_recording:
collector.stop_recording()
break
else:
print("未知指令,请重新输入")
except KeyboardInterrupt:
print("\n程序被中断")
if collector.is_recording:
collector.stop_recording()
break
except Exception as e:
print(f"发生错误: {e}")
if __name__ == "__main__":
main()

378
example_usage.py 普通文件
查看文件

@ -0,0 +1,378 @@
#!/usr/bin/env python3
"""
使用示例 - 演示语音机器人训练系统的完整使用流程
"""
import os
import sys
import time
import json
import numpy as np
from pathlib import Path
# 添加项目根目录到路径
sys.path.append(str(Path(__file__).parent.parent))
from data_collector import DataCollector, RobotState, ControlCommand
from model_trainer import VoiceRobotTrainer, TrainingConfig
from inference_engine import VoiceRobotSystem
def example_data_collection():
"""示例:数据采集流程"""
print("=== 数据采集示例 ===")
# 创建数据采集器
collector = DataCollector(output_dir="./example_data")
# 模拟数据采集过程
print("开始模拟数据采集...")
session_name = collector.start_recording("example_session")
# 模拟录制5秒钟的数据
time.sleep(5)
collector.stop_recording()
print(f"数据采集完成,会话: {session_name}")
return session_name
def example_manual_annotation(session_name):
"""示例:手动创建标注数据"""
print("=== 手动标注示例 ===")
session_dir = f"./example_data/{session_name}"
# 创建示例标注数据
annotations = [
{
"start_time": 0.0,
"end_time": 2.0,
"text": "向前移动",
"confidence": 1.0,
"annotator": "example",
"timestamp": time.time()
},
{
"start_time": 2.0,
"end_time": 4.0,
"text": "抓取物体",
"confidence": 1.0,
"annotator": "example",
"timestamp": time.time()
}
]
# 保存标注数据
annotations_path = os.path.join(session_dir, "annotations.json")
with open(annotations_path, 'w', encoding='utf-8') as f:
json.dump(annotations, f, indent=2, ensure_ascii=False)
print(f"标注数据已保存到: {annotations_path}")
# 创建训练数据
create_example_training_data(session_dir, annotations)
def create_example_training_data(session_dir, annotations):
"""创建示例训练数据"""
print("创建训练数据...")
training_data = []
for annotation in annotations:
# 模拟动作序列数据
duration = annotation["end_time"] - annotation["start_time"]
num_steps = int(duration * 50) # 50Hz采样率
# 生成模拟的机器人动作数据
joint_positions = []
end_effector_poses = []
gripper_states = []
timestamps = []
for i in range(num_steps):
t = i / 50.0
# 模拟关节位置(正弦波变化)
joint_pos = [0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
joint_positions.append(joint_pos)
# 模拟末端执行器位姿
pose = [0.5 + 0.1 * np.sin(2 * np.pi * t),
0.0 + 0.1 * np.cos(2 * np.pi * t),
0.3, 0.0, 0.0, 0.0]
end_effector_poses.append(pose)
# 模拟夹爪状态
if "抓取" in annotation["text"]:
gripper_state = min(1.0, t / duration) # 逐渐闭合
else:
gripper_state = 0.5 # 保持中间位置
gripper_states.append(gripper_state)
timestamps.append(t)
# 创建训练样本
sample = {
"text": annotation["text"],
"start_time": annotation["start_time"],
"end_time": annotation["end_time"],
"joint_positions": joint_positions,
"joint_velocities": [[0.0] * 6] * num_steps, # 简化为零速度
"end_effector_poses": end_effector_poses,
"gripper_states": gripper_states,
"timestamps": timestamps
}
training_data.append(sample)
# 保存训练数据
training_data_path = os.path.join(session_dir, "training_data.json")
with open(training_data_path, 'w', encoding='utf-8') as f:
json.dump(training_data, f, indent=2, ensure_ascii=False)
print(f"训练数据已保存到: {training_data_path}")
def example_model_training():
"""示例:模型训练流程"""
print("=== 模型训练示例 ===")
# 创建训练配置
config = TrainingConfig(
data_dir="./example_data",
output_dir="./example_models",
batch_size=2, # 小批次用于示例
num_epochs=10, # 少量epoch用于示例
learning_rate=1e-3,
early_stopping_patience=5
)
# 创建训练器
trainer = VoiceRobotTrainer(config)
try:
# 开始训练
trainer.train()
print("模型训练完成!")
return True
except Exception as e:
print(f"训练过程中出现错误: {e}")
return False
def example_inference():
"""示例:模型推理流程"""
print("=== 模型推理示例 ===")
model_path = "./example_models/best_model.pth"
if not os.path.exists(model_path):
print("模型文件不存在,请先完成训练")
return
# 创建推理系统
system = VoiceRobotSystem(model_path)
# 测试文本指令
test_commands = [
"向前移动",
"抓取物体",
"向左转动",
"放下物体"
]
print("测试文本指令:")
for command in test_commands:
print(f"\n指令: {command}")
try:
system.process_text_command(command)
except Exception as e:
print(f"处理指令失败: {e}")
print("推理测试完成!")
def example_complete_workflow():
"""示例:完整的工作流程"""
print("=== 完整工作流程示例 ===")
print("这个示例将演示从数据采集到模型部署的完整流程")
# 步骤1:数据采集
print("\n步骤1: 数据采集")
session_name = example_data_collection()
# 步骤2:数据标注
print("\n步骤2: 数据标注")
example_manual_annotation(session_name)
# 步骤3:模型训练
print("\n步骤3: 模型训练")
training_success = example_model_training()
if training_success:
# 步骤4:模型推理
print("\n步骤4: 模型推理")
example_inference()
print("\n=== 完整工作流程示例完成 ===")
def example_custom_robot_integration():
"""示例:自定义机器人集成"""
print("=== 自定义机器人集成示例 ===")
class CustomRobotController:
"""自定义机器人控制器示例"""
def __init__(self):
print("初始化自定义机器人控制器")
# 这里可以初始化机器人SDK、ROS节点等
def execute_action(self, prediction):
"""执行机器人动作"""
joint_positions = prediction['joint_positions']
end_effector_poses = prediction['end_effector_poses']
gripper_states = prediction['gripper_states']
text = prediction['text']
print(f"执行自定义机器人动作: {text}")
# 这里实现具体的机器人控制逻辑
# 例如:
# self.robot_arm.move_joints(joint_positions)
# self.robot_arm.move_to_pose(end_effector_poses)
# self.gripper.set_position(gripper_states[0])
print(f"关节目标位置: {joint_positions}")
print(f"末端执行器目标位姿: {end_effector_poses}")
print(f"夹爪目标状态: {gripper_states}")
# 使用自定义控制器
custom_controller = CustomRobotController()
# 模拟预测结果
mock_prediction = {
'joint_positions': np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6]),
'end_effector_poses': np.array([0.5, 0.0, 0.3, 0.0, 0.0, 0.0]),
'gripper_states': np.array([0.8]),
'text': '抓取红色方块'
}
custom_controller.execute_action(mock_prediction)
def example_data_analysis():
"""示例:数据分析和可视化"""
print("=== 数据分析示例 ===")
import matplotlib.pyplot as plt
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False
# 生成示例数据
time_steps = np.linspace(0, 5, 250) # 5秒,50Hz
joint_positions = np.array([
[0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
for t in time_steps
])
# 创建可视化
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
# 关节位置
axes[0, 0].plot(time_steps, joint_positions)
axes[0, 0].set_title('关节位置轨迹')
axes[0, 0].set_xlabel('时间 (秒)')
axes[0, 0].set_ylabel('位置 (弧度)')
axes[0, 0].grid(True)
axes[0, 0].legend([f'关节{i+1}' for i in range(6)])
# 末端执行器轨迹
end_effector_x = 0.5 + 0.1 * np.sin(2 * np.pi * time_steps)
end_effector_y = 0.0 + 0.1 * np.cos(2 * np.pi * time_steps)
axes[0, 1].plot(end_effector_x, end_effector_y)
axes[0, 1].set_title('末端执行器轨迹 (XY平面)')
axes[0, 1].set_xlabel('X位置 (米)')
axes[0, 1].set_ylabel('Y位置 (米)')
axes[0, 1].grid(True)
axes[0, 1].axis('equal')
# 夹爪状态
gripper_states = np.minimum(1.0, time_steps / 5.0)
axes[1, 0].plot(time_steps, gripper_states)
axes[1, 0].set_title('夹爪状态')
axes[1, 0].set_xlabel('时间 (秒)')
axes[1, 0].set_ylabel('开合程度')
axes[1, 0].grid(True)
# 速度分析
joint_velocities = np.gradient(joint_positions, axis=0) / (time_steps[1] - time_steps[0])
axes[1, 1].plot(time_steps, np.linalg.norm(joint_velocities, axis=1))
axes[1, 1].set_title('关节速度幅值')
axes[1, 1].set_xlabel('时间 (秒)')
axes[1, 1].set_ylabel('速度 (弧度/秒)')
axes[1, 1].grid(True)
plt.tight_layout()
# 保存图片
os.makedirs('./example_output', exist_ok=True)
plt.savefig('./example_output/data_analysis.png', dpi=300, bbox_inches='tight')
plt.close()
print("数据分析图表已保存到: ./example_output/data_analysis.png")
def main():
"""主函数"""
print("语音机器人训练系统 - 使用示例")
print("=" * 50)
examples = {
"1": ("完整工作流程", example_complete_workflow),
"2": ("数据采集", example_data_collection),
"3": ("模型训练", example_model_training),
"4": ("模型推理", example_inference),
"5": ("自定义机器人集成", example_custom_robot_integration),
"6": ("数据分析", example_data_analysis),
"0": ("运行所有示例", None)
}
print("可用示例:")
for key, (name, _) in examples.items():
print(f" {key}. {name}")
choice = input("\n请选择要运行的示例 (输入数字): ").strip()
if choice == "0":
# 运行所有示例
for key, (name, func) in examples.items():
if func is not None:
print(f"\n{'='*20} {name} {'='*20}")
try:
func()
except Exception as e:
print(f"示例 '{name}' 运行失败: {e}")
elif choice in examples:
name, func = examples[choice]
if func is not None:
print(f"\n运行示例: {name}")
try:
func()
except Exception as e:
print(f"示例运行失败: {e}")
else:
print("无效选择")
else:
print("无效选择")
if __name__ == "__main__":
main()

441
inference_engine.py 普通文件
查看文件

@ -0,0 +1,441 @@
#!/usr/bin/env python3
"""
推理引擎 - 用于部署训练好的语音-动作映射模型
基于LeRobot框架的语音输入法训练系统
"""
import os
import json
import torch
import numpy as np
import pickle
import speech_recognition as sr
import pyaudio
from typing import Dict, List, Optional, Any
import logging
from pathlib import Path
import time
import threading
from queue import Queue
from transformers import AutoTokenizer
from model_trainer import VoiceRobotModel, TrainingConfig
# 设置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class VoiceRobotInference:
"""语音机器人推理引擎"""
def __init__(self, model_path: str, device: str = "auto"):
"""
初始化推理引擎
Args:
model_path: 训练好的模型路径
device: 计算设备 ("auto", "cpu", "cuda")
"""
self.model_path = model_path
# 设置设备
if device == "auto":
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
self.device = torch.device(device)
logger.info(f"使用设备: {self.device}")
# 加载模型和配置
self._load_model()
# 初始化语音识别
self._init_speech_recognition()
# 语音处理队列
self.audio_queue = Queue()
self.result_queue = Queue()
self.is_listening = False
self.listen_thread = None
logger.info("推理引擎初始化完成")
def _load_model(self):
"""加载训练好的模型"""
logger.info(f"加载模型: {self.model_path}")
# 加载模型检查点
checkpoint = torch.load(self.model_path, map_location=self.device)
self.config = checkpoint['config']
# 创建模型
self.model = VoiceRobotModel(self.config)
self.model.load_state_dict(checkpoint['model_state_dict'])
self.model.to(self.device)
self.model.eval()
# 加载tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(self.config.text_encoder_model)
# 加载数据标准化器
scaler_path = os.path.join(os.path.dirname(self.model_path), "scalers.pkl")
if os.path.exists(scaler_path):
with open(scaler_path, 'rb') as f:
scalers = pickle.load(f)
self.action_scaler = scalers['action_scaler']
self.pose_scaler = scalers['pose_scaler']
self.gripper_scaler = scalers['gripper_scaler']
else:
logger.warning("未找到数据标准化器,预测结果可能不准确")
self.action_scaler = None
self.pose_scaler = None
self.gripper_scaler = None
logger.info("模型加载完成")
def _init_speech_recognition(self):
"""初始化语音识别"""
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone()
# 调整环境噪声
logger.info("调整环境噪声...")
with self.microphone as source:
self.recognizer.adjust_for_ambient_noise(source)
logger.info("语音识别初始化完成")
def predict_from_text(self, text: str) -> Dict[str, np.ndarray]:
"""
从文本预测机器人动作
Args:
text: 语音指令文本
Returns:
包含预测动作的字典
"""
logger.info(f"预测文本: {text}")
# 文本编码
encoding = self.tokenizer(
text,
truncation=True,
padding='max_length',
max_length=128,
return_tensors='pt'
)
# 移动到设备
input_ids = encoding['input_ids'].to(self.device)
attention_mask = encoding['attention_mask'].to(self.device)
# 模型推理
with torch.no_grad():
predictions = self.model(input_ids, attention_mask)
# 转换为numpy数组
joint_positions = predictions['joint_positions'].cpu().numpy()[0]
end_effector_poses = predictions['end_effector_poses'].cpu().numpy()[0]
gripper_states = predictions['gripper_states'].cpu().numpy()[0]
# 反标准化
if self.action_scaler is not None:
joint_positions = self.action_scaler.inverse_transform(joint_positions.reshape(1, -1))[0]
if self.pose_scaler is not None:
end_effector_poses = self.pose_scaler.inverse_transform(end_effector_poses.reshape(1, -1))[0]
if self.gripper_scaler is not None:
gripper_states = self.gripper_scaler.inverse_transform(gripper_states.reshape(1, -1))[0]
result = {
'joint_positions': joint_positions,
'end_effector_poses': end_effector_poses,
'gripper_states': gripper_states,
'text': text
}
logger.info(f"预测完成: 关节位置={joint_positions}, 末端位姿={end_effector_poses}, 夹爪状态={gripper_states}")
return result
def recognize_speech(self, audio_data) -> Optional[str]:
"""
识别语音
Args:
audio_data: 音频数据
Returns:
识别的文本,如果识别失败返回None
"""
try:
# 使用Google语音识别(需要网络连接)
text = self.recognizer.recognize_google(audio_data, language='zh-CN')
logger.info(f"语音识别结果: {text}")
return text
except sr.UnknownValueError:
logger.warning("无法识别语音")
return None
except sr.RequestError as e:
logger.error(f"语音识别服务错误: {e}")
return None
def start_listening(self):
"""开始监听语音"""
if self.is_listening:
logger.warning("已经在监听中...")
return
self.is_listening = True
self.listen_thread = threading.Thread(target=self._listen_loop)
self.listen_thread.start()
logger.info("开始监听语音...")
def stop_listening(self):
"""停止监听语音"""
if not self.is_listening:
logger.warning("当前没有在监听...")
return
self.is_listening = False
if self.listen_thread:
self.listen_thread.join()
logger.info("停止监听语音")
def _listen_loop(self):
"""语音监听循环"""
while self.is_listening:
try:
with self.microphone as source:
logger.info("请说话...")
# 监听音频,超时时间为1秒
audio = self.recognizer.listen(source, timeout=1, phrase_time_limit=5)
# 识别语音
text = self.recognize_speech(audio)
if text:
# 预测动作
prediction = self.predict_from_text(text)
# 将结果放入队列
self.result_queue.put(prediction)
except sr.WaitTimeoutError:
# 超时,继续监听
pass
except Exception as e:
logger.error(f"监听过程中出现错误: {e}")
time.sleep(0.1)
def get_latest_prediction(self) -> Optional[Dict]:
"""获取最新的预测结果"""
if not self.result_queue.empty():
return self.result_queue.get()
return None
def predict_from_audio_file(self, audio_file_path: str) -> Optional[Dict]:
"""
从音频文件预测动作
Args:
audio_file_path: 音频文件路径
Returns:
预测结果字典
"""
try:
with sr.AudioFile(audio_file_path) as source:
audio = self.recognizer.record(source)
text = self.recognize_speech(audio)
if text:
return self.predict_from_text(text)
else:
return None
except Exception as e:
logger.error(f"处理音频文件失败: {e}")
return None
class RobotController:
"""机器人控制器接口"""
def __init__(self):
"""初始化机器人控制器"""
# 这里应该连接到真实的机器人控制系统
# 例如:ROS节点、机器人SDK等
logger.info("机器人控制器初始化(模拟模式)")
def execute_action(self, prediction: Dict[str, np.ndarray]):
"""
执行机器人动作
Args:
prediction: 预测的动作参数
"""
joint_positions = prediction['joint_positions']
end_effector_poses = prediction['end_effector_poses']
gripper_states = prediction['gripper_states']
text = prediction['text']
logger.info(f"执行动作指令: {text}")
logger.info(f"关节位置: {joint_positions}")
logger.info(f"末端位姿: {end_effector_poses}")
logger.info(f"夹爪状态: {gripper_states}")
# 这里应该调用真实的机器人控制API
# 例如:
# self.robot_arm.move_to_joint_positions(joint_positions)
# self.robot_arm.move_to_pose(end_effector_poses)
# self.gripper.set_position(gripper_states[0])
# 模拟执行时间
time.sleep(1.0)
logger.info("动作执行完成")
class VoiceRobotSystem:
"""完整的语音机器人系统"""
def __init__(self, model_path: str):
"""
初始化语音机器人系统
Args:
model_path: 训练好的模型路径
"""
self.inference_engine = VoiceRobotInference(model_path)
self.robot_controller = RobotController()
self.is_running = False
self.main_thread = None
def start(self):
"""启动系统"""
if self.is_running:
logger.warning("系统已经在运行中...")
return
self.is_running = True
self.inference_engine.start_listening()
self.main_thread = threading.Thread(target=self._main_loop)
self.main_thread.start()
logger.info("语音机器人系统启动")
def stop(self):
"""停止系统"""
if not self.is_running:
logger.warning("系统当前没有运行...")
return
self.is_running = False
self.inference_engine.stop_listening()
if self.main_thread:
self.main_thread.join()
logger.info("语音机器人系统停止")
def _main_loop(self):
"""主循环"""
while self.is_running:
# 获取最新的预测结果
prediction = self.inference_engine.get_latest_prediction()
if prediction:
# 执行机器人动作
self.robot_controller.execute_action(prediction)
time.sleep(0.1) # 控制循环频率
def process_text_command(self, text: str):
"""处理文本指令"""
prediction = self.inference_engine.predict_from_text(text)
self.robot_controller.execute_action(prediction)
def process_audio_file(self, audio_file_path: str):
"""处理音频文件"""
prediction = self.inference_engine.predict_from_audio_file(audio_file_path)
if prediction:
self.robot_controller.execute_action(prediction)
else:
logger.warning("无法从音频文件中识别指令")
def main():
"""主函数 - 演示推理引擎的使用"""
import argparse
parser = argparse.ArgumentParser(description="语音机器人推理引擎")
parser.add_argument("--model", required=True, help="模型文件路径")
parser.add_argument("--mode", choices=["interactive", "text", "audio"], default="interactive", help="运行模式")
parser.add_argument("--text", help="文本指令(text模式)")
parser.add_argument("--audio", help="音频文件路径(audio模式)")
args = parser.parse_args()
if not os.path.exists(args.model):
logger.error(f"模型文件不存在: {args.model}")
return
# 创建系统
system = VoiceRobotSystem(args.model)
try:
if args.mode == "interactive":
# 交互模式
system.start()
print("=== 语音机器人系统 ===")
print("系统正在监听语音指令...")
print("输入 'quit' 退出程序")
while True:
command = input("\n输入文本指令 (或 'quit' 退出): ").strip()
if command.lower() == 'quit':
break
elif command:
system.process_text_command(command)
system.stop()
elif args.mode == "text":
# 文本模式
if not args.text:
logger.error("文本模式需要提供 --text 参数")
return
system.process_text_command(args.text)
elif args.mode == "audio":
# 音频模式
if not args.audio:
logger.error("音频模式需要提供 --audio 参数")
return
if not os.path.exists(args.audio):
logger.error(f"音频文件不存在: {args.audio}")
return
system.process_audio_file(args.audio)
except KeyboardInterrupt:
logger.info("程序被用户中断")
system.stop()
except Exception as e:
logger.error(f"程序运行出错: {e}")
system.stop()
if __name__ == "__main__":
main()

619
model_trainer.py 普通文件
查看文件

@ -0,0 +1,619 @@
#!/usr/bin/env python3
"""
模型训练管道 - 基于LeRobot框架训练语音-动作映射模型
基于LeRobot框架的语音输入法训练系统
"""
import os
import json
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import numpy as np
import h5py
from typing import Dict, List, Tuple, Optional, Any
from dataclasses import dataclass
import logging
from pathlib import Path
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import pickle
import time
from transformers import AutoTokenizer, AutoModel
# 设置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@dataclass
class TrainingConfig:
"""训练配置"""
# 数据相关
data_dir: str = "./collected_data"
output_dir: str = "./trained_models"
# 模型相关
text_encoder_model: str = "bert-base-chinese" # 中文BERT模型
hidden_dim: int = 256
action_dim: int = 6 # 关节数量
pose_dim: int = 6 # 末端执行器位姿维度
gripper_dim: int = 1 # 夹爪状态维度
# 训练相关
batch_size: int = 32
learning_rate: float = 1e-4
num_epochs: int = 100
validation_split: float = 0.2
early_stopping_patience: int = 10
# 设备
device: str = "cuda" if torch.cuda.is_available() else "cpu"
class VoiceRobotDataset(Dataset):
"""语音机器人数据集"""
def __init__(self, data_samples: List[Dict], tokenizer, scaler=None, max_length=128):
self.data_samples = data_samples
self.tokenizer = tokenizer
self.scaler = scaler
self.max_length = max_length
# 如果没有提供scaler,创建一个新的
if self.scaler is None:
self._fit_scaler()
def _fit_scaler(self):
"""拟合数据标准化器"""
all_actions = []
all_poses = []
all_grippers = []
for sample in self.data_samples:
# 合并所有时间步的数据
joint_positions = np.array(sample['joint_positions'])
end_effector_poses = np.array(sample['end_effector_poses'])
gripper_states = np.array(sample['gripper_states'])
all_actions.extend(joint_positions)
all_poses.extend(end_effector_poses)
all_grippers.extend(gripper_states.reshape(-1, 1))
# 创建标准化器
self.action_scaler = StandardScaler()
self.pose_scaler = StandardScaler()
self.gripper_scaler = StandardScaler()
self.action_scaler.fit(all_actions)
self.pose_scaler.fit(all_poses)
self.gripper_scaler.fit(all_grippers)
def __len__(self):
return len(self.data_samples)
def __getitem__(self, idx):
sample = self.data_samples[idx]
# 文本编码
text = sample['text']
encoding = self.tokenizer(
text,
truncation=True,
padding='max_length',
max_length=self.max_length,
return_tensors='pt'
)
# 动作数据处理
joint_positions = np.array(sample['joint_positions'])
end_effector_poses = np.array(sample['end_effector_poses'])
gripper_states = np.array(sample['gripper_states']).reshape(-1, 1)
# 数据标准化
joint_positions_norm = self.action_scaler.transform(joint_positions)
end_effector_poses_norm = self.pose_scaler.transform(end_effector_poses)
gripper_states_norm = self.gripper_scaler.transform(gripper_states)
# 转换为张量
joint_positions_tensor = torch.FloatTensor(joint_positions_norm)
end_effector_poses_tensor = torch.FloatTensor(end_effector_poses_norm)
gripper_states_tensor = torch.FloatTensor(gripper_states_norm)
return {
'input_ids': encoding['input_ids'].squeeze(),
'attention_mask': encoding['attention_mask'].squeeze(),
'joint_positions': joint_positions_tensor,
'end_effector_poses': end_effector_poses_tensor,
'gripper_states': gripper_states_tensor,
'text': text
}
class VoiceRobotModel(nn.Module):
"""语音机器人模型"""
def __init__(self, config: TrainingConfig):
super().__init__()
self.config = config
# 文本编码器
self.text_encoder = AutoModel.from_pretrained(config.text_encoder_model)
text_hidden_size = self.text_encoder.config.hidden_size
# 冻结文本编码器的部分层(可选)
# for param in self.text_encoder.parameters():
# param.requires_grad = False
# 动作解码器
self.action_decoder = nn.Sequential(
nn.Linear(text_hidden_size, config.hidden_dim),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(config.hidden_dim, config.hidden_dim),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(config.hidden_dim, config.action_dim)
)
# 位姿解码器
self.pose_decoder = nn.Sequential(
nn.Linear(text_hidden_size, config.hidden_dim),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(config.hidden_dim, config.hidden_dim),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(config.hidden_dim, config.pose_dim)
)
# 夹爪解码器
self.gripper_decoder = nn.Sequential(
nn.Linear(text_hidden_size, config.hidden_dim // 2),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(config.hidden_dim // 2, config.gripper_dim),
nn.Sigmoid() # 夹爪状态通常在0-1之间
)
def forward(self, input_ids, attention_mask):
# 文本编码
text_outputs = self.text_encoder(
input_ids=input_ids,
attention_mask=attention_mask
)
# 使用[CLS] token的表示
text_features = text_outputs.last_hidden_state[:, 0, :] # [batch_size, hidden_size]
# 解码动作
joint_positions = self.action_decoder(text_features)
end_effector_poses = self.pose_decoder(text_features)
gripper_states = self.gripper_decoder(text_features)
return {
'joint_positions': joint_positions,
'end_effector_poses': end_effector_poses,
'gripper_states': gripper_states
}
class VoiceRobotTrainer:
"""语音机器人训练器"""
def __init__(self, config: TrainingConfig):
self.config = config
self.device = torch.device(config.device)
# 创建输出目录
os.makedirs(config.output_dir, exist_ok=True)
# 初始化tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(config.text_encoder_model)
# 训练历史
self.train_history = {
'train_loss': [],
'val_loss': [],
'joint_loss': [],
'pose_loss': [],
'gripper_loss': []
}
def load_data(self) -> Tuple[List[Dict], List[Dict]]:
"""加载训练数据"""
logger.info("加载训练数据...")
all_samples = []
# 遍历数据目录中的所有会话
data_path = Path(self.config.data_dir)
for session_dir in data_path.iterdir():
if session_dir.is_dir():
training_data_path = session_dir / "training_data.json"
if training_data_path.exists():
with open(training_data_path, 'r', encoding='utf-8') as f:
session_data = json.load(f)
all_samples.extend(session_data)
if not all_samples:
raise ValueError("未找到训练数据,请先运行数据采集和标注")
logger.info(f"加载了 {len(all_samples)} 个训练样本")
# 划分训练集和验证集
train_samples, val_samples = train_test_split(
all_samples,
test_size=self.config.validation_split,
random_state=42
)
logger.info(f"训练集: {len(train_samples)} 样本")
logger.info(f"验证集: {len(val_samples)} 样本")
return train_samples, val_samples
def create_datasets(self, train_samples: List[Dict], val_samples: List[Dict]) -> Tuple[VoiceRobotDataset, VoiceRobotDataset]:
"""创建数据集"""
logger.info("创建数据集...")
# 创建训练数据集
train_dataset = VoiceRobotDataset(train_samples, self.tokenizer)
# 创建验证数据集(使用训练集的scaler)
val_dataset = VoiceRobotDataset(
val_samples,
self.tokenizer,
scaler={
'action_scaler': train_dataset.action_scaler,
'pose_scaler': train_dataset.pose_scaler,
'gripper_scaler': train_dataset.gripper_scaler
}
)
val_dataset.action_scaler = train_dataset.action_scaler
val_dataset.pose_scaler = train_dataset.pose_scaler
val_dataset.gripper_scaler = train_dataset.gripper_scaler
# 保存scaler
scaler_path = os.path.join(self.config.output_dir, "scalers.pkl")
with open(scaler_path, 'wb') as f:
pickle.dump({
'action_scaler': train_dataset.action_scaler,
'pose_scaler': train_dataset.pose_scaler,
'gripper_scaler': train_dataset.gripper_scaler
}, f)
return train_dataset, val_dataset
def create_model(self) -> VoiceRobotModel:
"""创建模型"""
logger.info("创建模型...")
model = VoiceRobotModel(self.config)
model.to(self.device)
# 打印模型信息
total_params = sum(p.numel() for p in model.parameters())
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
logger.info(f"模型参数总数: {total_params:,}")
logger.info(f"可训练参数: {trainable_params:,}")
return model
def compute_loss(self, predictions: Dict, targets: Dict) -> Tuple[torch.Tensor, Dict]:
"""计算损失"""
# 序列损失函数
mse_loss = nn.MSELoss()
# 计算各部分损失
joint_loss = 0
pose_loss = 0
gripper_loss = 0
batch_size = predictions['joint_positions'].size(0)
for i in range(batch_size):
# 关节位置损失
pred_joints = predictions['joint_positions'][i] # [action_dim]
target_joints = targets['joint_positions'][i] # [seq_len, action_dim]
# 使用序列的平均值作为目标
target_joints_mean = target_joints.mean(dim=0)
joint_loss += mse_loss(pred_joints, target_joints_mean)
# 末端执行器位姿损失
pred_pose = predictions['end_effector_poses'][i] # [pose_dim]
target_pose = targets['end_effector_poses'][i] # [seq_len, pose_dim]
target_pose_mean = target_pose.mean(dim=0)
pose_loss += mse_loss(pred_pose, target_pose_mean)
# 夹爪状态损失
pred_gripper = predictions['gripper_states'][i] # [gripper_dim]
target_gripper = targets['gripper_states'][i] # [seq_len, gripper_dim]
target_gripper_mean = target_gripper.mean(dim=0)
gripper_loss += mse_loss(pred_gripper, target_gripper_mean)
# 平均损失
joint_loss /= batch_size
pose_loss /= batch_size
gripper_loss /= batch_size
# 总损失(加权)
total_loss = joint_loss + pose_loss + gripper_loss * 0.5
loss_dict = {
'total_loss': total_loss,
'joint_loss': joint_loss,
'pose_loss': pose_loss,
'gripper_loss': gripper_loss
}
return total_loss, loss_dict
def train_epoch(self, model: VoiceRobotModel, dataloader: DataLoader, optimizer: optim.Optimizer) -> Dict:
"""训练一个epoch"""
model.train()
total_loss = 0
total_joint_loss = 0
total_pose_loss = 0
total_gripper_loss = 0
num_batches = 0
for batch in dataloader:
# 移动数据到设备
input_ids = batch['input_ids'].to(self.device)
attention_mask = batch['attention_mask'].to(self.device)
targets = {
'joint_positions': batch['joint_positions'].to(self.device),
'end_effector_poses': batch['end_effector_poses'].to(self.device),
'gripper_states': batch['gripper_states'].to(self.device)
}
# 前向传播
optimizer.zero_grad()
predictions = model(input_ids, attention_mask)
# 计算损失
loss, loss_dict = self.compute_loss(predictions, targets)
# 反向传播
loss.backward()
optimizer.step()
# 累计损失
total_loss += loss.item()
total_joint_loss += loss_dict['joint_loss'].item()
total_pose_loss += loss_dict['pose_loss'].item()
total_gripper_loss += loss_dict['gripper_loss'].item()
num_batches += 1
return {
'train_loss': total_loss / num_batches,
'joint_loss': total_joint_loss / num_batches,
'pose_loss': total_pose_loss / num_batches,
'gripper_loss': total_gripper_loss / num_batches
}
def validate_epoch(self, model: VoiceRobotModel, dataloader: DataLoader) -> Dict:
"""验证一个epoch"""
model.eval()
total_loss = 0
total_joint_loss = 0
total_pose_loss = 0
total_gripper_loss = 0
num_batches = 0
with torch.no_grad():
for batch in dataloader:
# 移动数据到设备
input_ids = batch['input_ids'].to(self.device)
attention_mask = batch['attention_mask'].to(self.device)
targets = {
'joint_positions': batch['joint_positions'].to(self.device),
'end_effector_poses': batch['end_effector_poses'].to(self.device),
'gripper_states': batch['gripper_states'].to(self.device)
}
# 前向传播
predictions = model(input_ids, attention_mask)
# 计算损失
loss, loss_dict = self.compute_loss(predictions, targets)
# 累计损失
total_loss += loss.item()
total_joint_loss += loss_dict['joint_loss'].item()
total_pose_loss += loss_dict['pose_loss'].item()
total_gripper_loss += loss_dict['gripper_loss'].item()
num_batches += 1
return {
'val_loss': total_loss / num_batches,
'joint_loss': total_joint_loss / num_batches,
'pose_loss': total_pose_loss / num_batches,
'gripper_loss': total_gripper_loss / num_batches
}
def train(self):
"""主训练函数"""
logger.info("开始训练...")
# 加载数据
train_samples, val_samples = self.load_data()
train_dataset, val_dataset = self.create_datasets(train_samples, val_samples)
# 创建数据加载器
train_dataloader = DataLoader(
train_dataset,
batch_size=self.config.batch_size,
shuffle=True,
num_workers=0 # 在Windows上设置为0
)
val_dataloader = DataLoader(
val_dataset,
batch_size=self.config.batch_size,
shuffle=False,
num_workers=0
)
# 创建模型
model = self.create_model()
# 创建优化器
optimizer = optim.Adam(model.parameters(), lr=self.config.learning_rate)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5)
# 早停机制
best_val_loss = float('inf')
patience_counter = 0
# 训练循环
for epoch in range(self.config.num_epochs):
start_time = time.time()
# 训练
train_metrics = self.train_epoch(model, train_dataloader, optimizer)
# 验证
val_metrics = self.validate_epoch(model, val_dataloader)
# 更新学习率
scheduler.step(val_metrics['val_loss'])
# 记录历史
self.train_history['train_loss'].append(train_metrics['train_loss'])
self.train_history['val_loss'].append(val_metrics['val_loss'])
self.train_history['joint_loss'].append(train_metrics['joint_loss'])
self.train_history['pose_loss'].append(train_metrics['pose_loss'])
self.train_history['gripper_loss'].append(train_metrics['gripper_loss'])
# 打印进度
epoch_time = time.time() - start_time
logger.info(
f"Epoch {epoch+1}/{self.config.num_epochs} - "
f"Train Loss: {train_metrics['train_loss']:.4f}, "
f"Val Loss: {val_metrics['val_loss']:.4f}, "
f"Time: {epoch_time:.2f}s"
)
# 保存最佳模型
if val_metrics['val_loss'] < best_val_loss:
best_val_loss = val_metrics['val_loss']
patience_counter = 0
# 保存模型
model_path = os.path.join(self.config.output_dir, "best_model.pth")
torch.save({
'model_state_dict': model.state_dict(),
'config': self.config,
'train_history': self.train_history,
'epoch': epoch
}, model_path)
logger.info(f"保存最佳模型到 {model_path}")
else:
patience_counter += 1
# 早停检查
if patience_counter >= self.config.early_stopping_patience:
logger.info(f"早停触发,在第 {epoch+1} 轮停止训练")
break
# 保存最终模型
final_model_path = os.path.join(self.config.output_dir, "final_model.pth")
torch.save({
'model_state_dict': model.state_dict(),
'config': self.config,
'train_history': self.train_history,
'epoch': epoch
}, final_model_path)
# 保存训练历史
history_path = os.path.join(self.config.output_dir, "training_history.json")
with open(history_path, 'w') as f:
json.dump(self.train_history, f, indent=2)
# 绘制训练曲线
self.plot_training_curves()
logger.info("训练完成!")
def plot_training_curves(self):
"""绘制训练曲线"""
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans'] # 支持中文显示
plt.rcParams['axes.unicode_minus'] = False
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
# 总损失
axes[0, 0].plot(self.train_history['train_loss'], label='训练损失')
axes[0, 0].plot(self.train_history['val_loss'], label='验证损失')
axes[0, 0].set_title('总损失')
axes[0, 0].set_xlabel('Epoch')
axes[0, 0].set_ylabel('Loss')
axes[0, 0].legend()
axes[0, 0].grid(True)
# 关节损失
axes[0, 1].plot(self.train_history['joint_loss'], label='关节损失')
axes[0, 1].set_title('关节位置损失')
axes[0, 1].set_xlabel('Epoch')
axes[0, 1].set_ylabel('Loss')
axes[0, 1].legend()
axes[0, 1].grid(True)
# 位姿损失
axes[1, 0].plot(self.train_history['pose_loss'], label='位姿损失')
axes[1, 0].set_title('末端执行器位姿损失')
axes[1, 0].set_xlabel('Epoch')
axes[1, 0].set_ylabel('Loss')
axes[1, 0].legend()
axes[1, 0].grid(True)
# 夹爪损失
axes[1, 1].plot(self.train_history['gripper_loss'], label='夹爪损失')
axes[1, 1].set_title('夹爪状态损失')
axes[1, 1].set_xlabel('Epoch')
axes[1, 1].set_ylabel('Loss')
axes[1, 1].legend()
axes[1, 1].grid(True)
plt.tight_layout()
# 保存图片
plot_path = os.path.join(self.config.output_dir, "training_curves.png")
plt.savefig(plot_path, dpi=300, bbox_inches='tight')
plt.close()
logger.info(f"训练曲线已保存到 {plot_path}")
def main():
"""主函数"""
# 创建配置
config = TrainingConfig(
data_dir="./collected_data",
output_dir="./trained_models",
batch_size=16, # 根据GPU内存调整
num_epochs=50,
learning_rate=1e-4
)
# 创建训练器
trainer = VoiceRobotTrainer(config)
# 开始训练
try:
trainer.train()
except Exception as e:
logger.error(f"训练过程中出现错误: {e}")
raise
if __name__ == "__main__":
main()

43
requirements.txt 普通文件
查看文件

@ -0,0 +1,43 @@
# 语音机器人训练系统依赖包
# 核心框架
torch>=1.9.0
numpy>=1.21.0
h5py>=3.1.0
# LeRobot框架
lerobot
# 数据处理
pandas>=1.3.0
scipy>=1.7.0
opencv-python>=4.5.0
pillow>=8.3.0
# 语音识别
SpeechRecognition>=3.8.1
pyaudio>=0.2.11
pydub>=0.25.1
# 游戏手柄控制
pygame>=2.0.0
# GUI和可视化
matplotlib>=3.4.0
# Web框架(用于标注工具的Web版本)
flask>=2.0.0
flask-cors>=3.0.10
# 机器学习相关
scikit-learn>=1.0.0
transformers>=4.12.0
# 数据序列化
jsonschema>=3.2.0
# 开发工具
pytest>=6.2.0
black>=21.0.0
flake8>=3.9.0

93
setup.py 普通文件
查看文件

@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""
语音输入法机器人训练系统安装脚本
"""
from setuptools import setup, find_packages
import os
# 读取README文件
def read_readme():
with open("README.md", "r", encoding="utf-8") as fh:
return fh.read()
# 读取requirements文件
def read_requirements():
with open("requirements.txt", "r", encoding="utf-8") as fh:
return [line.strip() for line in fh if line.strip() and not line.startswith("#")]
setup(
name="voice-robot-training-system",
version="1.0.0",
author="Manus AI",
author_email="contact@manus.ai",
description="基于LeRobot框架的语音输入法机器人训练系统",
long_description=read_readme(),
long_description_content_type="text/markdown",
url="https://github.com/manus-ai/voice-robot-training-system",
packages=find_packages(),
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
"Topic :: Software Development :: Libraries :: Python Modules",
],
python_requires=">=3.8",
install_requires=read_requirements(),
extras_require={
"dev": [
"pytest>=6.2.0",
"black>=21.0.0",
"flake8>=3.9.0",
"mypy>=0.910",
"pre-commit>=2.15.0",
],
"docs": [
"sphinx>=4.0.0",
"sphinx-rtd-theme>=0.5.0",
"myst-parser>=0.15.0",
],
"gpu": [
"torch[cuda]>=1.9.0",
],
},
entry_points={
"console_scripts": [
"voice-robot-collect=data_collector:main",
"voice-robot-annotate=annotation_tool:main",
"voice-robot-train=model_trainer:main",
"voice-robot-infer=inference_engine:main",
"voice-robot-example=examples.example_usage:main",
],
},
include_package_data=True,
package_data={
"": ["*.md", "*.txt", "*.yml", "*.yaml", "*.json"],
},
zip_safe=False,
keywords=[
"robotics",
"voice-control",
"machine-learning",
"deep-learning",
"lerobot",
"speech-recognition",
"human-robot-interaction",
"imitation-learning",
],
project_urls={
"Bug Reports": "https://github.com/manus-ai/voice-robot-training-system/issues",
"Source": "https://github.com/manus-ai/voice-robot-training-system",
"Documentation": "https://voice-robot-training-system.readthedocs.io/",
},
)