ToDo
这个提交包含在:
10
TODO.MD
普通文件
10
TODO.MD
普通文件
@ -0,0 +1,10 @@
|
||||
# ToDos
|
||||
## data_controller
|
||||
- 给手臂驱动加上真实的控制机器人的部分
|
||||
- _get_robot_state 按照机器人实际情况改
|
||||
|
||||
## trainer
|
||||
- 不能训练平均值,需要生成完整的动作序列
|
||||
|
||||
## microphone
|
||||
- 移除语言识别部分
|
||||
518
annotation_tool.py
普通文件
518
annotation_tool.py
普通文件
@ -0,0 +1,518 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据标注工具 - 用于为录制的机器人动作数据添加语音指令标注
|
||||
基于LeRobot框架的语音输入法训练系统
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import tkinter as tk
|
||||
from tkinter import ttk, filedialog, messagebox
|
||||
import h5py
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
|
||||
from matplotlib.figure import Figure
|
||||
import cv2
|
||||
from PIL import Image, ImageTk
|
||||
from typing import Dict, List, Optional, Any
|
||||
from dataclasses import dataclass, asdict
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
@dataclass
|
||||
class Annotation:
|
||||
"""标注数据结构"""
|
||||
start_time: float
|
||||
end_time: float
|
||||
text: str
|
||||
confidence: float = 1.0
|
||||
annotator: str = "default"
|
||||
timestamp: float = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.timestamp is None:
|
||||
self.timestamp = time.time()
|
||||
|
||||
|
||||
class AnnotationTool:
|
||||
"""数据标注工具GUI"""
|
||||
|
||||
def __init__(self):
|
||||
self.root = tk.Tk()
|
||||
self.root.title("语音机器人训练系统 - 数据标注工具")
|
||||
self.root.geometry("1200x800")
|
||||
|
||||
# 数据相关变量
|
||||
self.session_dir = None
|
||||
self.robot_data = None
|
||||
self.metadata = None
|
||||
self.annotations = []
|
||||
self.current_time = 0.0
|
||||
self.total_time = 0.0
|
||||
self.playing = False
|
||||
self.play_thread = None
|
||||
|
||||
# GUI组件
|
||||
self.setup_gui()
|
||||
|
||||
# 绑定键盘事件
|
||||
self.root.bind('<Key>', self.on_key_press)
|
||||
self.root.focus_set()
|
||||
|
||||
def setup_gui(self):
|
||||
"""设置GUI界面"""
|
||||
# 主框架
|
||||
main_frame = ttk.Frame(self.root)
|
||||
main_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
|
||||
|
||||
# 顶部控制面板
|
||||
control_frame = ttk.Frame(main_frame)
|
||||
control_frame.pack(fill=tk.X, pady=(0, 10))
|
||||
|
||||
# 文件操作按钮
|
||||
ttk.Button(control_frame, text="加载数据", command=self.load_data).pack(side=tk.LEFT, padx=(0, 5))
|
||||
ttk.Button(control_frame, text="保存标注", command=self.save_annotations).pack(side=tk.LEFT, padx=(0, 5))
|
||||
ttk.Button(control_frame, text="导出数据", command=self.export_data).pack(side=tk.LEFT, padx=(0, 5))
|
||||
|
||||
# 播放控制按钮
|
||||
ttk.Separator(control_frame, orient=tk.VERTICAL).pack(side=tk.LEFT, padx=10, fill=tk.Y)
|
||||
ttk.Button(control_frame, text="播放/暂停", command=self.toggle_play).pack(side=tk.LEFT, padx=(0, 5))
|
||||
ttk.Button(control_frame, text="重置", command=self.reset_playback).pack(side=tk.LEFT, padx=(0, 5))
|
||||
|
||||
# 时间显示
|
||||
self.time_label = ttk.Label(control_frame, text="时间: 0.00 / 0.00 秒")
|
||||
self.time_label.pack(side=tk.RIGHT)
|
||||
|
||||
# 中间内容区域
|
||||
content_frame = ttk.Frame(main_frame)
|
||||
content_frame.pack(fill=tk.BOTH, expand=True)
|
||||
|
||||
# 左侧:数据可视化
|
||||
left_frame = ttk.LabelFrame(content_frame, text="数据可视化")
|
||||
left_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=(0, 5))
|
||||
|
||||
# 创建matplotlib图表
|
||||
self.fig = Figure(figsize=(8, 6), dpi=100)
|
||||
self.canvas = FigureCanvasTkAgg(self.fig, left_frame)
|
||||
self.canvas.get_tk_widget().pack(fill=tk.BOTH, expand=True)
|
||||
|
||||
# 时间滑块
|
||||
self.time_scale = ttk.Scale(left_frame, from_=0, to=100, orient=tk.HORIZONTAL, command=self.on_time_change)
|
||||
self.time_scale.pack(fill=tk.X, pady=5)
|
||||
|
||||
# 右侧:标注面板
|
||||
right_frame = ttk.LabelFrame(content_frame, text="标注面板", width=300)
|
||||
right_frame.pack(side=tk.RIGHT, fill=tk.Y, padx=(5, 0))
|
||||
right_frame.pack_propagate(False)
|
||||
|
||||
# 当前标注输入
|
||||
ttk.Label(right_frame, text="语音指令:").pack(anchor=tk.W, pady=(10, 0))
|
||||
self.text_entry = tk.Text(right_frame, height=3, wrap=tk.WORD)
|
||||
self.text_entry.pack(fill=tk.X, pady=5)
|
||||
|
||||
# 时间范围设置
|
||||
time_frame = ttk.Frame(right_frame)
|
||||
time_frame.pack(fill=tk.X, pady=5)
|
||||
|
||||
ttk.Label(time_frame, text="开始时间:").grid(row=0, column=0, sticky=tk.W)
|
||||
self.start_time_var = tk.StringVar(value="0.00")
|
||||
ttk.Entry(time_frame, textvariable=self.start_time_var, width=10).grid(row=0, column=1, padx=(5, 0))
|
||||
|
||||
ttk.Label(time_frame, text="结束时间:").grid(row=1, column=0, sticky=tk.W)
|
||||
self.end_time_var = tk.StringVar(value="0.00")
|
||||
ttk.Entry(time_frame, textvariable=self.end_time_var, width=10).grid(row=1, column=1, padx=(5, 0))
|
||||
|
||||
# 标注操作按钮
|
||||
button_frame = ttk.Frame(right_frame)
|
||||
button_frame.pack(fill=tk.X, pady=10)
|
||||
|
||||
ttk.Button(button_frame, text="设置开始", command=self.set_start_time).pack(side=tk.LEFT, padx=(0, 5))
|
||||
ttk.Button(button_frame, text="设置结束", command=self.set_end_time).pack(side=tk.LEFT)
|
||||
|
||||
ttk.Button(right_frame, text="添加标注", command=self.add_annotation).pack(fill=tk.X, pady=5)
|
||||
ttk.Button(right_frame, text="删除选中", command=self.delete_annotation).pack(fill=tk.X, pady=5)
|
||||
|
||||
# 标注列表
|
||||
ttk.Label(right_frame, text="已有标注:").pack(anchor=tk.W, pady=(10, 0))
|
||||
|
||||
# 创建Treeview来显示标注
|
||||
columns = ('时间范围', '文本')
|
||||
self.annotation_tree = ttk.Treeview(right_frame, columns=columns, show='headings', height=8)
|
||||
|
||||
self.annotation_tree.heading('时间范围', text='时间范围')
|
||||
self.annotation_tree.heading('文本', text='语音指令')
|
||||
|
||||
self.annotation_tree.column('时间范围', width=100)
|
||||
self.annotation_tree.column('文本', width=180)
|
||||
|
||||
self.annotation_tree.pack(fill=tk.BOTH, expand=True, pady=5)
|
||||
|
||||
# 绑定选择事件
|
||||
self.annotation_tree.bind('<<TreeviewSelect>>', self.on_annotation_select)
|
||||
|
||||
# 底部状态栏
|
||||
self.status_label = ttk.Label(main_frame, text="请加载数据文件开始标注")
|
||||
self.status_label.pack(fill=tk.X, pady=(10, 0))
|
||||
|
||||
def load_data(self):
|
||||
"""加载数据文件"""
|
||||
session_dir = filedialog.askdirectory(title="选择数据会话目录")
|
||||
if not session_dir:
|
||||
return
|
||||
|
||||
try:
|
||||
# 检查必要文件
|
||||
robot_data_path = os.path.join(session_dir, "robot_data.h5")
|
||||
metadata_path = os.path.join(session_dir, "metadata.json")
|
||||
|
||||
if not os.path.exists(robot_data_path):
|
||||
messagebox.showerror("错误", "未找到机器人数据文件 (robot_data.h5)")
|
||||
return
|
||||
|
||||
if not os.path.exists(metadata_path):
|
||||
messagebox.showerror("错误", "未找到元数据文件 (metadata.json)")
|
||||
return
|
||||
|
||||
# 加载数据
|
||||
self.session_dir = session_dir
|
||||
|
||||
# 加载HDF5数据
|
||||
self.robot_data = h5py.File(robot_data_path, 'r')
|
||||
|
||||
# 加载元数据
|
||||
with open(metadata_path, 'r') as f:
|
||||
self.metadata = json.load(f)
|
||||
|
||||
# 设置时间范围
|
||||
timestamps = self.robot_data['timestamps'][:]
|
||||
self.total_time = timestamps[-1] - timestamps[0]
|
||||
self.time_scale.configure(to=self.total_time)
|
||||
|
||||
# 加载已有标注(如果存在)
|
||||
self.load_existing_annotations()
|
||||
|
||||
# 更新显示
|
||||
self.update_visualization()
|
||||
self.update_status(f"已加载数据: {os.path.basename(session_dir)}")
|
||||
|
||||
except Exception as e:
|
||||
messagebox.showerror("错误", f"加载数据失败: {str(e)}")
|
||||
|
||||
def load_existing_annotations(self):
|
||||
"""加载已有的标注数据"""
|
||||
annotations_path = os.path.join(self.session_dir, "annotations.json")
|
||||
if os.path.exists(annotations_path):
|
||||
try:
|
||||
with open(annotations_path, 'r', encoding='utf-8') as f:
|
||||
annotations_data = json.load(f)
|
||||
|
||||
self.annotations = []
|
||||
for ann_data in annotations_data:
|
||||
annotation = Annotation(**ann_data)
|
||||
self.annotations.append(annotation)
|
||||
|
||||
self.update_annotation_list()
|
||||
self.update_status(f"加载了 {len(self.annotations)} 个已有标注")
|
||||
|
||||
except Exception as e:
|
||||
print(f"加载标注文件失败: {e}")
|
||||
|
||||
def save_annotations(self):
|
||||
"""保存标注数据"""
|
||||
if not self.session_dir:
|
||||
messagebox.showwarning("警告", "请先加载数据")
|
||||
return
|
||||
|
||||
try:
|
||||
annotations_path = os.path.join(self.session_dir, "annotations.json")
|
||||
annotations_data = [asdict(ann) for ann in self.annotations]
|
||||
|
||||
with open(annotations_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(annotations_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
self.update_status(f"已保存 {len(self.annotations)} 个标注到 {annotations_path}")
|
||||
messagebox.showinfo("成功", "标注数据已保存")
|
||||
|
||||
except Exception as e:
|
||||
messagebox.showerror("错误", f"保存标注失败: {str(e)}")
|
||||
|
||||
def export_data(self):
|
||||
"""导出训练数据"""
|
||||
if not self.session_dir or not self.annotations:
|
||||
messagebox.showwarning("警告", "请先加载数据并添加标注")
|
||||
return
|
||||
|
||||
try:
|
||||
export_path = os.path.join(self.session_dir, "training_data.json")
|
||||
|
||||
# 准备训练数据格式
|
||||
training_data = []
|
||||
timestamps = self.robot_data['timestamps'][:]
|
||||
start_timestamp = timestamps[0]
|
||||
|
||||
for annotation in self.annotations:
|
||||
# 找到对应时间范围的数据索引
|
||||
start_idx = np.searchsorted(timestamps, start_timestamp + annotation.start_time)
|
||||
end_idx = np.searchsorted(timestamps, start_timestamp + annotation.end_time)
|
||||
|
||||
# 提取对应的机器人状态数据
|
||||
segment_data = {
|
||||
"text": annotation.text,
|
||||
"start_time": annotation.start_time,
|
||||
"end_time": annotation.end_time,
|
||||
"joint_positions": self.robot_data['joint_positions'][start_idx:end_idx].tolist(),
|
||||
"joint_velocities": self.robot_data['joint_velocities'][start_idx:end_idx].tolist(),
|
||||
"end_effector_poses": self.robot_data['end_effector_poses'][start_idx:end_idx].tolist(),
|
||||
"gripper_states": self.robot_data['gripper_states'][start_idx:end_idx].tolist(),
|
||||
"timestamps": (timestamps[start_idx:end_idx] - start_timestamp).tolist()
|
||||
}
|
||||
|
||||
training_data.append(segment_data)
|
||||
|
||||
# 保存训练数据
|
||||
with open(export_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(training_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
self.update_status(f"已导出 {len(training_data)} 个训练样本到 {export_path}")
|
||||
messagebox.showinfo("成功", f"训练数据已导出到 {export_path}")
|
||||
|
||||
except Exception as e:
|
||||
messagebox.showerror("错误", f"导出数据失败: {str(e)}")
|
||||
|
||||
def toggle_play(self):
|
||||
"""切换播放/暂停状态"""
|
||||
if not self.robot_data:
|
||||
return
|
||||
|
||||
self.playing = not self.playing
|
||||
|
||||
if self.playing:
|
||||
self.play_thread = threading.Thread(target=self.play_data)
|
||||
self.play_thread.start()
|
||||
else:
|
||||
if self.play_thread:
|
||||
self.play_thread = None
|
||||
|
||||
def play_data(self):
|
||||
"""播放数据"""
|
||||
timestamps = self.robot_data['timestamps'][:]
|
||||
start_time = timestamps[0]
|
||||
|
||||
while self.playing and self.current_time < self.total_time:
|
||||
# 更新当前时间
|
||||
self.current_time += 0.1 # 每次增加0.1秒
|
||||
|
||||
# 更新GUI
|
||||
self.root.after(0, self.update_time_display)
|
||||
self.root.after(0, self.update_visualization)
|
||||
|
||||
time.sleep(0.1) # 控制播放速度
|
||||
|
||||
self.playing = False
|
||||
|
||||
def reset_playback(self):
|
||||
"""重置播放"""
|
||||
self.playing = False
|
||||
self.current_time = 0.0
|
||||
self.update_time_display()
|
||||
self.update_visualization()
|
||||
|
||||
def on_time_change(self, value):
|
||||
"""时间滑块变化事件"""
|
||||
if not self.playing: # 只在非播放状态下响应滑块变化
|
||||
self.current_time = float(value)
|
||||
self.update_time_display()
|
||||
self.update_visualization()
|
||||
|
||||
def update_time_display(self):
|
||||
"""更新时间显示"""
|
||||
self.time_label.config(text=f"时间: {self.current_time:.2f} / {self.total_time:.2f} 秒")
|
||||
self.time_scale.set(self.current_time)
|
||||
|
||||
def update_visualization(self):
|
||||
"""更新数据可视化"""
|
||||
if not self.robot_data:
|
||||
return
|
||||
|
||||
self.fig.clear()
|
||||
|
||||
# 获取当前时间对应的数据索引
|
||||
timestamps = self.robot_data['timestamps'][:]
|
||||
start_timestamp = timestamps[0]
|
||||
current_timestamp = start_timestamp + self.current_time
|
||||
current_idx = np.searchsorted(timestamps, current_timestamp)
|
||||
|
||||
# 绘制关节位置
|
||||
ax1 = self.fig.add_subplot(2, 1, 1)
|
||||
joint_positions = self.robot_data['joint_positions'][:]
|
||||
|
||||
# 显示一段时间窗口的数据
|
||||
window_size = min(100, len(joint_positions))
|
||||
start_idx = max(0, current_idx - window_size // 2)
|
||||
end_idx = min(len(joint_positions), start_idx + window_size)
|
||||
|
||||
time_window = (timestamps[start_idx:end_idx] - start_timestamp)
|
||||
|
||||
for i in range(joint_positions.shape[1]):
|
||||
ax1.plot(time_window, joint_positions[start_idx:end_idx, i], label=f'Joint {i+1}')
|
||||
|
||||
# 标记当前时间
|
||||
ax1.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
|
||||
ax1.set_xlabel('Time (s)')
|
||||
ax1.set_ylabel('Joint Position (rad)')
|
||||
ax1.set_title('Joint Positions')
|
||||
ax1.legend()
|
||||
ax1.grid(True)
|
||||
|
||||
# 绘制末端执行器位置
|
||||
ax2 = self.fig.add_subplot(2, 1, 2)
|
||||
end_effector_poses = self.robot_data['end_effector_poses'][:]
|
||||
|
||||
for i in range(3): # 只显示位置信息 (x, y, z)
|
||||
ax2.plot(time_window, end_effector_poses[start_idx:end_idx, i], label=f'{"XYZ"[i]}')
|
||||
|
||||
ax2.axvline(x=self.current_time, color='red', linestyle='--', label='Current Time')
|
||||
ax2.set_xlabel('Time (s)')
|
||||
ax2.set_ylabel('Position (m)')
|
||||
ax2.set_title('End Effector Position')
|
||||
ax2.legend()
|
||||
ax2.grid(True)
|
||||
|
||||
# 绘制标注区间
|
||||
for annotation in self.annotations:
|
||||
ax1.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
|
||||
ax2.axvspan(annotation.start_time, annotation.end_time, alpha=0.3, color='yellow')
|
||||
|
||||
self.fig.tight_layout()
|
||||
self.canvas.draw()
|
||||
|
||||
def set_start_time(self):
|
||||
"""设置标注开始时间"""
|
||||
self.start_time_var.set(f"{self.current_time:.2f}")
|
||||
|
||||
def set_end_time(self):
|
||||
"""设置标注结束时间"""
|
||||
self.end_time_var.set(f"{self.current_time:.2f}")
|
||||
|
||||
def add_annotation(self):
|
||||
"""添加标注"""
|
||||
try:
|
||||
start_time = float(self.start_time_var.get())
|
||||
end_time = float(self.end_time_var.get())
|
||||
text = self.text_entry.get("1.0", tk.END).strip()
|
||||
|
||||
if not text:
|
||||
messagebox.showwarning("警告", "请输入语音指令文本")
|
||||
return
|
||||
|
||||
if start_time >= end_time:
|
||||
messagebox.showwarning("警告", "开始时间必须小于结束时间")
|
||||
return
|
||||
|
||||
# 创建新标注
|
||||
annotation = Annotation(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
text=text
|
||||
)
|
||||
|
||||
self.annotations.append(annotation)
|
||||
self.update_annotation_list()
|
||||
self.update_visualization()
|
||||
|
||||
# 清空输入框
|
||||
self.text_entry.delete("1.0", tk.END)
|
||||
|
||||
self.update_status(f"已添加标注: {text[:30]}...")
|
||||
|
||||
except ValueError:
|
||||
messagebox.showerror("错误", "时间格式不正确")
|
||||
except Exception as e:
|
||||
messagebox.showerror("错误", f"添加标注失败: {str(e)}")
|
||||
|
||||
def delete_annotation(self):
|
||||
"""删除选中的标注"""
|
||||
selection = self.annotation_tree.selection()
|
||||
if not selection:
|
||||
messagebox.showwarning("警告", "请选择要删除的标注")
|
||||
return
|
||||
|
||||
try:
|
||||
# 获取选中项的索引
|
||||
item = selection[0]
|
||||
index = self.annotation_tree.index(item)
|
||||
|
||||
# 删除标注
|
||||
deleted_annotation = self.annotations.pop(index)
|
||||
self.update_annotation_list()
|
||||
self.update_visualization()
|
||||
|
||||
self.update_status(f"已删除标注: {deleted_annotation.text[:30]}...")
|
||||
|
||||
except Exception as e:
|
||||
messagebox.showerror("错误", f"删除标注失败: {str(e)}")
|
||||
|
||||
def update_annotation_list(self):
|
||||
"""更新标注列表显示"""
|
||||
# 清空现有项目
|
||||
for item in self.annotation_tree.get_children():
|
||||
self.annotation_tree.delete(item)
|
||||
|
||||
# 添加标注项目
|
||||
for annotation in self.annotations:
|
||||
time_range = f"{annotation.start_time:.2f}-{annotation.end_time:.2f}s"
|
||||
text_preview = annotation.text[:50] + "..." if len(annotation.text) > 50 else annotation.text
|
||||
|
||||
self.annotation_tree.insert('', tk.END, values=(time_range, text_preview))
|
||||
|
||||
def on_annotation_select(self, event):
|
||||
"""标注选择事件"""
|
||||
selection = self.annotation_tree.selection()
|
||||
if selection:
|
||||
item = selection[0]
|
||||
index = self.annotation_tree.index(item)
|
||||
annotation = self.annotations[index]
|
||||
|
||||
# 更新输入框
|
||||
self.start_time_var.set(f"{annotation.start_time:.2f}")
|
||||
self.end_time_var.set(f"{annotation.end_time:.2f}")
|
||||
self.text_entry.delete("1.0", tk.END)
|
||||
self.text_entry.insert("1.0", annotation.text)
|
||||
|
||||
def on_key_press(self, event):
|
||||
"""键盘事件处理"""
|
||||
if event.keysym == 'space':
|
||||
self.toggle_play()
|
||||
elif event.keysym == 'Left':
|
||||
self.current_time = max(0, self.current_time - 1.0)
|
||||
self.update_time_display()
|
||||
self.update_visualization()
|
||||
elif event.keysym == 'Right':
|
||||
self.current_time = min(self.total_time, self.current_time + 1.0)
|
||||
self.update_time_display()
|
||||
self.update_visualization()
|
||||
|
||||
def update_status(self, message):
|
||||
"""更新状态栏"""
|
||||
self.status_label.config(text=message)
|
||||
|
||||
def run(self):
|
||||
"""运行GUI"""
|
||||
self.root.mainloop()
|
||||
|
||||
# 清理资源
|
||||
if self.robot_data:
|
||||
self.robot_data.close()
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
app = AnnotationTool()
|
||||
app.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
280
data_collector.py
普通文件
280
data_collector.py
普通文件
@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据采集模块 - 用于录制机器人动作和传感器数据
|
||||
基于LeRobot框架的语音输入法训练系统
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
import numpy as np
|
||||
import h5py
|
||||
import pygame
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
|
||||
@dataclass
|
||||
class RobotState:
|
||||
"""机器人状态数据结构"""
|
||||
timestamp: float
|
||||
joint_positions: List[float]
|
||||
joint_velocities: List[float]
|
||||
joint_torques: List[float]
|
||||
end_effector_pose: List[float] # [x, y, z, roll, pitch, yaw]
|
||||
gripper_state: float
|
||||
camera_image: Optional[np.ndarray] = None
|
||||
depth_image: Optional[np.ndarray] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ControlCommand:
|
||||
"""控制指令数据结构"""
|
||||
timestamp: float
|
||||
command_type: str # 'joystick', 'button', 'voice'
|
||||
command_data: Dict[str, Any]
|
||||
|
||||
|
||||
class DataCollector:
|
||||
"""数据采集器类"""
|
||||
|
||||
def __init__(self, output_dir: str = "./collected_data"):
|
||||
self.output_dir = output_dir
|
||||
self.is_recording = False
|
||||
self.recording_thread = None
|
||||
self.data_buffer = []
|
||||
self.command_buffer = []
|
||||
|
||||
# 创建输出目录
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# 初始化pygame用于手柄控制
|
||||
pygame.init()
|
||||
pygame.joystick.init()
|
||||
|
||||
# 检查手柄连接
|
||||
self.joystick = None
|
||||
if pygame.joystick.get_count() > 0:
|
||||
self.joystick = pygame.joystick.Joystick(0)
|
||||
self.joystick.init()
|
||||
print(f"检测到手柄: {self.joystick.get_name()}")
|
||||
else:
|
||||
print("警告: 未检测到手柄,将使用键盘模拟")
|
||||
|
||||
def start_recording(self, session_name: str = None) -> str:
|
||||
"""开始录制数据"""
|
||||
if self.is_recording:
|
||||
print("已经在录制中...")
|
||||
return None
|
||||
|
||||
if session_name is None:
|
||||
session_name = f"session_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
|
||||
self.session_name = session_name
|
||||
self.session_dir = os.path.join(self.output_dir, session_name)
|
||||
os.makedirs(self.session_dir, exist_ok=True)
|
||||
|
||||
# 清空缓冲区
|
||||
self.data_buffer.clear()
|
||||
self.command_buffer.clear()
|
||||
|
||||
self.is_recording = True
|
||||
self.recording_thread = threading.Thread(target=self._recording_loop)
|
||||
self.recording_thread.start()
|
||||
|
||||
print(f"开始录制会话: {session_name}")
|
||||
return session_name
|
||||
|
||||
def stop_recording(self):
|
||||
"""停止录制并保存数据"""
|
||||
if not self.is_recording:
|
||||
print("当前没有在录制...")
|
||||
return
|
||||
|
||||
self.is_recording = False
|
||||
if self.recording_thread:
|
||||
self.recording_thread.join()
|
||||
|
||||
# 保存数据
|
||||
self._save_data()
|
||||
print(f"录制完成,数据已保存到: {self.session_dir}")
|
||||
|
||||
def _recording_loop(self):
|
||||
"""录制循环"""
|
||||
print("录制循环开始...")
|
||||
print("使用手柄控制机器人,按 'q' 键停止录制")
|
||||
|
||||
while self.is_recording:
|
||||
try:
|
||||
# 处理pygame事件
|
||||
pygame.event.pump()
|
||||
|
||||
# 获取当前时间戳
|
||||
timestamp = time.time()
|
||||
|
||||
# 模拟获取机器人状态数据
|
||||
robot_state = self._get_robot_state(timestamp)
|
||||
self.data_buffer.append(robot_state)
|
||||
|
||||
# 获取控制指令
|
||||
control_command = self._get_control_command(timestamp)
|
||||
if control_command:
|
||||
self.command_buffer.append(control_command)
|
||||
|
||||
# 控制录制频率 (50Hz)
|
||||
time.sleep(0.02)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("接收到中断信号,停止录制...")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"录制过程中出现错误: {e}")
|
||||
break
|
||||
|
||||
def _get_robot_state(self, timestamp: float) -> RobotState:
|
||||
"""获取机器人状态(模拟数据)"""
|
||||
# 这里应该连接到真实的机器人API获取数据
|
||||
# 目前使用模拟数据
|
||||
return RobotState(
|
||||
timestamp=timestamp,
|
||||
joint_positions=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
|
||||
joint_velocities=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06],
|
||||
joint_torques=[1.0, 1.1, 1.2, 1.3, 1.4, 1.5],
|
||||
end_effector_pose=[0.5, 0.0, 0.3, 0.0, 0.0, 0.0],
|
||||
gripper_state=0.5,
|
||||
camera_image=np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8),
|
||||
depth_image=np.random.rand(480, 640).astype(np.float32)
|
||||
)
|
||||
|
||||
def _get_control_command(self, timestamp: float) -> Optional[ControlCommand]:
|
||||
"""获取控制指令"""
|
||||
if self.joystick:
|
||||
# 读取手柄输入
|
||||
axes = [self.joystick.get_axis(i) for i in range(self.joystick.get_numaxes())]
|
||||
buttons = [self.joystick.get_button(i) for i in range(self.joystick.get_numbuttons())]
|
||||
|
||||
# 检查是否有显著的输入变化
|
||||
if any(abs(axis) > 0.1 for axis in axes) or any(buttons):
|
||||
return ControlCommand(
|
||||
timestamp=timestamp,
|
||||
command_type="joystick",
|
||||
command_data={
|
||||
"axes": axes,
|
||||
"buttons": buttons
|
||||
}
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _save_data(self):
|
||||
"""保存录制的数据"""
|
||||
# 保存机器人状态数据到HDF5文件
|
||||
hdf5_path = os.path.join(self.session_dir, "robot_data.h5")
|
||||
with h5py.File(hdf5_path, 'w') as f:
|
||||
# 创建数据集
|
||||
num_samples = len(self.data_buffer)
|
||||
|
||||
# 时间戳
|
||||
f.create_dataset('timestamps', data=[state.timestamp for state in self.data_buffer])
|
||||
|
||||
# 关节数据
|
||||
joint_positions = np.array([state.joint_positions for state in self.data_buffer])
|
||||
joint_velocities = np.array([state.joint_velocities for state in self.data_buffer])
|
||||
joint_torques = np.array([state.joint_torques for state in self.data_buffer])
|
||||
|
||||
f.create_dataset('joint_positions', data=joint_positions)
|
||||
f.create_dataset('joint_velocities', data=joint_velocities)
|
||||
f.create_dataset('joint_torques', data=joint_torques)
|
||||
|
||||
# 末端执行器位姿
|
||||
end_effector_poses = np.array([state.end_effector_pose for state in self.data_buffer])
|
||||
f.create_dataset('end_effector_poses', data=end_effector_poses)
|
||||
|
||||
# 夹爪状态
|
||||
gripper_states = np.array([state.gripper_state for state in self.data_buffer])
|
||||
f.create_dataset('gripper_states', data=gripper_states)
|
||||
|
||||
# 图像数据
|
||||
if self.data_buffer and self.data_buffer[0].camera_image is not None:
|
||||
camera_images = np.array([state.camera_image for state in self.data_buffer])
|
||||
f.create_dataset('camera_images', data=camera_images, compression='gzip')
|
||||
|
||||
if self.data_buffer and self.data_buffer[0].depth_image is not None:
|
||||
depth_images = np.array([state.depth_image for state in self.data_buffer])
|
||||
f.create_dataset('depth_images', data=depth_images, compression='gzip')
|
||||
|
||||
# 保存控制指令到JSON文件
|
||||
commands_path = os.path.join(self.session_dir, "commands.json")
|
||||
with open(commands_path, 'w') as f:
|
||||
commands_data = [asdict(cmd) for cmd in self.command_buffer]
|
||||
json.dump(commands_data, f, indent=2)
|
||||
|
||||
# 保存会话元数据
|
||||
metadata = {
|
||||
"session_name": self.session_name,
|
||||
"start_time": self.data_buffer[0].timestamp if self.data_buffer else None,
|
||||
"end_time": self.data_buffer[-1].timestamp if self.data_buffer else None,
|
||||
"duration": (self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 0,
|
||||
"num_samples": len(self.data_buffer),
|
||||
"num_commands": len(self.command_buffer),
|
||||
"recording_frequency": len(self.data_buffer) / ((self.data_buffer[-1].timestamp - self.data_buffer[0].timestamp) if len(self.data_buffer) > 1 else 1)
|
||||
}
|
||||
|
||||
metadata_path = os.path.join(self.session_dir, "metadata.json")
|
||||
with open(metadata_path, 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
print(f"数据已保存:")
|
||||
print(f" - 机器人数据: {hdf5_path}")
|
||||
print(f" - 控制指令: {commands_path}")
|
||||
print(f" - 元数据: {metadata_path}")
|
||||
print(f" - 总样本数: {len(self.data_buffer)}")
|
||||
print(f" - 总指令数: {len(self.command_buffer)}")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数 - 演示数据采集器的使用"""
|
||||
collector = DataCollector()
|
||||
|
||||
print("=== 语音机器人训练系统 - 数据采集器 ===")
|
||||
print("指令:")
|
||||
print(" start <session_name> - 开始录制")
|
||||
print(" stop - 停止录制")
|
||||
print(" quit - 退出程序")
|
||||
|
||||
while True:
|
||||
try:
|
||||
command = input("\n请输入指令: ").strip().split()
|
||||
|
||||
if not command:
|
||||
continue
|
||||
|
||||
if command[0] == "start":
|
||||
session_name = command[1] if len(command) > 1 else None
|
||||
collector.start_recording(session_name)
|
||||
|
||||
elif command[0] == "stop":
|
||||
collector.stop_recording()
|
||||
|
||||
elif command[0] == "quit":
|
||||
if collector.is_recording:
|
||||
collector.stop_recording()
|
||||
break
|
||||
|
||||
else:
|
||||
print("未知指令,请重新输入")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n程序被中断")
|
||||
if collector.is_recording:
|
||||
collector.stop_recording()
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"发生错误: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
378
example_usage.py
普通文件
378
example_usage.py
普通文件
@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
使用示例 - 演示语音机器人训练系统的完整使用流程
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目根目录到路径
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
from data_collector import DataCollector, RobotState, ControlCommand
|
||||
from model_trainer import VoiceRobotTrainer, TrainingConfig
|
||||
from inference_engine import VoiceRobotSystem
|
||||
|
||||
|
||||
def example_data_collection():
|
||||
"""示例:数据采集流程"""
|
||||
print("=== 数据采集示例 ===")
|
||||
|
||||
# 创建数据采集器
|
||||
collector = DataCollector(output_dir="./example_data")
|
||||
|
||||
# 模拟数据采集过程
|
||||
print("开始模拟数据采集...")
|
||||
session_name = collector.start_recording("example_session")
|
||||
|
||||
# 模拟录制5秒钟的数据
|
||||
time.sleep(5)
|
||||
|
||||
collector.stop_recording()
|
||||
print(f"数据采集完成,会话: {session_name}")
|
||||
|
||||
return session_name
|
||||
|
||||
|
||||
def example_manual_annotation(session_name):
|
||||
"""示例:手动创建标注数据"""
|
||||
print("=== 手动标注示例 ===")
|
||||
|
||||
session_dir = f"./example_data/{session_name}"
|
||||
|
||||
# 创建示例标注数据
|
||||
annotations = [
|
||||
{
|
||||
"start_time": 0.0,
|
||||
"end_time": 2.0,
|
||||
"text": "向前移动",
|
||||
"confidence": 1.0,
|
||||
"annotator": "example",
|
||||
"timestamp": time.time()
|
||||
},
|
||||
{
|
||||
"start_time": 2.0,
|
||||
"end_time": 4.0,
|
||||
"text": "抓取物体",
|
||||
"confidence": 1.0,
|
||||
"annotator": "example",
|
||||
"timestamp": time.time()
|
||||
}
|
||||
]
|
||||
|
||||
# 保存标注数据
|
||||
annotations_path = os.path.join(session_dir, "annotations.json")
|
||||
with open(annotations_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(annotations, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"标注数据已保存到: {annotations_path}")
|
||||
|
||||
# 创建训练数据
|
||||
create_example_training_data(session_dir, annotations)
|
||||
|
||||
|
||||
def create_example_training_data(session_dir, annotations):
|
||||
"""创建示例训练数据"""
|
||||
print("创建训练数据...")
|
||||
|
||||
training_data = []
|
||||
|
||||
for annotation in annotations:
|
||||
# 模拟动作序列数据
|
||||
duration = annotation["end_time"] - annotation["start_time"]
|
||||
num_steps = int(duration * 50) # 50Hz采样率
|
||||
|
||||
# 生成模拟的机器人动作数据
|
||||
joint_positions = []
|
||||
end_effector_poses = []
|
||||
gripper_states = []
|
||||
timestamps = []
|
||||
|
||||
for i in range(num_steps):
|
||||
t = i / 50.0
|
||||
|
||||
# 模拟关节位置(正弦波变化)
|
||||
joint_pos = [0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
|
||||
joint_positions.append(joint_pos)
|
||||
|
||||
# 模拟末端执行器位姿
|
||||
pose = [0.5 + 0.1 * np.sin(2 * np.pi * t),
|
||||
0.0 + 0.1 * np.cos(2 * np.pi * t),
|
||||
0.3, 0.0, 0.0, 0.0]
|
||||
end_effector_poses.append(pose)
|
||||
|
||||
# 模拟夹爪状态
|
||||
if "抓取" in annotation["text"]:
|
||||
gripper_state = min(1.0, t / duration) # 逐渐闭合
|
||||
else:
|
||||
gripper_state = 0.5 # 保持中间位置
|
||||
gripper_states.append(gripper_state)
|
||||
|
||||
timestamps.append(t)
|
||||
|
||||
# 创建训练样本
|
||||
sample = {
|
||||
"text": annotation["text"],
|
||||
"start_time": annotation["start_time"],
|
||||
"end_time": annotation["end_time"],
|
||||
"joint_positions": joint_positions,
|
||||
"joint_velocities": [[0.0] * 6] * num_steps, # 简化为零速度
|
||||
"end_effector_poses": end_effector_poses,
|
||||
"gripper_states": gripper_states,
|
||||
"timestamps": timestamps
|
||||
}
|
||||
|
||||
training_data.append(sample)
|
||||
|
||||
# 保存训练数据
|
||||
training_data_path = os.path.join(session_dir, "training_data.json")
|
||||
with open(training_data_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(training_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"训练数据已保存到: {training_data_path}")
|
||||
|
||||
|
||||
def example_model_training():
|
||||
"""示例:模型训练流程"""
|
||||
print("=== 模型训练示例 ===")
|
||||
|
||||
# 创建训练配置
|
||||
config = TrainingConfig(
|
||||
data_dir="./example_data",
|
||||
output_dir="./example_models",
|
||||
batch_size=2, # 小批次用于示例
|
||||
num_epochs=10, # 少量epoch用于示例
|
||||
learning_rate=1e-3,
|
||||
early_stopping_patience=5
|
||||
)
|
||||
|
||||
# 创建训练器
|
||||
trainer = VoiceRobotTrainer(config)
|
||||
|
||||
try:
|
||||
# 开始训练
|
||||
trainer.train()
|
||||
print("模型训练完成!")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"训练过程中出现错误: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def example_inference():
|
||||
"""示例:模型推理流程"""
|
||||
print("=== 模型推理示例 ===")
|
||||
|
||||
model_path = "./example_models/best_model.pth"
|
||||
|
||||
if not os.path.exists(model_path):
|
||||
print("模型文件不存在,请先完成训练")
|
||||
return
|
||||
|
||||
# 创建推理系统
|
||||
system = VoiceRobotSystem(model_path)
|
||||
|
||||
# 测试文本指令
|
||||
test_commands = [
|
||||
"向前移动",
|
||||
"抓取物体",
|
||||
"向左转动",
|
||||
"放下物体"
|
||||
]
|
||||
|
||||
print("测试文本指令:")
|
||||
for command in test_commands:
|
||||
print(f"\n指令: {command}")
|
||||
try:
|
||||
system.process_text_command(command)
|
||||
except Exception as e:
|
||||
print(f"处理指令失败: {e}")
|
||||
|
||||
print("推理测试完成!")
|
||||
|
||||
|
||||
def example_complete_workflow():
|
||||
"""示例:完整的工作流程"""
|
||||
print("=== 完整工作流程示例 ===")
|
||||
print("这个示例将演示从数据采集到模型部署的完整流程")
|
||||
|
||||
# 步骤1:数据采集
|
||||
print("\n步骤1: 数据采集")
|
||||
session_name = example_data_collection()
|
||||
|
||||
# 步骤2:数据标注
|
||||
print("\n步骤2: 数据标注")
|
||||
example_manual_annotation(session_name)
|
||||
|
||||
# 步骤3:模型训练
|
||||
print("\n步骤3: 模型训练")
|
||||
training_success = example_model_training()
|
||||
|
||||
if training_success:
|
||||
# 步骤4:模型推理
|
||||
print("\n步骤4: 模型推理")
|
||||
example_inference()
|
||||
|
||||
print("\n=== 完整工作流程示例完成 ===")
|
||||
|
||||
|
||||
def example_custom_robot_integration():
|
||||
"""示例:自定义机器人集成"""
|
||||
print("=== 自定义机器人集成示例 ===")
|
||||
|
||||
class CustomRobotController:
|
||||
"""自定义机器人控制器示例"""
|
||||
|
||||
def __init__(self):
|
||||
print("初始化自定义机器人控制器")
|
||||
# 这里可以初始化机器人SDK、ROS节点等
|
||||
|
||||
def execute_action(self, prediction):
|
||||
"""执行机器人动作"""
|
||||
joint_positions = prediction['joint_positions']
|
||||
end_effector_poses = prediction['end_effector_poses']
|
||||
gripper_states = prediction['gripper_states']
|
||||
text = prediction['text']
|
||||
|
||||
print(f"执行自定义机器人动作: {text}")
|
||||
|
||||
# 这里实现具体的机器人控制逻辑
|
||||
# 例如:
|
||||
# self.robot_arm.move_joints(joint_positions)
|
||||
# self.robot_arm.move_to_pose(end_effector_poses)
|
||||
# self.gripper.set_position(gripper_states[0])
|
||||
|
||||
print(f"关节目标位置: {joint_positions}")
|
||||
print(f"末端执行器目标位姿: {end_effector_poses}")
|
||||
print(f"夹爪目标状态: {gripper_states}")
|
||||
|
||||
# 使用自定义控制器
|
||||
custom_controller = CustomRobotController()
|
||||
|
||||
# 模拟预测结果
|
||||
mock_prediction = {
|
||||
'joint_positions': np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6]),
|
||||
'end_effector_poses': np.array([0.5, 0.0, 0.3, 0.0, 0.0, 0.0]),
|
||||
'gripper_states': np.array([0.8]),
|
||||
'text': '抓取红色方块'
|
||||
}
|
||||
|
||||
custom_controller.execute_action(mock_prediction)
|
||||
|
||||
|
||||
def example_data_analysis():
|
||||
"""示例:数据分析和可视化"""
|
||||
print("=== 数据分析示例 ===")
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# 设置中文字体
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
|
||||
# 生成示例数据
|
||||
time_steps = np.linspace(0, 5, 250) # 5秒,50Hz
|
||||
joint_positions = np.array([
|
||||
[0.1 * np.sin(2 * np.pi * t + j) for j in range(6)]
|
||||
for t in time_steps
|
||||
])
|
||||
|
||||
# 创建可视化
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
|
||||
# 关节位置
|
||||
axes[0, 0].plot(time_steps, joint_positions)
|
||||
axes[0, 0].set_title('关节位置轨迹')
|
||||
axes[0, 0].set_xlabel('时间 (秒)')
|
||||
axes[0, 0].set_ylabel('位置 (弧度)')
|
||||
axes[0, 0].grid(True)
|
||||
axes[0, 0].legend([f'关节{i+1}' for i in range(6)])
|
||||
|
||||
# 末端执行器轨迹
|
||||
end_effector_x = 0.5 + 0.1 * np.sin(2 * np.pi * time_steps)
|
||||
end_effector_y = 0.0 + 0.1 * np.cos(2 * np.pi * time_steps)
|
||||
|
||||
axes[0, 1].plot(end_effector_x, end_effector_y)
|
||||
axes[0, 1].set_title('末端执行器轨迹 (XY平面)')
|
||||
axes[0, 1].set_xlabel('X位置 (米)')
|
||||
axes[0, 1].set_ylabel('Y位置 (米)')
|
||||
axes[0, 1].grid(True)
|
||||
axes[0, 1].axis('equal')
|
||||
|
||||
# 夹爪状态
|
||||
gripper_states = np.minimum(1.0, time_steps / 5.0)
|
||||
axes[1, 0].plot(time_steps, gripper_states)
|
||||
axes[1, 0].set_title('夹爪状态')
|
||||
axes[1, 0].set_xlabel('时间 (秒)')
|
||||
axes[1, 0].set_ylabel('开合程度')
|
||||
axes[1, 0].grid(True)
|
||||
|
||||
# 速度分析
|
||||
joint_velocities = np.gradient(joint_positions, axis=0) / (time_steps[1] - time_steps[0])
|
||||
axes[1, 1].plot(time_steps, np.linalg.norm(joint_velocities, axis=1))
|
||||
axes[1, 1].set_title('关节速度幅值')
|
||||
axes[1, 1].set_xlabel('时间 (秒)')
|
||||
axes[1, 1].set_ylabel('速度 (弧度/秒)')
|
||||
axes[1, 1].grid(True)
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
# 保存图片
|
||||
os.makedirs('./example_output', exist_ok=True)
|
||||
plt.savefig('./example_output/data_analysis.png', dpi=300, bbox_inches='tight')
|
||||
plt.close()
|
||||
|
||||
print("数据分析图表已保存到: ./example_output/data_analysis.png")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
print("语音机器人训练系统 - 使用示例")
|
||||
print("=" * 50)
|
||||
|
||||
examples = {
|
||||
"1": ("完整工作流程", example_complete_workflow),
|
||||
"2": ("数据采集", example_data_collection),
|
||||
"3": ("模型训练", example_model_training),
|
||||
"4": ("模型推理", example_inference),
|
||||
"5": ("自定义机器人集成", example_custom_robot_integration),
|
||||
"6": ("数据分析", example_data_analysis),
|
||||
"0": ("运行所有示例", None)
|
||||
}
|
||||
|
||||
print("可用示例:")
|
||||
for key, (name, _) in examples.items():
|
||||
print(f" {key}. {name}")
|
||||
|
||||
choice = input("\n请选择要运行的示例 (输入数字): ").strip()
|
||||
|
||||
if choice == "0":
|
||||
# 运行所有示例
|
||||
for key, (name, func) in examples.items():
|
||||
if func is not None:
|
||||
print(f"\n{'='*20} {name} {'='*20}")
|
||||
try:
|
||||
func()
|
||||
except Exception as e:
|
||||
print(f"示例 '{name}' 运行失败: {e}")
|
||||
elif choice in examples:
|
||||
name, func = examples[choice]
|
||||
if func is not None:
|
||||
print(f"\n运行示例: {name}")
|
||||
try:
|
||||
func()
|
||||
except Exception as e:
|
||||
print(f"示例运行失败: {e}")
|
||||
else:
|
||||
print("无效选择")
|
||||
else:
|
||||
print("无效选择")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
441
inference_engine.py
普通文件
441
inference_engine.py
普通文件
@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
推理引擎 - 用于部署训练好的语音-动作映射模型
|
||||
基于LeRobot框架的语音输入法训练系统
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
import pickle
|
||||
import speech_recognition as sr
|
||||
import pyaudio
|
||||
from typing import Dict, List, Optional, Any
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import time
|
||||
import threading
|
||||
from queue import Queue
|
||||
from transformers import AutoTokenizer
|
||||
from model_trainer import VoiceRobotModel, TrainingConfig
|
||||
|
||||
|
||||
# 设置日志
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VoiceRobotInference:
|
||||
"""语音机器人推理引擎"""
|
||||
|
||||
def __init__(self, model_path: str, device: str = "auto"):
|
||||
"""
|
||||
初始化推理引擎
|
||||
|
||||
Args:
|
||||
model_path: 训练好的模型路径
|
||||
device: 计算设备 ("auto", "cpu", "cuda")
|
||||
"""
|
||||
self.model_path = model_path
|
||||
|
||||
# 设置设备
|
||||
if device == "auto":
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
else:
|
||||
self.device = torch.device(device)
|
||||
|
||||
logger.info(f"使用设备: {self.device}")
|
||||
|
||||
# 加载模型和配置
|
||||
self._load_model()
|
||||
|
||||
# 初始化语音识别
|
||||
self._init_speech_recognition()
|
||||
|
||||
# 语音处理队列
|
||||
self.audio_queue = Queue()
|
||||
self.result_queue = Queue()
|
||||
self.is_listening = False
|
||||
self.listen_thread = None
|
||||
|
||||
logger.info("推理引擎初始化完成")
|
||||
|
||||
def _load_model(self):
|
||||
"""加载训练好的模型"""
|
||||
logger.info(f"加载模型: {self.model_path}")
|
||||
|
||||
# 加载模型检查点
|
||||
checkpoint = torch.load(self.model_path, map_location=self.device)
|
||||
self.config = checkpoint['config']
|
||||
|
||||
# 创建模型
|
||||
self.model = VoiceRobotModel(self.config)
|
||||
self.model.load_state_dict(checkpoint['model_state_dict'])
|
||||
self.model.to(self.device)
|
||||
self.model.eval()
|
||||
|
||||
# 加载tokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(self.config.text_encoder_model)
|
||||
|
||||
# 加载数据标准化器
|
||||
scaler_path = os.path.join(os.path.dirname(self.model_path), "scalers.pkl")
|
||||
if os.path.exists(scaler_path):
|
||||
with open(scaler_path, 'rb') as f:
|
||||
scalers = pickle.load(f)
|
||||
self.action_scaler = scalers['action_scaler']
|
||||
self.pose_scaler = scalers['pose_scaler']
|
||||
self.gripper_scaler = scalers['gripper_scaler']
|
||||
else:
|
||||
logger.warning("未找到数据标准化器,预测结果可能不准确")
|
||||
self.action_scaler = None
|
||||
self.pose_scaler = None
|
||||
self.gripper_scaler = None
|
||||
|
||||
logger.info("模型加载完成")
|
||||
|
||||
def _init_speech_recognition(self):
|
||||
"""初始化语音识别"""
|
||||
self.recognizer = sr.Recognizer()
|
||||
self.microphone = sr.Microphone()
|
||||
|
||||
# 调整环境噪声
|
||||
logger.info("调整环境噪声...")
|
||||
with self.microphone as source:
|
||||
self.recognizer.adjust_for_ambient_noise(source)
|
||||
|
||||
logger.info("语音识别初始化完成")
|
||||
|
||||
def predict_from_text(self, text: str) -> Dict[str, np.ndarray]:
|
||||
"""
|
||||
从文本预测机器人动作
|
||||
|
||||
Args:
|
||||
text: 语音指令文本
|
||||
|
||||
Returns:
|
||||
包含预测动作的字典
|
||||
"""
|
||||
logger.info(f"预测文本: {text}")
|
||||
|
||||
# 文本编码
|
||||
encoding = self.tokenizer(
|
||||
text,
|
||||
truncation=True,
|
||||
padding='max_length',
|
||||
max_length=128,
|
||||
return_tensors='pt'
|
||||
)
|
||||
|
||||
# 移动到设备
|
||||
input_ids = encoding['input_ids'].to(self.device)
|
||||
attention_mask = encoding['attention_mask'].to(self.device)
|
||||
|
||||
# 模型推理
|
||||
with torch.no_grad():
|
||||
predictions = self.model(input_ids, attention_mask)
|
||||
|
||||
# 转换为numpy数组
|
||||
joint_positions = predictions['joint_positions'].cpu().numpy()[0]
|
||||
end_effector_poses = predictions['end_effector_poses'].cpu().numpy()[0]
|
||||
gripper_states = predictions['gripper_states'].cpu().numpy()[0]
|
||||
|
||||
# 反标准化
|
||||
if self.action_scaler is not None:
|
||||
joint_positions = self.action_scaler.inverse_transform(joint_positions.reshape(1, -1))[0]
|
||||
|
||||
if self.pose_scaler is not None:
|
||||
end_effector_poses = self.pose_scaler.inverse_transform(end_effector_poses.reshape(1, -1))[0]
|
||||
|
||||
if self.gripper_scaler is not None:
|
||||
gripper_states = self.gripper_scaler.inverse_transform(gripper_states.reshape(1, -1))[0]
|
||||
|
||||
result = {
|
||||
'joint_positions': joint_positions,
|
||||
'end_effector_poses': end_effector_poses,
|
||||
'gripper_states': gripper_states,
|
||||
'text': text
|
||||
}
|
||||
|
||||
logger.info(f"预测完成: 关节位置={joint_positions}, 末端位姿={end_effector_poses}, 夹爪状态={gripper_states}")
|
||||
|
||||
return result
|
||||
|
||||
def recognize_speech(self, audio_data) -> Optional[str]:
|
||||
"""
|
||||
识别语音
|
||||
|
||||
Args:
|
||||
audio_data: 音频数据
|
||||
|
||||
Returns:
|
||||
识别的文本,如果识别失败返回None
|
||||
"""
|
||||
try:
|
||||
# 使用Google语音识别(需要网络连接)
|
||||
text = self.recognizer.recognize_google(audio_data, language='zh-CN')
|
||||
logger.info(f"语音识别结果: {text}")
|
||||
return text
|
||||
except sr.UnknownValueError:
|
||||
logger.warning("无法识别语音")
|
||||
return None
|
||||
except sr.RequestError as e:
|
||||
logger.error(f"语音识别服务错误: {e}")
|
||||
return None
|
||||
|
||||
def start_listening(self):
|
||||
"""开始监听语音"""
|
||||
if self.is_listening:
|
||||
logger.warning("已经在监听中...")
|
||||
return
|
||||
|
||||
self.is_listening = True
|
||||
self.listen_thread = threading.Thread(target=self._listen_loop)
|
||||
self.listen_thread.start()
|
||||
|
||||
logger.info("开始监听语音...")
|
||||
|
||||
def stop_listening(self):
|
||||
"""停止监听语音"""
|
||||
if not self.is_listening:
|
||||
logger.warning("当前没有在监听...")
|
||||
return
|
||||
|
||||
self.is_listening = False
|
||||
if self.listen_thread:
|
||||
self.listen_thread.join()
|
||||
|
||||
logger.info("停止监听语音")
|
||||
|
||||
def _listen_loop(self):
|
||||
"""语音监听循环"""
|
||||
while self.is_listening:
|
||||
try:
|
||||
with self.microphone as source:
|
||||
logger.info("请说话...")
|
||||
# 监听音频,超时时间为1秒
|
||||
audio = self.recognizer.listen(source, timeout=1, phrase_time_limit=5)
|
||||
|
||||
# 识别语音
|
||||
text = self.recognize_speech(audio)
|
||||
|
||||
if text:
|
||||
# 预测动作
|
||||
prediction = self.predict_from_text(text)
|
||||
|
||||
# 将结果放入队列
|
||||
self.result_queue.put(prediction)
|
||||
|
||||
except sr.WaitTimeoutError:
|
||||
# 超时,继续监听
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"监听过程中出现错误: {e}")
|
||||
time.sleep(0.1)
|
||||
|
||||
def get_latest_prediction(self) -> Optional[Dict]:
|
||||
"""获取最新的预测结果"""
|
||||
if not self.result_queue.empty():
|
||||
return self.result_queue.get()
|
||||
return None
|
||||
|
||||
def predict_from_audio_file(self, audio_file_path: str) -> Optional[Dict]:
|
||||
"""
|
||||
从音频文件预测动作
|
||||
|
||||
Args:
|
||||
audio_file_path: 音频文件路径
|
||||
|
||||
Returns:
|
||||
预测结果字典
|
||||
"""
|
||||
try:
|
||||
with sr.AudioFile(audio_file_path) as source:
|
||||
audio = self.recognizer.record(source)
|
||||
|
||||
text = self.recognize_speech(audio)
|
||||
if text:
|
||||
return self.predict_from_text(text)
|
||||
else:
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理音频文件失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class RobotController:
|
||||
"""机器人控制器接口"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化机器人控制器"""
|
||||
# 这里应该连接到真实的机器人控制系统
|
||||
# 例如:ROS节点、机器人SDK等
|
||||
logger.info("机器人控制器初始化(模拟模式)")
|
||||
|
||||
def execute_action(self, prediction: Dict[str, np.ndarray]):
|
||||
"""
|
||||
执行机器人动作
|
||||
|
||||
Args:
|
||||
prediction: 预测的动作参数
|
||||
"""
|
||||
joint_positions = prediction['joint_positions']
|
||||
end_effector_poses = prediction['end_effector_poses']
|
||||
gripper_states = prediction['gripper_states']
|
||||
text = prediction['text']
|
||||
|
||||
logger.info(f"执行动作指令: {text}")
|
||||
logger.info(f"关节位置: {joint_positions}")
|
||||
logger.info(f"末端位姿: {end_effector_poses}")
|
||||
logger.info(f"夹爪状态: {gripper_states}")
|
||||
|
||||
# 这里应该调用真实的机器人控制API
|
||||
# 例如:
|
||||
# self.robot_arm.move_to_joint_positions(joint_positions)
|
||||
# self.robot_arm.move_to_pose(end_effector_poses)
|
||||
# self.gripper.set_position(gripper_states[0])
|
||||
|
||||
# 模拟执行时间
|
||||
time.sleep(1.0)
|
||||
logger.info("动作执行完成")
|
||||
|
||||
|
||||
class VoiceRobotSystem:
|
||||
"""完整的语音机器人系统"""
|
||||
|
||||
def __init__(self, model_path: str):
|
||||
"""
|
||||
初始化语音机器人系统
|
||||
|
||||
Args:
|
||||
model_path: 训练好的模型路径
|
||||
"""
|
||||
self.inference_engine = VoiceRobotInference(model_path)
|
||||
self.robot_controller = RobotController()
|
||||
|
||||
self.is_running = False
|
||||
self.main_thread = None
|
||||
|
||||
def start(self):
|
||||
"""启动系统"""
|
||||
if self.is_running:
|
||||
logger.warning("系统已经在运行中...")
|
||||
return
|
||||
|
||||
self.is_running = True
|
||||
self.inference_engine.start_listening()
|
||||
self.main_thread = threading.Thread(target=self._main_loop)
|
||||
self.main_thread.start()
|
||||
|
||||
logger.info("语音机器人系统启动")
|
||||
|
||||
def stop(self):
|
||||
"""停止系统"""
|
||||
if not self.is_running:
|
||||
logger.warning("系统当前没有运行...")
|
||||
return
|
||||
|
||||
self.is_running = False
|
||||
self.inference_engine.stop_listening()
|
||||
|
||||
if self.main_thread:
|
||||
self.main_thread.join()
|
||||
|
||||
logger.info("语音机器人系统停止")
|
||||
|
||||
def _main_loop(self):
|
||||
"""主循环"""
|
||||
while self.is_running:
|
||||
# 获取最新的预测结果
|
||||
prediction = self.inference_engine.get_latest_prediction()
|
||||
|
||||
if prediction:
|
||||
# 执行机器人动作
|
||||
self.robot_controller.execute_action(prediction)
|
||||
|
||||
time.sleep(0.1) # 控制循环频率
|
||||
|
||||
def process_text_command(self, text: str):
|
||||
"""处理文本指令"""
|
||||
prediction = self.inference_engine.predict_from_text(text)
|
||||
self.robot_controller.execute_action(prediction)
|
||||
|
||||
def process_audio_file(self, audio_file_path: str):
|
||||
"""处理音频文件"""
|
||||
prediction = self.inference_engine.predict_from_audio_file(audio_file_path)
|
||||
if prediction:
|
||||
self.robot_controller.execute_action(prediction)
|
||||
else:
|
||||
logger.warning("无法从音频文件中识别指令")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数 - 演示推理引擎的使用"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="语音机器人推理引擎")
|
||||
parser.add_argument("--model", required=True, help="模型文件路径")
|
||||
parser.add_argument("--mode", choices=["interactive", "text", "audio"], default="interactive", help="运行模式")
|
||||
parser.add_argument("--text", help="文本指令(text模式)")
|
||||
parser.add_argument("--audio", help="音频文件路径(audio模式)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.model):
|
||||
logger.error(f"模型文件不存在: {args.model}")
|
||||
return
|
||||
|
||||
# 创建系统
|
||||
system = VoiceRobotSystem(args.model)
|
||||
|
||||
try:
|
||||
if args.mode == "interactive":
|
||||
# 交互模式
|
||||
system.start()
|
||||
|
||||
print("=== 语音机器人系统 ===")
|
||||
print("系统正在监听语音指令...")
|
||||
print("输入 'quit' 退出程序")
|
||||
|
||||
while True:
|
||||
command = input("\n输入文本指令 (或 'quit' 退出): ").strip()
|
||||
|
||||
if command.lower() == 'quit':
|
||||
break
|
||||
elif command:
|
||||
system.process_text_command(command)
|
||||
|
||||
system.stop()
|
||||
|
||||
elif args.mode == "text":
|
||||
# 文本模式
|
||||
if not args.text:
|
||||
logger.error("文本模式需要提供 --text 参数")
|
||||
return
|
||||
|
||||
system.process_text_command(args.text)
|
||||
|
||||
elif args.mode == "audio":
|
||||
# 音频模式
|
||||
if not args.audio:
|
||||
logger.error("音频模式需要提供 --audio 参数")
|
||||
return
|
||||
|
||||
if not os.path.exists(args.audio):
|
||||
logger.error(f"音频文件不存在: {args.audio}")
|
||||
return
|
||||
|
||||
system.process_audio_file(args.audio)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("程序被用户中断")
|
||||
system.stop()
|
||||
except Exception as e:
|
||||
logger.error(f"程序运行出错: {e}")
|
||||
system.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
619
model_trainer.py
普通文件
619
model_trainer.py
普通文件
@ -0,0 +1,619 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
模型训练管道 - 基于LeRobot框架训练语音-动作映射模型
|
||||
基于LeRobot框架的语音输入法训练系统
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
import numpy as np
|
||||
import h5py
|
||||
from typing import Dict, List, Tuple, Optional, Any
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import pickle
|
||||
import time
|
||||
from transformers import AutoTokenizer, AutoModel
|
||||
|
||||
|
||||
# 设置日志
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainingConfig:
|
||||
"""训练配置"""
|
||||
# 数据相关
|
||||
data_dir: str = "./collected_data"
|
||||
output_dir: str = "./trained_models"
|
||||
|
||||
# 模型相关
|
||||
text_encoder_model: str = "bert-base-chinese" # 中文BERT模型
|
||||
hidden_dim: int = 256
|
||||
action_dim: int = 6 # 关节数量
|
||||
pose_dim: int = 6 # 末端执行器位姿维度
|
||||
gripper_dim: int = 1 # 夹爪状态维度
|
||||
|
||||
# 训练相关
|
||||
batch_size: int = 32
|
||||
learning_rate: float = 1e-4
|
||||
num_epochs: int = 100
|
||||
validation_split: float = 0.2
|
||||
early_stopping_patience: int = 10
|
||||
|
||||
# 设备
|
||||
device: str = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
class VoiceRobotDataset(Dataset):
|
||||
"""语音机器人数据集"""
|
||||
|
||||
def __init__(self, data_samples: List[Dict], tokenizer, scaler=None, max_length=128):
|
||||
self.data_samples = data_samples
|
||||
self.tokenizer = tokenizer
|
||||
self.scaler = scaler
|
||||
self.max_length = max_length
|
||||
|
||||
# 如果没有提供scaler,创建一个新的
|
||||
if self.scaler is None:
|
||||
self._fit_scaler()
|
||||
|
||||
def _fit_scaler(self):
|
||||
"""拟合数据标准化器"""
|
||||
all_actions = []
|
||||
all_poses = []
|
||||
all_grippers = []
|
||||
|
||||
for sample in self.data_samples:
|
||||
# 合并所有时间步的数据
|
||||
joint_positions = np.array(sample['joint_positions'])
|
||||
end_effector_poses = np.array(sample['end_effector_poses'])
|
||||
gripper_states = np.array(sample['gripper_states'])
|
||||
|
||||
all_actions.extend(joint_positions)
|
||||
all_poses.extend(end_effector_poses)
|
||||
all_grippers.extend(gripper_states.reshape(-1, 1))
|
||||
|
||||
# 创建标准化器
|
||||
self.action_scaler = StandardScaler()
|
||||
self.pose_scaler = StandardScaler()
|
||||
self.gripper_scaler = StandardScaler()
|
||||
|
||||
self.action_scaler.fit(all_actions)
|
||||
self.pose_scaler.fit(all_poses)
|
||||
self.gripper_scaler.fit(all_grippers)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data_samples)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
sample = self.data_samples[idx]
|
||||
|
||||
# 文本编码
|
||||
text = sample['text']
|
||||
encoding = self.tokenizer(
|
||||
text,
|
||||
truncation=True,
|
||||
padding='max_length',
|
||||
max_length=self.max_length,
|
||||
return_tensors='pt'
|
||||
)
|
||||
|
||||
# 动作数据处理
|
||||
joint_positions = np.array(sample['joint_positions'])
|
||||
end_effector_poses = np.array(sample['end_effector_poses'])
|
||||
gripper_states = np.array(sample['gripper_states']).reshape(-1, 1)
|
||||
|
||||
# 数据标准化
|
||||
joint_positions_norm = self.action_scaler.transform(joint_positions)
|
||||
end_effector_poses_norm = self.pose_scaler.transform(end_effector_poses)
|
||||
gripper_states_norm = self.gripper_scaler.transform(gripper_states)
|
||||
|
||||
# 转换为张量
|
||||
joint_positions_tensor = torch.FloatTensor(joint_positions_norm)
|
||||
end_effector_poses_tensor = torch.FloatTensor(end_effector_poses_norm)
|
||||
gripper_states_tensor = torch.FloatTensor(gripper_states_norm)
|
||||
|
||||
return {
|
||||
'input_ids': encoding['input_ids'].squeeze(),
|
||||
'attention_mask': encoding['attention_mask'].squeeze(),
|
||||
'joint_positions': joint_positions_tensor,
|
||||
'end_effector_poses': end_effector_poses_tensor,
|
||||
'gripper_states': gripper_states_tensor,
|
||||
'text': text
|
||||
}
|
||||
|
||||
|
||||
class VoiceRobotModel(nn.Module):
|
||||
"""语音机器人模型"""
|
||||
|
||||
def __init__(self, config: TrainingConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
# 文本编码器
|
||||
self.text_encoder = AutoModel.from_pretrained(config.text_encoder_model)
|
||||
text_hidden_size = self.text_encoder.config.hidden_size
|
||||
|
||||
# 冻结文本编码器的部分层(可选)
|
||||
# for param in self.text_encoder.parameters():
|
||||
# param.requires_grad = False
|
||||
|
||||
# 动作解码器
|
||||
self.action_decoder = nn.Sequential(
|
||||
nn.Linear(text_hidden_size, config.hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(config.hidden_dim, config.hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(config.hidden_dim, config.action_dim)
|
||||
)
|
||||
|
||||
# 位姿解码器
|
||||
self.pose_decoder = nn.Sequential(
|
||||
nn.Linear(text_hidden_size, config.hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(config.hidden_dim, config.hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(config.hidden_dim, config.pose_dim)
|
||||
)
|
||||
|
||||
# 夹爪解码器
|
||||
self.gripper_decoder = nn.Sequential(
|
||||
nn.Linear(text_hidden_size, config.hidden_dim // 2),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(config.hidden_dim // 2, config.gripper_dim),
|
||||
nn.Sigmoid() # 夹爪状态通常在0-1之间
|
||||
)
|
||||
|
||||
def forward(self, input_ids, attention_mask):
|
||||
# 文本编码
|
||||
text_outputs = self.text_encoder(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask
|
||||
)
|
||||
|
||||
# 使用[CLS] token的表示
|
||||
text_features = text_outputs.last_hidden_state[:, 0, :] # [batch_size, hidden_size]
|
||||
|
||||
# 解码动作
|
||||
joint_positions = self.action_decoder(text_features)
|
||||
end_effector_poses = self.pose_decoder(text_features)
|
||||
gripper_states = self.gripper_decoder(text_features)
|
||||
|
||||
return {
|
||||
'joint_positions': joint_positions,
|
||||
'end_effector_poses': end_effector_poses,
|
||||
'gripper_states': gripper_states
|
||||
}
|
||||
|
||||
|
||||
class VoiceRobotTrainer:
|
||||
"""语音机器人训练器"""
|
||||
|
||||
def __init__(self, config: TrainingConfig):
|
||||
self.config = config
|
||||
self.device = torch.device(config.device)
|
||||
|
||||
# 创建输出目录
|
||||
os.makedirs(config.output_dir, exist_ok=True)
|
||||
|
||||
# 初始化tokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(config.text_encoder_model)
|
||||
|
||||
# 训练历史
|
||||
self.train_history = {
|
||||
'train_loss': [],
|
||||
'val_loss': [],
|
||||
'joint_loss': [],
|
||||
'pose_loss': [],
|
||||
'gripper_loss': []
|
||||
}
|
||||
|
||||
def load_data(self) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""加载训练数据"""
|
||||
logger.info("加载训练数据...")
|
||||
|
||||
all_samples = []
|
||||
|
||||
# 遍历数据目录中的所有会话
|
||||
data_path = Path(self.config.data_dir)
|
||||
for session_dir in data_path.iterdir():
|
||||
if session_dir.is_dir():
|
||||
training_data_path = session_dir / "training_data.json"
|
||||
if training_data_path.exists():
|
||||
with open(training_data_path, 'r', encoding='utf-8') as f:
|
||||
session_data = json.load(f)
|
||||
all_samples.extend(session_data)
|
||||
|
||||
if not all_samples:
|
||||
raise ValueError("未找到训练数据,请先运行数据采集和标注")
|
||||
|
||||
logger.info(f"加载了 {len(all_samples)} 个训练样本")
|
||||
|
||||
# 划分训练集和验证集
|
||||
train_samples, val_samples = train_test_split(
|
||||
all_samples,
|
||||
test_size=self.config.validation_split,
|
||||
random_state=42
|
||||
)
|
||||
|
||||
logger.info(f"训练集: {len(train_samples)} 样本")
|
||||
logger.info(f"验证集: {len(val_samples)} 样本")
|
||||
|
||||
return train_samples, val_samples
|
||||
|
||||
def create_datasets(self, train_samples: List[Dict], val_samples: List[Dict]) -> Tuple[VoiceRobotDataset, VoiceRobotDataset]:
|
||||
"""创建数据集"""
|
||||
logger.info("创建数据集...")
|
||||
|
||||
# 创建训练数据集
|
||||
train_dataset = VoiceRobotDataset(train_samples, self.tokenizer)
|
||||
|
||||
# 创建验证数据集(使用训练集的scaler)
|
||||
val_dataset = VoiceRobotDataset(
|
||||
val_samples,
|
||||
self.tokenizer,
|
||||
scaler={
|
||||
'action_scaler': train_dataset.action_scaler,
|
||||
'pose_scaler': train_dataset.pose_scaler,
|
||||
'gripper_scaler': train_dataset.gripper_scaler
|
||||
}
|
||||
)
|
||||
val_dataset.action_scaler = train_dataset.action_scaler
|
||||
val_dataset.pose_scaler = train_dataset.pose_scaler
|
||||
val_dataset.gripper_scaler = train_dataset.gripper_scaler
|
||||
|
||||
# 保存scaler
|
||||
scaler_path = os.path.join(self.config.output_dir, "scalers.pkl")
|
||||
with open(scaler_path, 'wb') as f:
|
||||
pickle.dump({
|
||||
'action_scaler': train_dataset.action_scaler,
|
||||
'pose_scaler': train_dataset.pose_scaler,
|
||||
'gripper_scaler': train_dataset.gripper_scaler
|
||||
}, f)
|
||||
|
||||
return train_dataset, val_dataset
|
||||
|
||||
def create_model(self) -> VoiceRobotModel:
|
||||
"""创建模型"""
|
||||
logger.info("创建模型...")
|
||||
model = VoiceRobotModel(self.config)
|
||||
model.to(self.device)
|
||||
|
||||
# 打印模型信息
|
||||
total_params = sum(p.numel() for p in model.parameters())
|
||||
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
logger.info(f"模型参数总数: {total_params:,}")
|
||||
logger.info(f"可训练参数: {trainable_params:,}")
|
||||
|
||||
return model
|
||||
|
||||
def compute_loss(self, predictions: Dict, targets: Dict) -> Tuple[torch.Tensor, Dict]:
|
||||
"""计算损失"""
|
||||
# 序列损失函数
|
||||
mse_loss = nn.MSELoss()
|
||||
|
||||
# 计算各部分损失
|
||||
joint_loss = 0
|
||||
pose_loss = 0
|
||||
gripper_loss = 0
|
||||
|
||||
batch_size = predictions['joint_positions'].size(0)
|
||||
|
||||
for i in range(batch_size):
|
||||
# 关节位置损失
|
||||
pred_joints = predictions['joint_positions'][i] # [action_dim]
|
||||
target_joints = targets['joint_positions'][i] # [seq_len, action_dim]
|
||||
|
||||
# 使用序列的平均值作为目标
|
||||
target_joints_mean = target_joints.mean(dim=0)
|
||||
joint_loss += mse_loss(pred_joints, target_joints_mean)
|
||||
|
||||
# 末端执行器位姿损失
|
||||
pred_pose = predictions['end_effector_poses'][i] # [pose_dim]
|
||||
target_pose = targets['end_effector_poses'][i] # [seq_len, pose_dim]
|
||||
target_pose_mean = target_pose.mean(dim=0)
|
||||
pose_loss += mse_loss(pred_pose, target_pose_mean)
|
||||
|
||||
# 夹爪状态损失
|
||||
pred_gripper = predictions['gripper_states'][i] # [gripper_dim]
|
||||
target_gripper = targets['gripper_states'][i] # [seq_len, gripper_dim]
|
||||
target_gripper_mean = target_gripper.mean(dim=0)
|
||||
gripper_loss += mse_loss(pred_gripper, target_gripper_mean)
|
||||
|
||||
# 平均损失
|
||||
joint_loss /= batch_size
|
||||
pose_loss /= batch_size
|
||||
gripper_loss /= batch_size
|
||||
|
||||
# 总损失(加权)
|
||||
total_loss = joint_loss + pose_loss + gripper_loss * 0.5
|
||||
|
||||
loss_dict = {
|
||||
'total_loss': total_loss,
|
||||
'joint_loss': joint_loss,
|
||||
'pose_loss': pose_loss,
|
||||
'gripper_loss': gripper_loss
|
||||
}
|
||||
|
||||
return total_loss, loss_dict
|
||||
|
||||
def train_epoch(self, model: VoiceRobotModel, dataloader: DataLoader, optimizer: optim.Optimizer) -> Dict:
|
||||
"""训练一个epoch"""
|
||||
model.train()
|
||||
total_loss = 0
|
||||
total_joint_loss = 0
|
||||
total_pose_loss = 0
|
||||
total_gripper_loss = 0
|
||||
num_batches = 0
|
||||
|
||||
for batch in dataloader:
|
||||
# 移动数据到设备
|
||||
input_ids = batch['input_ids'].to(self.device)
|
||||
attention_mask = batch['attention_mask'].to(self.device)
|
||||
|
||||
targets = {
|
||||
'joint_positions': batch['joint_positions'].to(self.device),
|
||||
'end_effector_poses': batch['end_effector_poses'].to(self.device),
|
||||
'gripper_states': batch['gripper_states'].to(self.device)
|
||||
}
|
||||
|
||||
# 前向传播
|
||||
optimizer.zero_grad()
|
||||
predictions = model(input_ids, attention_mask)
|
||||
|
||||
# 计算损失
|
||||
loss, loss_dict = self.compute_loss(predictions, targets)
|
||||
|
||||
# 反向传播
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
# 累计损失
|
||||
total_loss += loss.item()
|
||||
total_joint_loss += loss_dict['joint_loss'].item()
|
||||
total_pose_loss += loss_dict['pose_loss'].item()
|
||||
total_gripper_loss += loss_dict['gripper_loss'].item()
|
||||
num_batches += 1
|
||||
|
||||
return {
|
||||
'train_loss': total_loss / num_batches,
|
||||
'joint_loss': total_joint_loss / num_batches,
|
||||
'pose_loss': total_pose_loss / num_batches,
|
||||
'gripper_loss': total_gripper_loss / num_batches
|
||||
}
|
||||
|
||||
def validate_epoch(self, model: VoiceRobotModel, dataloader: DataLoader) -> Dict:
|
||||
"""验证一个epoch"""
|
||||
model.eval()
|
||||
total_loss = 0
|
||||
total_joint_loss = 0
|
||||
total_pose_loss = 0
|
||||
total_gripper_loss = 0
|
||||
num_batches = 0
|
||||
|
||||
with torch.no_grad():
|
||||
for batch in dataloader:
|
||||
# 移动数据到设备
|
||||
input_ids = batch['input_ids'].to(self.device)
|
||||
attention_mask = batch['attention_mask'].to(self.device)
|
||||
|
||||
targets = {
|
||||
'joint_positions': batch['joint_positions'].to(self.device),
|
||||
'end_effector_poses': batch['end_effector_poses'].to(self.device),
|
||||
'gripper_states': batch['gripper_states'].to(self.device)
|
||||
}
|
||||
|
||||
# 前向传播
|
||||
predictions = model(input_ids, attention_mask)
|
||||
|
||||
# 计算损失
|
||||
loss, loss_dict = self.compute_loss(predictions, targets)
|
||||
|
||||
# 累计损失
|
||||
total_loss += loss.item()
|
||||
total_joint_loss += loss_dict['joint_loss'].item()
|
||||
total_pose_loss += loss_dict['pose_loss'].item()
|
||||
total_gripper_loss += loss_dict['gripper_loss'].item()
|
||||
num_batches += 1
|
||||
|
||||
return {
|
||||
'val_loss': total_loss / num_batches,
|
||||
'joint_loss': total_joint_loss / num_batches,
|
||||
'pose_loss': total_pose_loss / num_batches,
|
||||
'gripper_loss': total_gripper_loss / num_batches
|
||||
}
|
||||
|
||||
def train(self):
|
||||
"""主训练函数"""
|
||||
logger.info("开始训练...")
|
||||
|
||||
# 加载数据
|
||||
train_samples, val_samples = self.load_data()
|
||||
train_dataset, val_dataset = self.create_datasets(train_samples, val_samples)
|
||||
|
||||
# 创建数据加载器
|
||||
train_dataloader = DataLoader(
|
||||
train_dataset,
|
||||
batch_size=self.config.batch_size,
|
||||
shuffle=True,
|
||||
num_workers=0 # 在Windows上设置为0
|
||||
)
|
||||
val_dataloader = DataLoader(
|
||||
val_dataset,
|
||||
batch_size=self.config.batch_size,
|
||||
shuffle=False,
|
||||
num_workers=0
|
||||
)
|
||||
|
||||
# 创建模型
|
||||
model = self.create_model()
|
||||
|
||||
# 创建优化器
|
||||
optimizer = optim.Adam(model.parameters(), lr=self.config.learning_rate)
|
||||
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5)
|
||||
|
||||
# 早停机制
|
||||
best_val_loss = float('inf')
|
||||
patience_counter = 0
|
||||
|
||||
# 训练循环
|
||||
for epoch in range(self.config.num_epochs):
|
||||
start_time = time.time()
|
||||
|
||||
# 训练
|
||||
train_metrics = self.train_epoch(model, train_dataloader, optimizer)
|
||||
|
||||
# 验证
|
||||
val_metrics = self.validate_epoch(model, val_dataloader)
|
||||
|
||||
# 更新学习率
|
||||
scheduler.step(val_metrics['val_loss'])
|
||||
|
||||
# 记录历史
|
||||
self.train_history['train_loss'].append(train_metrics['train_loss'])
|
||||
self.train_history['val_loss'].append(val_metrics['val_loss'])
|
||||
self.train_history['joint_loss'].append(train_metrics['joint_loss'])
|
||||
self.train_history['pose_loss'].append(train_metrics['pose_loss'])
|
||||
self.train_history['gripper_loss'].append(train_metrics['gripper_loss'])
|
||||
|
||||
# 打印进度
|
||||
epoch_time = time.time() - start_time
|
||||
logger.info(
|
||||
f"Epoch {epoch+1}/{self.config.num_epochs} - "
|
||||
f"Train Loss: {train_metrics['train_loss']:.4f}, "
|
||||
f"Val Loss: {val_metrics['val_loss']:.4f}, "
|
||||
f"Time: {epoch_time:.2f}s"
|
||||
)
|
||||
|
||||
# 保存最佳模型
|
||||
if val_metrics['val_loss'] < best_val_loss:
|
||||
best_val_loss = val_metrics['val_loss']
|
||||
patience_counter = 0
|
||||
|
||||
# 保存模型
|
||||
model_path = os.path.join(self.config.output_dir, "best_model.pth")
|
||||
torch.save({
|
||||
'model_state_dict': model.state_dict(),
|
||||
'config': self.config,
|
||||
'train_history': self.train_history,
|
||||
'epoch': epoch
|
||||
}, model_path)
|
||||
|
||||
logger.info(f"保存最佳模型到 {model_path}")
|
||||
else:
|
||||
patience_counter += 1
|
||||
|
||||
# 早停检查
|
||||
if patience_counter >= self.config.early_stopping_patience:
|
||||
logger.info(f"早停触发,在第 {epoch+1} 轮停止训练")
|
||||
break
|
||||
|
||||
# 保存最终模型
|
||||
final_model_path = os.path.join(self.config.output_dir, "final_model.pth")
|
||||
torch.save({
|
||||
'model_state_dict': model.state_dict(),
|
||||
'config': self.config,
|
||||
'train_history': self.train_history,
|
||||
'epoch': epoch
|
||||
}, final_model_path)
|
||||
|
||||
# 保存训练历史
|
||||
history_path = os.path.join(self.config.output_dir, "training_history.json")
|
||||
with open(history_path, 'w') as f:
|
||||
json.dump(self.train_history, f, indent=2)
|
||||
|
||||
# 绘制训练曲线
|
||||
self.plot_training_curves()
|
||||
|
||||
logger.info("训练完成!")
|
||||
|
||||
def plot_training_curves(self):
|
||||
"""绘制训练曲线"""
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans'] # 支持中文显示
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
|
||||
# 总损失
|
||||
axes[0, 0].plot(self.train_history['train_loss'], label='训练损失')
|
||||
axes[0, 0].plot(self.train_history['val_loss'], label='验证损失')
|
||||
axes[0, 0].set_title('总损失')
|
||||
axes[0, 0].set_xlabel('Epoch')
|
||||
axes[0, 0].set_ylabel('Loss')
|
||||
axes[0, 0].legend()
|
||||
axes[0, 0].grid(True)
|
||||
|
||||
# 关节损失
|
||||
axes[0, 1].plot(self.train_history['joint_loss'], label='关节损失')
|
||||
axes[0, 1].set_title('关节位置损失')
|
||||
axes[0, 1].set_xlabel('Epoch')
|
||||
axes[0, 1].set_ylabel('Loss')
|
||||
axes[0, 1].legend()
|
||||
axes[0, 1].grid(True)
|
||||
|
||||
# 位姿损失
|
||||
axes[1, 0].plot(self.train_history['pose_loss'], label='位姿损失')
|
||||
axes[1, 0].set_title('末端执行器位姿损失')
|
||||
axes[1, 0].set_xlabel('Epoch')
|
||||
axes[1, 0].set_ylabel('Loss')
|
||||
axes[1, 0].legend()
|
||||
axes[1, 0].grid(True)
|
||||
|
||||
# 夹爪损失
|
||||
axes[1, 1].plot(self.train_history['gripper_loss'], label='夹爪损失')
|
||||
axes[1, 1].set_title('夹爪状态损失')
|
||||
axes[1, 1].set_xlabel('Epoch')
|
||||
axes[1, 1].set_ylabel('Loss')
|
||||
axes[1, 1].legend()
|
||||
axes[1, 1].grid(True)
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
# 保存图片
|
||||
plot_path = os.path.join(self.config.output_dir, "training_curves.png")
|
||||
plt.savefig(plot_path, dpi=300, bbox_inches='tight')
|
||||
plt.close()
|
||||
|
||||
logger.info(f"训练曲线已保存到 {plot_path}")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# 创建配置
|
||||
config = TrainingConfig(
|
||||
data_dir="./collected_data",
|
||||
output_dir="./trained_models",
|
||||
batch_size=16, # 根据GPU内存调整
|
||||
num_epochs=50,
|
||||
learning_rate=1e-4
|
||||
)
|
||||
|
||||
# 创建训练器
|
||||
trainer = VoiceRobotTrainer(config)
|
||||
|
||||
# 开始训练
|
||||
try:
|
||||
trainer.train()
|
||||
except Exception as e:
|
||||
logger.error(f"训练过程中出现错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
43
requirements.txt
普通文件
43
requirements.txt
普通文件
@ -0,0 +1,43 @@
|
||||
# 语音机器人训练系统依赖包
|
||||
|
||||
# 核心框架
|
||||
torch>=1.9.0
|
||||
numpy>=1.21.0
|
||||
h5py>=3.1.0
|
||||
|
||||
# LeRobot框架
|
||||
lerobot
|
||||
|
||||
# 数据处理
|
||||
pandas>=1.3.0
|
||||
scipy>=1.7.0
|
||||
opencv-python>=4.5.0
|
||||
pillow>=8.3.0
|
||||
|
||||
# 语音识别
|
||||
SpeechRecognition>=3.8.1
|
||||
pyaudio>=0.2.11
|
||||
pydub>=0.25.1
|
||||
|
||||
# 游戏手柄控制
|
||||
pygame>=2.0.0
|
||||
|
||||
# GUI和可视化
|
||||
matplotlib>=3.4.0
|
||||
|
||||
# Web框架(用于标注工具的Web版本)
|
||||
flask>=2.0.0
|
||||
flask-cors>=3.0.10
|
||||
|
||||
# 机器学习相关
|
||||
scikit-learn>=1.0.0
|
||||
transformers>=4.12.0
|
||||
|
||||
# 数据序列化
|
||||
jsonschema>=3.2.0
|
||||
|
||||
# 开发工具
|
||||
pytest>=6.2.0
|
||||
black>=21.0.0
|
||||
flake8>=3.9.0
|
||||
|
||||
93
setup.py
普通文件
93
setup.py
普通文件
@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
语音输入法机器人训练系统安装脚本
|
||||
"""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
import os
|
||||
|
||||
# 读取README文件
|
||||
def read_readme():
|
||||
with open("README.md", "r", encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
# 读取requirements文件
|
||||
def read_requirements():
|
||||
with open("requirements.txt", "r", encoding="utf-8") as fh:
|
||||
return [line.strip() for line in fh if line.strip() and not line.startswith("#")]
|
||||
|
||||
setup(
|
||||
name="voice-robot-training-system",
|
||||
version="1.0.0",
|
||||
author="Manus AI",
|
||||
author_email="contact@manus.ai",
|
||||
description="基于LeRobot框架的语音输入法机器人训练系统",
|
||||
long_description=read_readme(),
|
||||
long_description_content_type="text/markdown",
|
||||
url="https://github.com/manus-ai/voice-robot-training-system",
|
||||
packages=find_packages(),
|
||||
classifiers=[
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
],
|
||||
python_requires=">=3.8",
|
||||
install_requires=read_requirements(),
|
||||
extras_require={
|
||||
"dev": [
|
||||
"pytest>=6.2.0",
|
||||
"black>=21.0.0",
|
||||
"flake8>=3.9.0",
|
||||
"mypy>=0.910",
|
||||
"pre-commit>=2.15.0",
|
||||
],
|
||||
"docs": [
|
||||
"sphinx>=4.0.0",
|
||||
"sphinx-rtd-theme>=0.5.0",
|
||||
"myst-parser>=0.15.0",
|
||||
],
|
||||
"gpu": [
|
||||
"torch[cuda]>=1.9.0",
|
||||
],
|
||||
},
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"voice-robot-collect=data_collector:main",
|
||||
"voice-robot-annotate=annotation_tool:main",
|
||||
"voice-robot-train=model_trainer:main",
|
||||
"voice-robot-infer=inference_engine:main",
|
||||
"voice-robot-example=examples.example_usage:main",
|
||||
],
|
||||
},
|
||||
include_package_data=True,
|
||||
package_data={
|
||||
"": ["*.md", "*.txt", "*.yml", "*.yaml", "*.json"],
|
||||
},
|
||||
zip_safe=False,
|
||||
keywords=[
|
||||
"robotics",
|
||||
"voice-control",
|
||||
"machine-learning",
|
||||
"deep-learning",
|
||||
"lerobot",
|
||||
"speech-recognition",
|
||||
"human-robot-interaction",
|
||||
"imitation-learning",
|
||||
],
|
||||
project_urls={
|
||||
"Bug Reports": "https://github.com/manus-ai/voice-robot-training-system/issues",
|
||||
"Source": "https://github.com/manus-ai/voice-robot-training-system",
|
||||
"Documentation": "https://voice-robot-training-system.readthedocs.io/",
|
||||
},
|
||||
)
|
||||
|
||||
在新工单中引用
屏蔽一个用户