Skip to content

Commit 59ca7f3

Browse files
joey-zhoujoey-zhou
committed
refactor:tts音频合并逻辑处理
1 parent bdf672f commit 59ca7f3

File tree

10 files changed

+79
-45
lines changed

10 files changed

+79
-45
lines changed

‎src/main/java/com/xiaozhi/common/exception/GlobalExceptionHandler.java‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.springframework.web.bind.annotation.ResponseStatus;
88
import org.springframework.web.bind.annotation.RestControllerAdvice;
99
import org.springframework.web.context.request.WebRequest;
10+
import org.springframework.web.servlet.resource.NoResourceFoundException;
1011

1112
import com.xiaozhi.common.web.AjaxResult;
1213

@@ -25,7 +26,7 @@ public class GlobalExceptionHandler {
2526
@ExceptionHandler(UsernameNotFoundException.class)
2627
@ResponseStatus(HttpStatus.BAD_REQUEST)
2728
public AjaxResult handleUsernameNotFoundException(UsernameNotFoundException e, WebRequest request) {
28-
logger.error(e.getMessage(), e);
29+
logger.warn("用户名不存在异常: {}", e.getMessage(), e);
2930
return AjaxResult.error("用户名不存在");
3031
}
3132

@@ -35,17 +36,37 @@ public AjaxResult handleUsernameNotFoundException(UsernameNotFoundException e, W
3536
@ExceptionHandler(UserPasswordNotMatchException.class)
3637
@ResponseStatus(HttpStatus.BAD_REQUEST)
3738
public AjaxResult handleUserPasswordNotMatchException(UserPasswordNotMatchException e, WebRequest request) {
38-
logger.error(e.getMessage(), e);
39+
logger.warn("用户密码不匹配异常: {}", e.getMessage(), e);
3940
return AjaxResult.error("用户密码不正确");
4041
}
4142

4243
/**
43-
* 系统异常
44+
* 静态资源找不到异常
45+
*/
46+
@ExceptionHandler(NoResourceFoundException.class)
47+
@ResponseStatus(HttpStatus.NOT_FOUND)
48+
public AjaxResult handleNoResourceFoundException(NoResourceFoundException e, WebRequest request) {
49+
logger.warn("静态资源找不到: {}", e.getResourcePath());
50+
return AjaxResult.error(HttpStatus.NOT_FOUND.value(), "请求的资源不存在");
51+
}
52+
53+
/**
54+
* 业务异常处理
55+
*/
56+
@ExceptionHandler(RuntimeException.class)
57+
@ResponseStatus(HttpStatus.BAD_REQUEST)
58+
public AjaxResult handleRuntimeException(RuntimeException e, WebRequest request) {
59+
logger.error("业务异常: {}", e.getMessage(), e);
60+
return AjaxResult.error("操作失败：" + e.getMessage());
61+
}
62+
63+
/**
64+
* 系统异常 - 作为最后的兜底处理
4465
*/
4566
@ExceptionHandler(Exception.class)
4667
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
4768
public AjaxResult handleException(Exception e, WebRequest request) {
48-
logger.error(e.getMessage(), e);
69+
logger.error("系统异常: {}", e.getMessage(), e);
4970
return AjaxResult.error("服务器错误，请联系管理员");
5071
}
51-
}
72+
}

‎src/main/java/com/xiaozhi/communication/common/ChatSession.java‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import java.time.LocalDateTime;
1818
import java.time.ZoneId;
1919
import java.time.format.DateTimeFormatter;
20+
import java.time.temporal.ChronoUnit;
2021
import java.util.HashMap;
2122
import java.util.List;
2223
import java.util.Map;
@@ -84,7 +85,7 @@ public abstract class ChatSession {
8485
*/
8586
protected final ConcurrentHashMap<String, Object> attributes = new ConcurrentHashMap<>();
8687

87-
//--------------------设备mcp-------------------------
88+
// --------------------设备mcp-------------------------
8889
private DeviceMcpHolder deviceMcpHolder = new DeviceMcpHolder();
8990

9091
public ChatSession(String sessionId) {
@@ -100,49 +101,51 @@ public Object getAttribute(String key) {
100101
return attributes.get(key);
101102
}
102103

103-
public void setAssistantTimeMillis(Long assistantTimeMillis){
104+
public void setAssistantTimeMillis(Long assistantTimeMillis) {
104105
setAttribute("assistantTimeMillis", assistantTimeMillis);
105106
}
106107

107-
public Long getAssistantTimeMillis(){
108+
public Long getAssistantTimeMillis() {
108109
return (Long) getAttribute("assistantTimeMillis");
109110
}
110111

111-
public void setUserTimeMillis(Long userTimeMillis){
112+
public void setUserTimeMillis(Long userTimeMillis) {
112113
setAttribute("userTimeMillis", userTimeMillis);
113114
}
114115

115-
public Long getUserTimeMillis(){
116+
public Long getUserTimeMillis() {
116117
return (Long) getAttribute("userTimeMillis");
117118
}
118119

119120
/**
120121
* 音频文件约定路径为：audio/{device-id}/{role-id}/{timestamp}-user.wav
121122
* {device-id}/{role-id}/{timestamp}-user 能确定唯一性，不会有并发的麻烦。
122123
* 除非多设备在嵌入式软件里强行修改mac地址（deviceId目前是基于mac地址的)
124+
*
123125
* @param who
124126
* @return
125127
*/
126-
private Path getAudioPath(String who, Long timeMillis ){
128+
private Path getAudioPath(String who, Long timeMillis) {
129+
130+
Instant instant = Instant.ofEpochMilli(timeMillis).truncatedTo(ChronoUnit.SECONDS);
127131

128-
Instant instant = Instant.ofEpochMilli(timeMillis);
129132
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
130-
String datetime = localDateTime.format(DateTimeFormatter.ISO_DATE_TIME).replace(":","");
133+
String datetime = localDateTime.format(DateTimeFormatter.ISO_DATE_TIME).replace(":", "");
131134
SysDevice device = this.getSysDevice();
132135
// 判断设备ID是否有不适合路径的特殊字符，它很可能是mac地址需要转换。
133-
String deviceId = device.getDeviceId().replace(":","-");
136+
String deviceId = device.getDeviceId().replace(":", "-");
134137
String roleId = device.getRoleId().toString();
135-
String filename = "%s-%s.wav".formatted(datetime,who);
136-
Path path = Path.of(AudioUtils.AUDIO_PATH,deviceId,roleId,filename);
138+
String filename = "%s-%s.wav".formatted(datetime, who);
139+
Path path = Path.of(AudioUtils.AUDIO_PATH, deviceId, roleId, filename);
137140
return path;
138141
}
139142

140-
public Path getUserAudioPath(){
141-
return getAudioPath("user",this.getUserTimeMillis());
143+
public Path getUserAudioPath() {
144+
return getAudioPath("user", this.getUserTimeMillis());
142145
}
143146

144147
public Path getAssistantAudioPath() {
145-
return getAudioPath("assistant",getAssistantTimeMillis());
148+
return getAudioPath("assistant", getAssistantTimeMillis());
146149
}
147150

148151
public ToolsSessionHolder getFunctionSessionHolder() {
@@ -157,15 +160,16 @@ public List<ToolCallback> getToolCallbacks() {
157160
return toolsSessionHolder.getAllFunction();
158161
}
159162

160-
161-
162163
/**
163164
* 会话连接是否打开中
165+
*
164166
* @return
165167
*/
166168
public abstract boolean isOpen();
169+
167170
/**
168171
* 音频通道是否打开可用
172+
*
169173
* @return
170174
*/
171175
public abstract boolean isAudioChannelOpen();
@@ -179,6 +183,7 @@ public List<ToolCallback> getToolCallbacks() {
179183
/**
180184
* 设置 Conversation，需要与当前活跃角色一致。
181185
* 当切换角色时，会释放当前 Conversation，并新建一个对应于新角色的Conversation。
186+
*
182187
* @param conversation
183188
*/
184189
public void setConversation(Conversation conversation) {
@@ -187,6 +192,7 @@ public void setConversation(Conversation conversation) {
187192

188193
/**
189194
* 获取与当前活跃角色一致的 Conversation。
195+
*
190196
* @return
191197
*/
192198
public Conversation getConversation() {

‎src/main/java/com/xiaozhi/communication/common/MessageHandler.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,7 @@ private void handleListenMessage(ChatSession chatSession, ListenMessage message)
290290
logger.info("停止监听");
291291

292292
// 关闭音频流
293+
sessionManager.completeAudioStream(sessionId);
293294
sessionManager.closeAudioStream(sessionId);
294295
sessionManager.setStreamingState(sessionId, false);
295296
// 重置VAD会话

‎src/main/java/com/xiaozhi/communication/server/websocket/WebSocketHandler.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -76,25 +76,15 @@ public void afterConnectionEstablished(WebSocketSession session) {
7676
@Override
7777
protected void handleTextMessage(WebSocketSession session, TextMessage message) {
7878
String sessionId = session.getId();
79-
String deviceId = sessionManager.getDeviceConfig(sessionId).getDeviceId();
80-
SysDevice device = deviceService.selectDeviceById(deviceId);
79+
SysDevice device = sessionManager.getDeviceConfig(sessionId);
8180
String payload = message.getPayload();
82-
if (device == null) {
83-
deviceId = getHeadersFromSession(session).get("device-id");
84-
if (deviceId == null) {
85-
logger.error("无法确定设备ID");
86-
return;
87-
} else {
88-
device = deviceService.selectDeviceById(deviceId);
89-
}
90-
}
9181

9282
try {
9383
var msg = JsonUtil.fromJson(payload, Message.class);
9484
if (Objects.requireNonNull(msg) instanceof HelloMessage m) {
9585
handleHelloMessage(session, m);
9686
} else {
97-
if (device.getRoleId() == null) {
87+
if (device == null || device.getRoleId() == null) {
9888
// 设备未绑定，处理未绑定设备的消息
9989
messageHandler.handleUnboundDevice(sessionId, device);
10090
}
@@ -179,7 +169,10 @@ private void handleHelloMessage(WebSocketSession session, HelloMessage message)
179169
//如果客户端开启mcp协议，异步初始化MCP工具
180170
ChatSession chatSession = sessionManager.getSession(sessionId);
181171
Thread.startVirtualThread(() -> {
182-
deviceMcpService.initialize(chatSession);
172+
SysDevice device = sessionManager.getDeviceConfig(sessionId);
173+
if (device.getRoleId() != null) {
174+
deviceMcpService.initialize(chatSession);
175+
}
183176
});
184177
}
185178
} catch (Exception e) {

‎src/main/java/com/xiaozhi/dialogue/llm/ChatService.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -313,10 +313,8 @@ public void onToken(String token) {
313313
public void onComplete(String toolName) {
314314
// 检查该会话是否已完成处理
315315
// 处理当前缓冲区剩余的内容（如果有）
316-
logger.info("onComplete:currentSentence.length {}, text:{}", currentSentence.length(), currentSentence.toString());
317316
if (currentSentence.length() > 0 && containsSubstantialContent(currentSentence.toString())
318317
&& !finalSentenceSent.get()) {
319-
logger.info("进了第一层，句子数：{}", sentenceCount.get());
320318
String sentence = currentSentence.toString().trim();
321319
boolean isFirst = sentenceCount.get() == 0;
322320
boolean isLast = true; // 这是最后一个句子
@@ -325,7 +323,6 @@ public void onComplete(String toolName) {
325323
sentenceCount.incrementAndGet();
326324
finalSentenceSent.set(true);
327325
} else if (!finalSentenceSent.get()) {
328-
logger.info("进了第二层，句子数：{}", sentenceCount.get());
329326
// 如果没有剩余内容但也没有发送过最后一个句子，发送一个空的最后句子标记
330327
// 这确保即使没有剩余内容，也会发送最后一个句子标记
331328
boolean isFirst = sentenceCount.get() == 0;

‎src/main/java/com/xiaozhi/dialogue/llm/memory/DatabaseChatMemory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.xiaozhi.dialogue.llm.memory;
22

33
import com.xiaozhi.common.web.PageFilter;
4-
import com.xiaozhi.dialogue.tts.factory.TtsServiceFactory;
54
import com.xiaozhi.entity.Base;
65
import com.xiaozhi.entity.SysDevice;
76
import com.xiaozhi.entity.SysMessage;
@@ -13,6 +12,8 @@
1312
import org.springframework.beans.factory.annotation.Autowired;
1413
import org.springframework.stereotype.Service;
1514

15+
import java.time.Instant;
16+
import java.time.temporal.ChronoUnit;
1617
import java.util.ArrayList;
1718
import java.util.Comparator;
1819
import java.util.Date;
@@ -54,7 +55,8 @@ public void addMessage(String deviceId, String sessionId, String sender, String
5455
message.setMessage(content);
5556
message.setRoleId(roleId);
5657
message.setMessageType(messageType);
57-
message.setCreateTime(new Date(timeMillis));
58+
Instant instant = Instant.ofEpochMilli(timeMillis).truncatedTo(ChronoUnit.SECONDS);
59+
message.setCreateTime(Date.from(instant));
5860
messageService.add(message);
5961
} catch (Exception e) {
6062
logger.error("保存消息时出错: {}", e.getMessage(), e);

‎src/main/java/com/xiaozhi/dialogue/service/DialogueService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,7 @@ private void handleSentence(
502502

503503
// 添加到句子队列
504504
CopyOnWriteArrayList<Sentence> queue = sentenceQueue.get(sessionId);
505+
if (queue == null) {return;}
505506
queue.add(sentence);
506507

507508
// 如果句子为空且是结束状态，直接标记为准备好（不需要生成音频）

‎src/main/java/com/xiaozhi/dialogue/service/VadService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public class VadService {
4444
private AutomaticGainControl agc;
4545

4646
// 语音检测前缓冲时长(毫秒)
47-
private int preBufferMs = 500;
47+
private int preBufferMs = 300;
4848

4949
// 会话状态和锁
5050
private final ConcurrentHashMap<String, VadState> states = new ConcurrentHashMap<>();
@@ -444,7 +444,7 @@ public VadResult processAudio(String sessionId, byte[] opusData) {
444444
float adjustedSilenceThreshold = adjustVadThreshold(silenceThreshold, agcStats);
445445

446446
// 添加到预缓冲区
447-
// state.addToPreBuffer(pcmData);
447+
state.addToPreBuffer(pcmData);
448448

449449
// 处理短帧数据
450450
if (pcmData.length < MIN_PCM_LENGTH && !state.isSpeaking()) {

‎src/main/java/com/xiaozhi/mapper/MessageMapper.xml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<mapper namespace="com.xiaozhi.dao.MessageMapper">
44

55
<sql id="messageSql">
6-
sys_message.messageId, sys_message.message, sys_message.sender, sys_message.roleId, sys_message.audioPath, sys_message.state, sys_message.createTime, sys_message.messageType
6+
sys_message.messageId, sys_message.message, sys_message.sender, sys_message.roleId, sys_message.state, sys_message.createTime, sys_message.messageType
77
</sql>
88

99
<sql id="deviceSql">
@@ -18,7 +18,18 @@
1818
SELECT
1919
<include refid="messageSql"></include>,
2020
<include refid="deviceSql"></include>,
21-
<include refid="roleSql"></include>
21+
<include refid="roleSql"></include>,
22+
CONCAT(
23+
'audio/',
24+
REPLACE(sys_device.deviceId, ':', '-'),
25+
'/',
26+
sys_message.roleId,
27+
'/',
28+
DATE_FORMAT(sys_message.createTime, '%Y-%m-%dT%H%i%s'),
29+
'-',
30+
sys_message.sender,
31+
'.wav'
32+
) AS audioPath
2233
FROM
2334
sys_message
2435
LEFT JOIN sys_device ON sys_message.deviceId = sys_device.deviceId

‎src/main/java/com/xiaozhi/service/impl/SysDeviceServiceImpl.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,9 +180,11 @@ public int update(SysDevice device) {
180180
int rows = deviceMapper.update(device);
181181
// 更新设备信息后清空记忆缓存并重新注册设备信息
182182
device = deviceMapper.selectDeviceById(device.getDeviceId());
183-
ChatSession session = sessionManager.getSessionByDeviceId(device.getDeviceId());
183+
ChatSession session = null;
184+
if (device != null) {
185+
session = sessionManager.getSessionByDeviceId(device.getDeviceId());
186+
}
184187
if (session != null) {
185-
186188
session.setSysDevice(device);
187189
}
188190
return rows;

0 commit comments

Comments
 (0)