优化TX_TTS

sql-scripts/easycallcenter365.sql

@@ -8654,3 +8654,20 @@ INSERT INTO `sys_user_role` VALUES ('6', '2');
 INSERT INTO `sys_user_role` VALUES ('6', '3');
 INSERT INTO `sys_user_role` VALUES ('7', '2');
 INSERT INTO `sys_user_role` VALUES ('100', '2');
+
+-- ----------------------------
+-- Table structure for cc_call_round
+-- ----------------------------
+CREATE TABLE `cc_call_round` (
+  `id` varchar(80) NOT NULL COMMENT 'primary key: uuid-roundNo',
+  `uuid` varchar(50) NOT NULL DEFAULT '' COMMENT 'call UUID',
+  `round_no` int NOT NULL DEFAULT '0' COMMENT 'round number (1-based)',
+  `asr_text` varchar(500) NOT NULL DEFAULT '' COMMENT 'ASR recognized text',
+  `llm_text` varchar(500) NOT NULL DEFAULT '' COMMENT 'LLM response text',
+  `asr_cost_ms` bigint NOT NULL DEFAULT '0' COMMENT 'ASR duration (ms)',
+  `llm_cost_ms` bigint NOT NULL DEFAULT '0' COMMENT 'LLM duration (ms)',
+  `tts_cost_ms` bigint NOT NULL DEFAULT '0' COMMENT 'TTS playback duration (ms)',
+  `create_time` bigint NOT NULL DEFAULT '0' COMMENT 'record create timestamp',
+  PRIMARY KEY (`id`),
+  KEY `idx_uuid` (`uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='per-round call timing statistics';

src/main/java/com/telerobot/fs/robot/RobotBase.java

@@ -166,6 +166,36 @@ public abstract class RobotBase implements IEslEventListener {
      */
     protected volatile long playbackEndTime;
 
+    /**
+     *  每轮耗时统计: ASR等待耗时(ms)
+     */
+    protected volatile long roundAsrCostMs;
+
+    /**
+     *  每轮耗时统计: LLM耗时(ms)
+     */
+    protected volatile long roundLlmCostMs;
+
+    /**
+     *  每轮耗时统计: TTS播报耗时(ms)
+     */
+    protected volatile long roundTtsCostMs;
+
+    /**
+     *  ASR等待开始时间戳
+     */
+    protected volatile long asrWaitStartTime;
+
+    /**
+     *  每轮 ASR 识别文本
+     */
+    protected volatile String roundAsrText;
+
+    /**
+     *  每轮 LLM 回复文本
+     */
+    protected volatile String roundLlmText;
+
     /**
      *  检测到讲话开始事件之后，最大等待语音识别结果的超时时长;
      */

src/main/java/com/telerobot/fs/robot/RobotChat.java

@@ -259,6 +259,9 @@ public class RobotChat extends RobotBase {
             if(EventNames.PLAYBACK_START.equalsIgnoreCase(detail)) {
                 chatRobot.setTtsChannelState(TtsChannelState.OPENED);
                 chatRobot.flushTtsRequestQueue();
+                if (playbackStartTime <= 0) {
+                    playbackStartTime = System.currentTimeMillis();
+                }
                 long timeSpent = System.currentTimeMillis() - playbackStartTime;
                 logger.info("{} PLAYBACK_START event,  time cost = {} ms. ", getTraceId(), timeSpent);
             }
@@ -423,11 +426,17 @@ public class RobotChat extends RobotBase {
                ttsChannelClosed = true;
                recvPlayBackEndEvent = true;
                playbackEndTime = System.currentTimeMillis();
+               if (playbackStartTime > 0) {
+                   roundTtsCostMs = playbackEndTime - playbackStartTime;
+               }
                releasePlayBackFinishedSignal();
            }
            if("Speech-Open".equalsIgnoreCase(event)){
                chatRobot.setTtsChannelState(TtsChannelState.OPENED);
                chatRobot.flushTtsRequestQueue();
+               if (playbackStartTime <= 0) {
+                   playbackStartTime = System.currentTimeMillis();
+               }
                long timeSpent = System.currentTimeMillis() - playbackStartTime;
                logger.info("{} Speech-Open event,  time cost = {} ms. ", getTraceId(), timeSpent);
            }
@@ -559,6 +568,10 @@ public class RobotChat extends RobotBase {
 
                 if (!StringUtil.isNullOrEmpty(asrResponse)) {
                     asrResultEx.add(asrResponse);
+                    roundAsrText = asrResponse;
+                    if (asrWaitStartTime > 0) {
+                        roundAsrCostMs = System.currentTimeMillis() - asrWaitStartTime;
+                    }
                     // #region debug-point xfyun-asr-no-response-asr-cache
                     logger.info("{} dbg_asr_event cached vad result, queueSize={}, response={}",
                             getTraceId(),
@@ -573,6 +586,11 @@ public class RobotChat extends RobotBase {
                     interruptRobotSpeech();
                     releasePlayBackFinishedSignal();
                     ThreadUtil.sleep(100);
+                } else if(chatRobot.getAccount().interruptFlag == 2 && !recvPlayBackEndEvent) {
+                    logger.info("{} interruptFlag=2 barge-in detected by vad event, interrupt current robot speech.", getTraceId());
+                    interruptRobotSpeech();
+                    releasePlayBackFinishedSignal();
+                    ThreadUtil.sleep(100);
                 } else if(chatRobot.getAccount().interruptFlag == 1 && !recvPlayBackEndEvent) {
                     if (checkSpeechInterrupt(asrResponse)) {
                         interruptRobotSpeech();
@@ -685,6 +703,10 @@ public class RobotChat extends RobotBase {
                     try {
                         String tmpResult = URLDecoder.decode(speechResult,"utf-8").replace(" ","");
                         asrResultEx.add(tmpResult);
+                        roundAsrText = tmpResult;
+                        if (asrWaitStartTime > 0) {
+                            roundAsrCostMs = System.currentTimeMillis() - asrWaitStartTime;
+                        }
                         logger.info("{} kaldi asr response: {}",getTraceId(), tmpResult);
                     } catch (Throwable e) {
                         logger.error("{} URLDecoder.decode Error: {}", getTraceId(), speechResult);
@@ -753,6 +775,35 @@ public class RobotChat extends RobotBase {
         if (checkCallSession()) {
             return;
         }
+        // print round timing summary for previous round and save to database
+        if (roundAsrCostMs > 0 || roundLlmCostMs > 0 || roundTtsCostMs > 0) {
+            long roundTotalMs = roundAsrCostMs + roundLlmCostMs + roundTtsCostMs;
+            logger.info("{} round timing summary: ASR={}ms, LLM={}ms, TTS={}ms, total={}ms",
+                    getTraceId(), roundAsrCostMs, roundLlmCostMs, roundTtsCostMs, roundTotalMs);
+            try {
+                com.telerobot.fs.entity.po.CallRound callRound = new com.telerobot.fs.entity.po.CallRound();
+                long roundNo = talkRound.sum();
+                callRound.setId(uuid + "-" + roundNo);
+                callRound.setUuid(uuid);
+                callRound.setRoundNo((int) roundNo);
+                callRound.setAsrText(roundAsrText != null ? roundAsrText : "");
+                callRound.setLlmText(roundLlmText != null ? roundLlmText : "");
+                callRound.setAsrCostMs(roundAsrCostMs);
+                callRound.setLlmCostMs(roundLlmCostMs);
+                callRound.setTtsCostMs(roundTtsCostMs);
+                callRound.setCreateTime(System.currentTimeMillis());
+                com.telerobot.fs.service.CallRoundService callRoundService =
+                        com.telerobot.fs.config.AppContextProvider.getBean(com.telerobot.fs.service.CallRoundService.class);
+                callRoundService.insertCallRound(callRound);
+            } catch (Throwable e) {
+                logger.warn("{} failed to save call round timing: {}", getTraceId(), e.getMessage());
+            }
+            roundAsrCostMs = 0;
+            roundLlmCostMs = 0;
+            roundTtsCostMs = 0;
+            roundAsrText = null;
+            roundLlmText = null;
+        }
         // #region debug-point xfyun-asr-no-response-interact-enter
         logger.info("{} dbg_interact enter, asrQueueSize={}, recvPlayBackEndEvent={}, ttsChannelClosed={}, inSpeaking={}, noVoiceCounter={}, transferToAgent={}, keepAiDuringTransferWait={}, manualAnsweredTime={}, isReleased={}",
                 getTraceId(),
@@ -794,6 +845,7 @@ public class RobotChat extends RobotBase {
 
         // 识别开始时间
         Long startTime = System.currentTimeMillis();
+        playbackStartTime = 0; // will be set on Speech-Open or first TTS event
         LlmAiphoneRes aiphoneRes;
 
             try {
@@ -855,6 +907,8 @@ public class RobotChat extends RobotBase {
 
                 talkRound.increment();
                 Long spentCost = System.currentTimeMillis() - startTime;
+                roundLlmCostMs = spentCost;
+                roundLlmText = aiphoneRes.getBody();
                 logger.info("{}  talkWithLargeModel spent time:  {}  ms, aiphoneRes = {}",
                         getTraceId(), spentCost, JSON.toJSONString(aiphoneRes)
                 );
@@ -1176,6 +1230,7 @@ public class RobotChat extends RobotBase {
         }
 
         long startWaitTimeMills = System.currentTimeMillis();
+        asrWaitStartTime = startWaitTimeMills;
         logger.info("{} wait for customer speaking  ...", getTraceId());
 
         Integer maxWaitTimeCustomerSpeaking = Integer.parseInt(SystemConfig.getValue("max-wait-time-customer-speaking", "7000")) ;