企微聊天语音聊天接收处理

fs-qw-api-msg/src/main/java/com/fs/app/controller/QwMsgController.java

@@ -2,7 +2,9 @@ package com.fs.app.controller;
 
 import com.alibaba.fastjson.JSON;
 import com.fs.app.socket.QwImSocket;
+import com.fs.app.util.AudioUtils;
 import com.fs.common.core.redis.RedisCache;
+import com.fs.common.utils.StringUtils;
 import com.fs.common.utils.uuid.IdUtils;
 import com.fs.fastGpt.service.AiHookService;
 import com.fs.qw.domain.QwUser;
@@ -182,10 +184,14 @@ public class QwMsgController {
                         aiHookService.qwHookNotifyAddMsg(id,receiver,content,wxWorkMsgResp.getUuid());
                     }
 
-                    // 保存聊天消息
-                    QwMessageListVO message = aiHookService.saveQwMsg(id, userId, content, wxWorkMsgResp.getUuid(), sendType, wxWorkMsgResp.getJson(), 1);
-                    QwImSocket.broadcast(message);
-
+                    // 处理文本消息
+                    if (wxWorkMessageDTO.getMsgtype() == 2 || wxWorkMessageDTO.getMsgtype() == 0) {
+                        processTextMessage(id, userId, content, wxWorkMsgResp, sendType);
+                    }
+                    // 语音消息
+                    if (wxWorkMessageDTO.getMsgtype() == 16) {
+                        processVoiceMessage(serverId, content, wxWorkMessageDTO, wxWorkMsgResp, id, userId, sendType);
+                    }
                 }
                 //语音通话
                 if (wxWorkMessageDTO.getMsgtype()==40){
@@ -242,6 +248,64 @@ public class QwMsgController {
         return map;
     }
 
+    /**
+     * 处理文本消息
+     * @param id                企微用户ID
+     * @param userId            消息发送者ID
+     * @param content           消息内容
+     * @param wxWorkMsgResp     回调信息对象
+     * @param sendType          发送者类型 1客户 2销售
+     */
+    private void processTextMessage(Long id, Long userId, String content, WxWorkMsgResp wxWorkMsgResp, Integer sendType) {
+        // 保存聊天消息
+        QwMessageListVO message = aiHookService.saveQwMsg(id, userId, content, wxWorkMsgResp.getUuid(), sendType, wxWorkMsgResp.getJson(), 1);
+        QwImSocket.broadcast(message);
+    }
+
+    /**
+     * 处理语音消息
+     * @param serverId          服务器ID
+     * @param wxWorkMessageDTO  消息DTO
+     * @param content           翻译后的内容
+     * @param wxWorkMsgResp     回调信息对象
+     * @param id                企微用户ID
+     * @param userId            消息发送者ID
+     * @param sendType          发送者类型 1客户 2销售
+     */
+    private void processVoiceMessage(Long serverId, String content, WxWorkMessageDTO wxWorkMessageDTO, WxWorkMsgResp wxWorkMsgResp, Long id, Long userId, Integer sendType) {
+        String voiceFileName = IdUtils.fastSimpleUUID() + ".silk";
+        WxWorkResponseDTO<String> fileUrlResp =
+                aiHookService.getFileUrl(wxWorkMsgResp.getUuid(), wxWorkMessageDTO.getVoice_id(), wxWorkMessageDTO.getAes_key(), 5, voiceFileName, wxWorkMessageDTO.getVoice_size(), serverId);
+        if (fileUrlResp.getErrcode() != 0) {
+            log.warn("获取语音地址失败: {}", fileUrlResp.getErrmsg());
+            return;
+        }
+
+        // silk转map3
+        String url = AudioUtils.convertSilk2Mp3(fileUrlResp.getData());
+        if (StringUtils.isBlank(url)) {
+            log.warn("转换silk语音格式失败");
+            return;
+        }
+
+        // 转换内容为空时再尝试一次
+        if (StringUtils.isBlank(content)) {
+            WxwSpeechToTextEntityDTO ste = new WxwSpeechToTextEntityDTO();
+            ste.setMsgid(wxWorkMessageDTO.getMsg_id());
+            ste.setUuid(wxWorkMsgResp.getUuid());
+            WxWorkResponseDTO<WxwSpeechToTextEntityRespDTO> dto = wxWorkService.SpeechToTextEntity(ste, serverId);
+            content = dto.getData().getText();
+        }
+
+        JSONObject json = new JSONObject();
+        json.put("url", url);
+        json.put("content", content);
+
+        // 保存聊天消息
+        QwMessageListVO message = aiHookService.saveQwMsg(id, userId, json.toString(), wxWorkMsgResp.getUuid(), sendType, wxWorkMsgResp.getJson(), 4);
+        QwImSocket.broadcast(message);
+    }
+
     /**
      * 处理图片消息
      * @param serverId          服务器ID

fs-qw-api-msg/src/main/java/com/fs/app/util/AudioUtils.java

@@ -0,0 +1,160 @@
+package com.fs.app.util;
+
+import com.fs.common.utils.uuid.IdUtils;
+import com.fs.system.oss.CloudStorageService;
+import com.fs.system.oss.OSSFactory;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.springframework.http.HttpStatus;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@Slf4j
+public class AudioUtils {
+
+    /**
+     * silk转换为mp3
+     * @param silkUrl silk语音链接地址
+     * @return  mp3链接地址
+     */
+    public static String convertSilk2Mp3(String silkUrl) {
+        String uniqueId = IdUtils.fastSimpleUUID();
+        Path uploadDirPath = Paths.get(System.getProperty("java.io.tmpdir"), "/");
+        Path downloadedSilkFilePath = uploadDirPath.resolve(uniqueId + ".silk");
+        Path pcmFilePath = uploadDirPath.resolve(uniqueId + ".pcm");
+        Path mp3FilePath = uploadDirPath.resolve(uniqueId + ".mp3");
+
+        try {
+            // 1. 从网络下载 SILK 文件
+            downloadFile(silkUrl, downloadedSilkFilePath);
+            log.info("SILK file downloaded to: {}", downloadedSilkFilePath);
+
+            // 2. 使用 silk-v3-decoder 解码 SILK 到 PCM
+            List<String> silkDecodeCommand = new ArrayList<>();
+            silkDecodeCommand.add("silk_v3_decoder");
+            silkDecodeCommand.add(downloadedSilkFilePath.toString());
+            silkDecodeCommand.add(pcmFilePath.toString());
+
+            ProcessBuilder silkDecoderPb = new ProcessBuilder(silkDecodeCommand);
+            silkDecoderPb.redirectErrorStream(true); // 将错误流合并到标准输出
+            Process silkDecoderProcess = silkDecoderPb.start();
+
+            String silkDecoderOutput = readInputStreamToString(silkDecoderProcess.getInputStream());
+
+            boolean silkDecoderExited = silkDecoderProcess.waitFor(60, TimeUnit.SECONDS);
+            if (!silkDecoderExited || silkDecoderProcess.exitValue() != 0) {
+                log.error("silk conversion failed or timed out. error: {}", silkDecoderOutput);
+                return null;
+            }
+            log.info("SILK decoder to PCM successfully.");
+
+            // 3. 使用 FFmpeg 将 PCM 转码为 MP3
+            Process ffmpegProcess = getFfmpegProcess(pcmFilePath, mp3FilePath);
+            String ffmpegOutput = readInputStreamToString(ffmpegProcess.getInputStream());
+
+            boolean ffmpegExited = ffmpegProcess.waitFor(120, TimeUnit.SECONDS);
+            if (!ffmpegExited || ffmpegProcess.exitValue() != 0) {
+                log.error("ffmpeg conversion failed or timed out. error: {}", ffmpegOutput);
+                return null;
+            }
+            log.info("ffmpeg conversion to MP3 successfully.");
+
+            // 4. 上传oss
+            String fileName = mp3FilePath.getFileName().toString();
+            String suffix = fileName.substring(fileName.lastIndexOf("."));
+            CloudStorageService storage = OSSFactory.build();
+            return storage.uploadSuffix(Files.newInputStream(mp3FilePath), suffix);
+
+        } catch (IOException | InterruptedException | NullPointerException e) {
+            log.error("Conversion error: {}", e.getMessage());
+            return null;
+        } finally {
+            // 清理临时文件 (重要!)
+            try {
+                if (Files.exists(downloadedSilkFilePath)) Files.delete(downloadedSilkFilePath);
+                if (Files.exists(pcmFilePath)) Files.delete(pcmFilePath);
+                if (Files.exists(mp3FilePath)) Files.delete(mp3FilePath);
+            } catch (IOException e) {
+                log.error("Error cleaning up temporary files:: {}", e.getMessage());
+            }
+        }
+    }
+
+    /**
+     * 执行ffmpeg
+     * @param pcmFilePath   pcm文件
+     * @param mp3FilePath   mp3地址
+     * @return  process
+     * @throws IOException exception
+     */
+    private static Process getFfmpegProcess(Path pcmFilePath, Path mp3FilePath) throws IOException {
+        List<String> ffmpegCommand = new ArrayList<>();
+        ffmpegCommand.add("ffmpeg");
+        ffmpegCommand.add("-y");
+        ffmpegCommand.add("-f");
+        ffmpegCommand.add("s16le");
+        ffmpegCommand.add("-ar");
+        ffmpegCommand.add("24000"); // 注意：这里假设是 24kHz，如果你的 SILK 文件是其他采样率，请调整
+        ffmpegCommand.add("-ac");
+        ffmpegCommand.add("1");
+        ffmpegCommand.add("-i");
+        ffmpegCommand.add(pcmFilePath.toString());
+        ffmpegCommand.add(mp3FilePath.toString());
+
+        ProcessBuilder ffmpegPb = new ProcessBuilder(ffmpegCommand);
+        ffmpegPb.redirectErrorStream(true);
+        return ffmpegPb.start();
+    }
+
+    /**
+     * 处理文件流
+     * @param is 输入流
+     * @return  输出
+     * @throws IOException exception
+     */
+    private static String readInputStreamToString(InputStream is) throws IOException {
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        byte[] buffer = new byte[1024];
+        int len;
+        while ((len = is.read(buffer)) != -1) {
+            bos.write(buffer, 0, len);
+        }
+        return bos.toString("UTF-8"); // 使用 UTF-8 编码
+    }
+
+    /**
+     * 下载网络文件
+     * @param fileUrl       网络文件
+     * @param destination   临时文件
+     * @throws IOException  exception
+     */
+    private static void downloadFile(String fileUrl, Path destination) throws IOException {
+        try (CloseableHttpClient httpClient = HttpClients.createDefault();
+             CloseableHttpResponse response = httpClient.execute(new HttpGet(fileUrl));
+             InputStream inputStream = response.getEntity().getContent();
+             FileOutputStream outputStream = new FileOutputStream(destination.toFile())) {
+
+            if (response.getStatusLine().getStatusCode() != HttpStatus.OK.value()) {
+                throw new IOException("Failed to download file from " + fileUrl + ", HTTP Status: " + response.getStatusLine().getStatusCode());
+            }
+
+            byte[] buffer = new byte[4096];
+            int bytesRead;
+            while ((bytesRead = inputStream.read(buffer)) != -1) {
+                outputStream.write(buffer, 0, bytesRead);
+            }
+        }
+    }
+}

fs-service-system/src/main/java/com/fs/fastGpt/service/AiHookService.java

@@ -28,7 +28,7 @@ public interface AiHookService {
      * @param uuid     UUID
      * @param sendType 发送者类型 1用户 2客服
      * @param json     消息json
-     * @param msgType  消息类型 1文本 2图片 3动态表情
+     * @param msgType  消息类型 1文本 2图片 3动态表情 4语音
      */
     QwMessageListVO saveQwMsg(Long qwUserId, Long userId, String content, String uuid, int sendType, String json, int msgType);
 
@@ -44,4 +44,17 @@ public interface AiHookService {
      * @return  WxWorkResponseDTO
      */
     WxWorkResponseDTO<String> getFileUrl(String uuid, String fileId, String aesKey, String authKey, String fileName, Integer fileSize, Long serverId);
+
+    /**
+     * 获取文件地址
+     * @param uuid      uuid
+     * @param fileId    fileId
+     * @param aesKey    aesKey
+     * @param fileType  fileType
+     * @param fileName  fileName
+     * @param fileSize  fileSize
+     * @param serverId  serverId
+     * @return  WxWorkResponseDTO
+     */
+    WxWorkResponseDTO<String> getFileUrl(String uuid, String fileId, String aesKey, Integer fileType, String fileName, Integer fileSize, Long serverId);
 }

fs-service-system/src/main/java/com/fs/fastGpt/service/impl/AiHookServiceImpl.java

@@ -1269,7 +1269,7 @@ public class AiHookServiceImpl implements AiHookService {
      * @param uuid     UUID
      * @param sendType 发送者类型 1用户 2客服
      * @param json     消息json
-     * @param msgType  消息类型 1文本 2图片 3动态表情
+     * @param msgType  消息类型 1文本 2图片 3动态表情 4语音
      */
     @Transactional(rollbackFor = Exception.class)
     @Override
@@ -1349,6 +1349,8 @@ public class AiHookServiceImpl implements AiHookService {
             type = "image";
         } else if (msgType == 3) {
             type = "emotionDynamic";
+        } else if (msgType == 4) {
+            type = "voice";
         }
         listVO.setType(type);
         listVO.setStatus("succeed");
@@ -1383,6 +1385,29 @@ public class AiHookServiceImpl implements AiHookService {
         return wxWorkService.downloadWeChatFile(weChatFileDTO, serverId);
     }
 
+    /**
+     * 获取文件地址
+     * @param uuid      uuid
+     * @param fileId    fileId
+     * @param aesKey    aesKey
+     * @param fileType  fileType
+     * @param fileName  fileName
+     * @param fileSize  fileSize
+     * @param serverId  serverId
+     * @return  WxWorkResponseDTO
+     */
+    @Override
+    public WxWorkResponseDTO<String> getFileUrl(String uuid, String fileId, String aesKey, Integer fileType, String fileName, Integer fileSize, Long serverId) {
+        WxDownloadFileDTO downloadFileDTO = new WxDownloadFileDTO();
+        downloadFileDTO.setUuid(uuid);
+        downloadFileDTO.setFileid(fileId);
+        downloadFileDTO.setAes_key(aesKey);
+        downloadFileDTO.setFiletype(fileType);
+        downloadFileDTO.setFile_name(fileName);
+        downloadFileDTO.setSize(fileSize);
+        return wxWorkService.downloadFile(downloadFileDTO, serverId);
+    }
+
     /**
      * 查询外部联系人
      * @param userId    用户ID

fs-service-system/src/main/java/com/fs/qw/service/impl/QwMsgServiceImpl.java

@@ -448,6 +448,8 @@ public class QwMsgServiceImpl extends ServiceImpl<QwMsgMapper, QwMsg> implements
                 listVO.setType("image");
             } else if (qwMsg.getMsgType() == 3) {
                 listVO.setType("emotionDynamic");
+            } else if (qwMsg.getMsgType() == 4) {
+                listVO.setType("voice");
             }
             listVO.setMsgId(qwMsg.getMsgId());
             listVO.setLastContent(qwMsgs.get(0).getContent());
@@ -482,6 +484,8 @@ public class QwMsgServiceImpl extends ServiceImpl<QwMsgMapper, QwMsg> implements
                 type = "image";
             } else if (record.getMsgType() == 3) {
                 type = "emotionDynamic";
+            } else if (record.getMsgType() == 4) {
+                type = "voice";
             }
             listVO.setType(type);
             listVO.setStatus("succeed");

fs-service-system/src/main/java/com/fs/wxwork/dto/WxDownloadFileDTO.java

@@ -0,0 +1,31 @@
+package com.fs.wxwork.dto;
+
+import lombok.Data;
+
+@Data
+public class WxDownloadFileDTO {
+    /**
+     * uuid
+     */
+    private String uuid;
+    /**
+     * 文件id
+     */
+    private String fileid;
+    /**
+     * aes_key
+     */
+    private String aes_key;
+    /**
+     * 下载文件类型1原图 2 中图 3缩略图 4视频 5文件语音
+     */
+    private Integer filetype;
+    /**
+     * 文件名
+     */
+    private String file_name;
+    /**
+     * 文件大小
+     */
+    private Integer size;
+}

fs-service-system/src/main/java/com/fs/wxwork/dto/WxWorkMessageDTO.java

@@ -29,4 +29,8 @@ public class WxWorkMessageDTO {
 
     // 动态表情
     private String url;
+
+    // 语音
+    private String voice_id;
+    private Integer voice_size;
 }

fs-service-system/src/main/java/com/fs/wxwork/service/WxWorkService.java

@@ -222,6 +222,14 @@ public interface WxWorkService {
      */
     WxWorkResponseDTO<String> downloadWeChatFile(WxwDownloadWeChatFileDTO param, Long serverId);
 
+    /**
+     * CDN下载文件重置版本
+     * @param param     参数
+     * @param serverId  服务器ID
+     * @return  WxWorkResponseDTO
+     */
+    WxWorkResponseDTO<String> downloadFile(WxDownloadFileDTO param, Long serverId);
+
     /**
      * CDN上传网络图片
      * @param param     参数

fs-service-system/src/main/java/com/fs/wxwork/service/WxWorkServiceImpl.java

@@ -291,6 +291,18 @@ public class WxWorkServiceImpl implements WxWorkService {
         return WxWorkHttpUtil.postWithType(url, param, new TypeReference<WxWorkResponseDTO<String>>() {});
     }
 
+    /**
+     * CDN下载文件重置版本
+     * @param param     参数
+     * @param serverId  服务器ID
+     * @return  WxWorkResponseDTO
+     */
+    @Override
+    public WxWorkResponseDTO<String> downloadFile(WxDownloadFileDTO param, Long serverId) {
+        String url = getUrl(serverId) + "/DownloadFile";
+        return WxWorkHttpUtil.postWithType(url, param, new TypeReference<WxWorkResponseDTO<String>>() {});
+    }
+
     /**
      * CDN上传网络图片
      * @param param     参数