Merge remote-tracking branch 'origin/master'

fs-qw-api-msg/src/main/java/com/fs/app/controller/QwMsgController.java

@@ -334,12 +334,20 @@ public class QwMsgController {
                             ste.setUuid(wxWorkMsgResp.getUuid());
                             WxWorkResponseDTO<WxwSpeechToTextEntityRespDTO> dto = wxWorkService.SpeechToTextEntity(ste, serverId);
                             System.out.println(dto);
-                            if(dto.getErrcode() != 0 || Objects.isNull(dto.getData()) || StringUtils.isBlank(dto.getData().getText())){
+
+                            int maxRetries = 3;
+                            int retryCount = 0;
+
+                            while((dto.getErrcode() != 0 || Objects.isNull(dto.getData()) || StringUtils.isBlank(dto.getData().getText()))
+                                    && retryCount < maxRetries) {
                                 try {
-                                    TimeUnit.SECONDS.sleep(2); // 阻塞2秒
+                                    TimeUnit.SECONDS.sleep(1);
+                                    retryCount++;
+                                    log.info("id:{}, 语音转换第{}次重试", id, retryCount);
                                 } catch (InterruptedException e) {
-                                    Thread.currentThread().interrupt(); // 处理中断异常
-                                    log.info("id:{}, 第一次语音转换失败", id);
+                                    Thread.currentThread().interrupt();
+                                    log.info("id:{}, 语音转换等待被中断", id);
+                                    break;
                                 }
                                 dto = wxWorkService.SpeechToTextEntity(ste, serverId);
                             }

fs-service/src/main/java/com/fs/fastGpt/domain/FastGptChatConversation.java

@@ -1,13 +1,14 @@
 package com.fs.fastGpt.domain;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.Data;
 
 @Data
 public class FastGptChatConversation {
     private JSONObject userInfo;
-    private JSONObject aiInfo;
-    private JSONObject history;
+    private String aiInfo;
+    private JSONArray history;
     private String isRepository;
     private String userContent;
     private String aiContent;

fs-service/src/main/java/com/fs/fastGpt/service/impl/AiHookServiceImpl.java

@@ -615,7 +615,7 @@ public class AiHookServiceImpl implements AiHookService {
                 return R.ok();
             }
             String contentKh = result.getChoices().get(0).getMessage().getContent();
-            String finalContentKh = null;
+            /*String finalContentKh = null;
             JSONArray jsonArray = JSON.parseArray(contentKh);
             for (int i = 0; i < jsonArray.size(); i++) {
                 com.alibaba.fastjson.JSONObject jsonObject = (com.alibaba.fastjson.JSONObject)jsonArray.get(i);
@@ -626,8 +626,9 @@ public class AiHookServiceImpl implements AiHookService {
                 }
             }
             Gson gson = new Gson();
-            contentKh = finalContentKh;
-            FastGptChatConversation fastGptChatConversation = gson.fromJson(finalContentKh, FastGptChatConversation.class);
+            contentKh = finalContentKh;*/
+            Gson gson = new Gson();
+            FastGptChatConversation fastGptChatConversation = gson.fromJson(contentKh, FastGptChatConversation.class);
             String content = fastGptChatConversation.getAiContent();
             //计算token
             List<ChatDetailTStreamFResult.ResponseNode> responseData = result.getResponseData();
@@ -657,23 +658,27 @@ public class AiHookServiceImpl implements AiHookService {
                 if (result.isLongText()){
                     //新增用户信息
                     addUserInfoNew(contentKh, qwExternalContacts.getId(),fastGptChatSession);
-                    if (type==16){
+                    //发送图片消息
+                    sendImgMsg(contentKh,sender,uid,serverId);
+                    /*if (type==16){
                         sendAiVoiceMsg(content,sender,uid,serverId,user);
-                    }else {
+                    }else {*/
                         sendAiMsg(content,sender,uid,serverId);
-                    }
+                    //}
 
                 }else {
 
                     List<String> countList = countString(content);
                     //新增用户信息
                     addUserInfoNew(contentKh, qwExternalContacts.getId(),fastGptChatSession);
+                    //发送图片消息
+                    sendImgMsg(contentKh,sender,uid,serverId);
                     for (String msg : countList) {
-                        if (type==16){
+                        /*if (type==16){
                             sendAiVoiceMsg(msg,sender,uid,serverId,user);
-                        }else {
+                        }else {*/
                             sendAiMsg(msg,sender,uid,serverId);
-                        }
+                        //}
                         try {
                             Thread.sleep(500);
                         } catch (InterruptedException e) {
@@ -699,6 +704,39 @@ public class AiHookServiceImpl implements AiHookService {
         return R.ok();
     }
 
+    private void sendImgMsg(String contentKh, Long sender, String uuid, Long serverId) {
+        com.alibaba.fastjson.JSONObject jsonObject = com.alibaba.fastjson.JSONObject.parseObject(contentKh);
+        JSONArray imgUrls = jsonObject.getJSONArray("imgUrl");
+
+
+        if(imgUrls != null && !imgUrls.isEmpty()){
+            WxWorkResponseDTO<WxwUploadCdnLinkImgRespDTO> dto = new WxWorkResponseDTO<>();
+            for (Object imgUrl : imgUrls) {
+                //1.上传cdn网络图片
+                if(imgUrl != null){
+                    String imgUrlString = imgUrl.toString();
+                    WxwUploadCdnLinkImgDTO wxwUploadCdnLinkImgDTO = new WxwUploadCdnLinkImgDTO();
+                    wxwUploadCdnLinkImgDTO.setUuid(uuid);
+                    wxwUploadCdnLinkImgDTO.setUrl(imgUrlString);
+                    dto  = wxWorkService.uploadCdnLinkImg(wxwUploadCdnLinkImgDTO,serverId);
+                }
+                //图片上传成功后再发送图片
+                if("成功".equals(dto.getErrmsg()) && imgUrl != null){
+                    WxwUploadCdnLinkImgRespDTO imgRespDTO = dto.getData();
+                    WxwSendCDNImgMsgDTO wxwSendCDNImgMsgDTO = new WxwSendCDNImgMsgDTO();
+                    wxwSendCDNImgMsgDTO.setSend_userid(sender);
+                    wxwSendCDNImgMsgDTO.setUuid(uuid);
+                    wxwSendCDNImgMsgDTO.setIsRoom(false);
+                    wxwSendCDNImgMsgDTO.setCdnkey(imgRespDTO.getCdn_key());
+                    wxwSendCDNImgMsgDTO.setAeskey(imgRespDTO.getAes_key());
+                    wxwSendCDNImgMsgDTO.setMd5(imgRespDTO.getMd5());
+                    wxwSendCDNImgMsgDTO.setFileSize(imgRespDTO.getSize());
+                    wxWorkService.SendCDNImgMsg(wxwSendCDNImgMsgDTO, serverId);
+                }
+            }
+        }
+    }
+
     /**
      * 根据发送者id设置用户是否为首次回复
      * @param sender 发送者id
@@ -1540,7 +1578,7 @@ public class AiHookServiceImpl implements AiHookService {
 
             //添加关键词
             addPromptWordNew(messageList,msgC,qwExternalContactsId,role,fastGptChatSession);
-            R r = chatService.initiatingTakeChat(param, "http://1.95.196.10:3000/api/", appKey);
+            R r = chatService.initiatingTakeChat(param, "http://129.28.170.206:3000/api/", appKey);
             Object data1 = r.get("data");
             if(!(data1 instanceof KnowledgeBaseResult)){
                 ChatDetailTStreamFResult data = (ChatDetailTStreamFResult) r.get("data");
@@ -1595,9 +1633,12 @@ public class AiHookServiceImpl implements AiHookService {
     private void addPromptWordNew(List<ChatParam.Message> messageList,String count,Long extId,FastGptRole role,FastGptChatSession fastGptChatSession){
 
         FastGptChatConversation conversation = new FastGptChatConversation();
-        conversation.setAiInfo(new com.alibaba.fastjson.JSONObject());
         conversation.setUserInfo(new com.alibaba.fastjson.JSONObject());
-        conversation.setHistory(new com.alibaba.fastjson.JSONObject());
+        conversation.setHistory(new com.alibaba.fastjson.JSONArray());
+
+        if(role.getReminderWords() != null && !role.getReminderWords().isEmpty()){
+            conversation.setAiInfo(role.getReminderWords());
+        }
 
         //组装客户信息
         String sessionUserInfo = fastGptChatSession.getUserInfo();
@@ -1628,19 +1669,23 @@ public class AiHookServiceImpl implements AiHookService {
 
         List<FastGptChatMsg> msgs=fastGptChatMsgService.selectFastGptChatMsgByMsgSessionIdAndExtId(fastGptChatSession.getSessionId(),extId);
         if (!msgs.isEmpty()){
-            com.alibaba.fastjson.JSONObject history = conversation.getHistory();
+            com.alibaba.fastjson.JSONArray historyArray = new com.alibaba.fastjson.JSONArray();
             Collections.reverse(msgs);
             msgs.remove(msgs.size() - 1);
             for (FastGptChatMsg msg : msgs) {
                 Integer sendType = msg.getSendType();
                 String content = msg.getContent();
                 if(sendType!=1){
-                    if (content!=null&&content.length()>150){
+                    if (content!=null&&content.length()>500){
                         continue;
                     }
                 }
-                history.put(sendType==1?"user":"ai",content);
+                com.alibaba.fastjson.JSONObject msgObj = new com.alibaba.fastjson.JSONObject();
+                msgObj.put("role", sendType==1?"user":"ai");
+                msgObj.put("content", content);
+                historyArray.add(msgObj);
             }
+            conversation.setHistory(historyArray);
         }
 
         if (count!=null&& !count.isEmpty()){
@@ -1659,9 +1704,8 @@ public class AiHookServiceImpl implements AiHookService {
     private void addPromptWord(List<ChatParam.Message> messageList,String count,Long extId,String words,String countInfo,Long sessionId){
 
         FastGptChatConversation conversation = new FastGptChatConversation();
-        conversation.setAiInfo(new com.alibaba.fastjson.JSONObject());
         conversation.setUserInfo(new com.alibaba.fastjson.JSONObject());
-        conversation.setHistory(new com.alibaba.fastjson.JSONObject());
+        conversation.setHistory(new com.alibaba.fastjson.JSONArray());
 
 
         // 这里获取后台的提示词进行匹配
@@ -1717,19 +1761,23 @@ public class AiHookServiceImpl implements AiHookService {
 
         List<FastGptChatMsg> msgs=fastGptChatMsgService.selectFastGptChatMsgByMsgSessionIdAndExtId(sessionId,extId);
         if (!msgs.isEmpty()){
-            com.alibaba.fastjson.JSONObject history = conversation.getHistory();
+            com.alibaba.fastjson.JSONArray historyArray = new com.alibaba.fastjson.JSONArray();
             Collections.reverse(msgs);
             msgs.remove(msgs.size() - 1);
             for (FastGptChatMsg msg : msgs) {
                 Integer sendType = msg.getSendType();
                 String content = msg.getContent();
                 if(sendType!=1){
-                    if (content!=null&&content.length()>150){
+                    if (content!=null&&content.length()>500){
                         continue;
                     }
                 }
-                history.put(sendType==1?"user":"ai",content);
+                com.alibaba.fastjson.JSONObject msgObj = new com.alibaba.fastjson.JSONObject();
+                msgObj.put("role", sendType==1?"user":"ai");
+                msgObj.put("content", content);
+                historyArray.add(msgObj);
             }
+            conversation.setHistory(historyArray);
         }
 
         if (count!=null&& !count.isEmpty()){

fs-service/src/main/java/com/fs/fastgptApi/param/ChatParamNew.java

@@ -0,0 +1,162 @@
+package com.fs.fastgptApi.param;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+* 对话接口
+*/
+@Data
+public class ChatParamNew {
+
+    /**
+    * 聊天id
+    */
+    private String chatId;
+
+    //知识库id
+//    private String dataId;
+//    private String datasetId;
+
+    /**
+    * 是否开启 stream模式 (stream模式下会通过event进行区分/非stream模式结果保存在responseData)
+    */
+    private Boolean stream;
+
+    /**
+    * 是否返回中间值（模块状态，响应的完整结果等）
+    */
+    private Boolean detail;
+
+
+    /**
+    * 模块变量，一个对象，会替换模块中，输入框内容里的{{key}}
+    */
+    private Variables variables;
+
+    /**
+    * 聊天信息（对话框）
+    */
+    private List<Message> messages;
+
+
+    @Data
+    public static class Variables {
+        //客户的id？
+        private String uid;
+        /**
+        * 客户名称
+        */
+        private String name;
+
+    }
+
+    @Data
+    public static class Message {
+
+        /**
+         * 关键：content 不再是 String，而是 内容块列表
+         */
+        private Object content;
+        /**
+        * 用户权限
+         * 字段用来定义消息的发送者角色，具体包括三种选择：system、user、和 assistant。
+         *
+         * system（系统）:通常用于设置聊天的上下文或者提供系统级别的指示和配置信息。
+         * user（用户）:代表实际的用户输入，即用户向聊天系统提出的问题或者发起的对话内容。
+         * assistant（助手）:代表智能助手的回复或者动作，是模型根据用户输入给出的响应。
+        */
+        private String role;
+
+        /**
+         * 获取字符串类型的 content
+         */
+        public String getContentAsString() {
+            if (content instanceof String) {
+                return (String) content;
+            }
+            return null;
+        }
+
+        /**
+         * 获取列表类型的 content
+         */
+        public List<ContentItem> getContentAsList() {
+            if (content instanceof List) {
+                return (List<ContentItem>) content;
+            }
+            return null;
+        }
+
+        /**
+         * 从 JSONObject 中解析 content
+         */
+        public void parseContentFrom(JSONObject jsonObject) {
+            Object rawContent = jsonObject.get("content");
+            if (rawContent instanceof String) {
+                this.content = rawContent;
+            } else if (rawContent instanceof JSONArray) {
+                JSONArray array = (JSONArray) rawContent;
+                List<ContentItem> items = new ArrayList<>();
+                for (int i = 0; i < array.size(); i++) {
+                    items.add(array.getObject(i, ContentItem.class));
+                }
+                this.content = items;
+            }
+        }
+
+        /**
+         * 判断 content 是否为字符串
+         */
+        public boolean isContentString() {
+            return content instanceof String;
+        }
+
+        /**
+         * 判断 content 是否为列表
+         */
+        public boolean isContentList() {
+            return content instanceof List;
+        }
+
+
+    }
+    @Data
+    public static class ContentItem {
+        /**
+         * 类型：text / image_url / file_url
+         */
+        private String type;
+
+        /**
+         * 文本内容（只有 type=text 时有值）
+         */
+        private String text;
+
+        /**
+         * 图片链接对象（只有 type=image_url 时有值）
+         */
+        private ImageUrl image_url;
+
+        /**
+         * 文件对象（只有 type=file_url 时有值）
+         */
+        private FileUrl file_url;
+    }
+
+    @Data
+    public static class ImageUrl {
+        private String url;
+    }
+
+    @Data
+    public static class FileUrl {
+        private String name;
+        private String url;
+    }
+
+}

fs-service/src/main/java/com/fs/fastgptApi/service/ChatService.java

@@ -3,6 +3,7 @@ package com.fs.fastgptApi.service;
 
 import com.fs.common.core.domain.R;
 import com.fs.fastgptApi.param.ChatParam;
+import com.fs.fastgptApi.param.ChatParamNew;
 
 /**
 * 对话接口
@@ -13,4 +14,6 @@ public interface ChatService {
     * 发起对话
     */
     R initiatingTakeChat(ChatParam param,String url,String appKey);
+
+    R initiatingTakeChatNew(ChatParamNew param, String url, String appKey);
 }

fs-service/src/main/java/com/fs/fastgptApi/service/Impl/ChatServiceImpl.java

@@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.fs.common.core.domain.R;
 import com.fs.fastgptApi.config.FastGptApiConfig;
 import com.fs.fastgptApi.param.ChatParam;
+import com.fs.fastgptApi.param.ChatParamNew;
 import com.fs.fastgptApi.result.ChatDetailFStreamFResult;
 import com.fs.fastgptApi.result.ChatDetailTStreamFResult;
 import com.fs.fastgptApi.result.KnowledgeBaseResult;
@@ -113,7 +114,90 @@ public class ChatServiceImpl implements ChatService {
     }
 
 
+    @Override
+    public R initiatingTakeChatNew(ChatParamNew param, String url, String appKey) {
+
+        String json = HttpUtil.sendPost(param, url+FastGptApiConfig.completionsUrl, appKey);
+
+        JSONObject jsonObject = JSON.parseObject(json);
+
+        // 判断是否有回应消息
+        if(jsonObject.containsKey("choices")){
+            // 获取消息列表
+            JSONArray choices = jsonObject.getJSONArray("choices");
+            // 回复消息不为空
+            if(!choices.isEmpty()){
+                // 判断是否转人工
+                if(JSON.parseObject(choices.get(0).toString()).getJSONObject("message").getString("content").contains("【转人工】")){
+                    jsonObject.put("artificial", true);
+                }
+                if(JSON.parseObject(choices.get(0).toString()).getJSONObject("message").getString("content").contains("FunctionCallBegin")){
+                    jsonObject.put("artificial", true);
+                }
+
+
+                // 替换AI回复里面的所有的【】包括里面的字符串
+                List<JSONObject> list = choices.stream().map(e -> {
+                    JSONObject result = JSON.parseObject(e.toString());
+                    JSONObject message = result.getJSONObject("message");
+                    message.put("content", message.getString("content"));
+                    return result;
+                }).collect(Collectors.toList());
+                jsonObject.put("choices", list);
+            }
+        }
+        // 判断是否匹配知识库逻辑
+        if(jsonObject.containsKey("responseData")){
+            // 遍历查找主对话框
+            JSONArray responseData = jsonObject.getJSONArray("responseData");
+            // 筛选出主对话框里面的信息，里面的historyPreview是匹配信息
+            responseData.stream().filter(e -> "主对话框".equals(JSON.parseObject(e.toString()).getString("moduleName"))).findFirst().ifPresent(e -> {
+                // 进入方法证明以及查找主对话框
+                JSONObject eJson = JSON.parseObject(e.toString());
+                // 如果主对话框里面信息为空则不处理
+                if(!eJson.containsKey("historyPreview") || StringUtils.isEmpty(eJson.getString("historyPreview"))) return;
+                // 循环里面的所有数据
+                JSONArray historyPreview = eJson.getJSONArray("historyPreview");
+                // 判断是否包含未匹配信息  -> true：未匹配到知识库  false：已匹配到知识库
+                boolean knowledge = historyPreview.stream().anyMatch(h -> {
+                    JSONObject hJson = JSON.parseObject(h.toString());
+                    return hJson.getString("value").contains("<Data>\n\n</Data>");
+                });
+                // 之所以取反，是为了后面实体类方便查看
+                jsonObject.put("knowledge", !knowledge);
+            });
+        }
 
 
+        if (jsonObject.containsKey("code") && !("200".equals(jsonObject.getString("code")))) {
+            return R.error().put("data", JSON.parseObject(json, KnowledgeBaseResult.class));
+        }
+        if (!param.getDetail() && !param.getStream()){
+
+            ChatDetailFStreamFResult result = jsonObject.toJavaObject(ChatDetailFStreamFResult.class);
+
+            return R.ok().put("data", result);
+        }
+        // flase true 的方式不建议(即stream流的方式)，都是输出完才返回，没意义
+        if (!param.getDetail() && param.getStream()){
+
+//            ChatDetailFStreamTResult result = jsonObject.toJavaObject(ChatDetailFStreamTResult.class);
+
+            return R.ok().put("data", json);
+        }
+
+        if (param.getDetail()&&!param.getStream()){
+            ChatDetailTStreamFResult result = jsonObject.toJavaObject(ChatDetailTStreamFResult.class);
+            return R.ok().put("data", result);
+        }
+
+        // true true 的方式不建议(即stream流的方式)，都是输出完才返回，没意义
+        if (param.getDetail()&&param.getStream()){
+
+            return R.ok().put("data", json);
+        }
+        return null;
+    }
+
 
 }

fs-service/src/main/java/com/fs/qw/service/impl/QwUserServiceImpl.java

@@ -1320,6 +1320,7 @@ public class QwUserServiceImpl implements IQwUserService
 
         System.out.println("回调地址"+"http://saasqwapimsg.ylrzcloud.com/msg/callback/"+serverId + "/"+loginParam.getTenantId());
         wxWorkSetCallbackUrlDTO.setUrl("http://saasqwapimsg.ylrzcloud.com/msg/callback/"+serverId+ "/"+loginParam.getTenantId());
+        //wxWorkSetCallbackUrlDTO.setUrl("http://cn-hk-bgp-4.ofalias.net:55081/msg/callback/"+serverId+ "/"+loginParam.getTenantId());
         wxWorkSetCallbackUrlDTO.setUuid(data.getUuid());
         wxWorkService.SetCallbackUrl(wxWorkSetCallbackUrlDTO,serverId);
 

fs-service/src/main/java/com/fs/tenant/service/impl/TenantInfoServiceImpl.java

@@ -281,17 +281,18 @@ public class TenantInfoServiceImpl extends ServiceImpl<TenantInfoMapper, TenantI
 
     @Override
     public List<SysMenu> selectMenuList(SysMenu menu, Long userId) {
-        List<SysMenu> menuList = null;
-        // 管理员显示所有菜单信息
-        if (SysUser.isAdmin(userId))
-        {
-            menuList = baseMapper.selectMenuList(menu);
-        }
-        else
-        {
-            menu.getParams().put("userId", userId);
-            menuList = baseMapper.selectMenuListByUserId(menu);
-        }
+        List<SysMenu> menuList = baseMapper.selectMenuList(menu);
+
+//        // 管理员显示所有菜单信息
+//        if (SysUser.isAdmin(userId))
+//        {
+//
+//        }
+//        else
+//        {
+//            menu.getParams().put("userId", userId);
+//            menuList = baseMapper.selectMenuListByUserId(menu);
+//        }
         return menuList;
     }
 
@@ -340,17 +341,17 @@ public class TenantInfoServiceImpl extends ServiceImpl<TenantInfoMapper, TenantI
 
     @Override
     public List<CompanyMenu> selectCompanyMenuList(CompanyMenu menu, Long userId) {
-        List<CompanyMenu> menuList = null;
+        List<CompanyMenu> menuList = baseMapper.selectCompanyMenuList(menu);
         // 管理员显示所有菜单信息
-        if (SysUser.isAdmin(userId))
-        {
-            menuList = baseMapper.selectCompanyMenuList(menu);
-        }
-        else
-        {
-            menu.getParams().put("userId", userId);
-            menuList = baseMapper.selectCompanyMenuList(menu);
-        }
+//        if (SysUser.isAdmin(userId))
+//        {
+//            menuList = baseMapper.selectCompanyMenuList(menu);
+//        }
+//        else
+//        {
+//            menu.getParams().put("userId", userId);
+//            menuList = baseMapper.selectCompanyMenuList(menu);
+//        }
         return menuList;
     }
 

fs-service/src/main/resources/mapper/tenant/TenantInfoMapper.xml

@@ -102,7 +102,7 @@
         left join sys_role_menu rm on m.menu_id = rm.menu_id
         left join sys_user_role ur on rm.role_id = ur.role_id
         left join sys_role ro on ur.role_id = ro.role_id
-        where ur.user_id = #{params.userId}
+        where
         <if test="menuName != null and menuName != ''">
             AND m.menu_name like concat('%', #{menuName}, '%')
         </if>