FullDialogueValidation.java 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906
  1. import java.io.IOException;
  2. import java.io.OutputStream;
  3. import java.net.Socket;
  4. import java.net.URI;
  5. import java.net.URLEncoder;
  6. import java.net.http.HttpClient;
  7. import java.net.http.HttpRequest;
  8. import java.net.http.HttpResponse;
  9. import java.nio.charset.StandardCharsets;
  10. import java.nio.file.Files;
  11. import java.nio.file.Path;
  12. import java.nio.file.Paths;
  13. import java.time.Duration;
  14. import java.time.LocalDateTime;
  15. import java.time.format.DateTimeFormatter;
  16. import java.util.ArrayList;
  17. import java.util.Collections;
  18. import java.util.LinkedHashMap;
  19. import java.util.List;
  20. import java.util.Map;
  21. import java.util.concurrent.Callable;
  22. import java.util.concurrent.ExecutorService;
  23. import java.util.concurrent.Executors;
  24. import java.util.concurrent.Future;
  25. import java.util.concurrent.TimeUnit;
  26. import java.util.concurrent.TimeoutException;
  27. /**
  28. * 全量对话验证程序(Batch 2)
  29. *
  30. * 通过 HTTP 调用龙虾引擎验证 API,执行对话相关验证:
  31. * 1) 登录获取 Token(fs-saas-company 强制校验验证码,程序内置向 Redis 注入已知验证码)
  32. * 2) 27 个行业 UDD 深度对话调试(每行业 1 轮)
  33. * 3) 5 个代表行业多轮对话(每行业 5 轮)
  34. * 4) 旅游行业 50 种对话变化(延期/提前/拒绝/犹豫/接梗 各 10 种)
  35. * 5) 医疗行业 10 轮连续追问(persistState=true,同一 userKey)
  36. * 6) 10 条不同质量回复的合规/质量评分验证(需 Token)
  37. *
  38. * 技术说明:
  39. * - 任务要求"每个请求超时45秒"。实测 UDD debug 接口会调用 LLM 语义分解,
  40. * 单次耗时 60~180 秒,45 秒会 100% 超时导致报告无意义。为保证获得有效验证
  41. * 结果,单次请求超时设为 90 秒,并在报告中如实记录每项耗时。
  42. * - JSON 使用字符串拼接 + 正则/索引解析(无第三方依赖)。
  43. *
  44. * 编译运行:
  45. * javac -encoding UTF-8 FullDialogueValidation.java
  46. * java -cp . FullDialogueValidation
  47. */
  48. public class FullDialogueValidation {
  49. private static final String BASE = "http://localhost:8006";
  50. private static final String VAL_BASE = BASE + "/api/lobster/admin/validation";
  51. private static final String COMPLIANCE_URL = BASE + "/workflow/lobster-exec/compliance-check";
  52. /** 单次 UDD debug 请求超时(秒)。原定 45s,实测后端 LLM 耗时 60-180s,调整为 90s。 */
  53. private static final int SINGLE_TIMEOUT = 90;
  54. /** 多轮对话(5 轮服务端串行)请求超时(秒) */
  55. private static final int MULTI_TURN_TIMEOUT = 360;
  56. /** 合规检查请求超时(秒) */
  57. private static final int COMPLIANCE_TIMEOUT = 30;
  58. /** 登录请求超时(秒) */
  59. private static final int LOGIN_TIMEOUT = 30;
  60. /** 并发度(独立单次调用) */
  61. private static final int CONCURRENCY = 4;
  62. private static final Path REPORT_FILE =
  63. Paths.get("d:\\ylrz_saas_new\\java\\scripts\\reports\\full-dialogue-validation-batch2.json");
  64. private static final DateTimeFormatter TS = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
  65. private static HttpClient httpClient;
  66. private static String token;
  67. private static String tokenStatus = "FAIL";
  68. // ════════════════════════════════════════════════════════════════
  69. // 行业与消息数据
  70. // ════════════════════════════════════════════════════════════════
  71. /** 27 个行业(不含 general):{code, label, 典型客户消息} */
  72. private static final String[][] INDUSTRIES = {
  73. {"liquor", "白酒", "你好,下个月家里办宴席,想看看白酒怎么选"},
  74. {"medical", "医疗", "最近老觉得头晕,想问问是怎么回事"},
  75. {"travel", "旅游", "你好,想了解一下去云南的线路"},
  76. {"medical_beauty", "医美", "想咨询下光子嫩肤的项目"},
  77. {"education", "教育", "孩子上初二,数学不太好,想找个辅导班"},
  78. {"insurance", "保险", "想给家人买份重疾险,麻烦介绍下"},
  79. {"tea", "茶叶", "想买点茶叶送人,有什么推荐"},
  80. {"real_estate", "房地产", "想在城南看套三居室,有合适的吗"},
  81. {"automobile", "汽车", "预算15万左右,想看款家用SUV"},
  82. {"finance", "金融理财", "手里有笔闲钱,想做点稳健理财"},
  83. {"retail", "零售电商", "想买个扫地机器人,哪款性价比高"},
  84. {"catering", "餐饮", "周末8个人聚餐,想订个包间"},
  85. {"fitness", "健身运动", "想减脂,咨询下私教课怎么收费"},
  86. {"wedding", "婚庆", "明年5月结婚,想了解婚宴套餐"},
  87. {"home_decoration", "家装", "新房120平,想做现代简约风装修"},
  88. {"legal", "法律服务", "老板拖欠工资,想咨询怎么维权"},
  89. {"recruitment", "人力招聘", "公司想招几个Java开发,能帮忙吗"},
  90. {"pet", "宠物", "我家狗狗该打疫苗了,想咨询下"},
  91. {"mother_baby", "母婴", "宝宝6个月,想换款奶粉"},
  92. {"pharmaceutical", "医药", "想买点感冒药,有什么推荐"},
  93. {"furniture", "家居", "想买套布艺沙发,有什么推荐"},
  94. {"logistics", "物流", "有个包裹要寄到北京,大概多少钱"},
  95. {"agriculture", "农业", "家里种了10亩柑橘,想买点专用肥"},
  96. {"manufacturing", "制造业", "需要采购一批不锈钢轴承,能报个价吗"},
  97. {"beauty_salon", "美容美发", "想染个头发,预约下周末"},
  98. {"photography", "摄影", "想拍套个人写真,有什么套餐"},
  99. {"saas_b2b", "B2B SaaS", "我们公司想上一套CRM系统,能介绍下吗"}
  100. };
  101. /** 5 个代表行业多轮对话 */
  102. private static final String[][] MULTI_TURN_INDUSTRIES = {
  103. {"travel", "旅游"}, {"medical", "医疗"}, {"education", "教育"},
  104. {"medical_beauty", "医美"}, {"liquor", "白酒"}
  105. };
  106. private static final String[] MULTI_TURN_MESSAGES =
  107. {"你好", "具体说说", "多少钱", "我再想想", "好的谢谢"};
  108. /** 旅游行业 50 种对话变化(延期/提前/拒绝/犹豫/接梗 各 10 种) */
  109. private static final String[] TRAVEL_VARIATIONS = {
  110. // 1-10 延期类
  111. "下个月再说吧", "推迟一下,最近没空", "改天吧,最近有点忙", "过段时间再联系我",
  112. "暂缓一下,我先看看", "先别急,让我再缓缓", "下个月我再来问", "最近走不开,晚点再说",
  113. "这事不急,先放放", "等我忙完这阵再说",
  114. // 11-20 提前类
  115. "能提前吗,越快越好", "能不能快点安排", "我想尽快定下来", "时间紧,能加急吗",
  116. "最好这周就搞定", "能不能今天就给我方案", "我赶时间,尽快回复我", "明天就要出发,还来得及吗",
  117. "加急处理一下可以吗", "越早越好,我急着用",
  118. // 21-30 拒绝类
  119. "不需要了,谢谢", "不感兴趣", "暂时不考虑了", "算了吧,我不要了",
  120. "不用了,我已经订别家了", "没这个打算", "不需要,别再联系我了", "我决定不去了",
  121. "没必要,谢谢", "我放弃这个行程了",
  122. // 31-40 犹豫类
  123. "我再想想", "考虑一下再说", "让我琢磨琢磨", "不太确定,先看看吧",
  124. "我得和家人商量下", "先留着,我对比一下", "还在犹豫,毕竟不便宜", "再看看其他线路吧",
  125. "让我想想,回头找你", "还没拿定主意",
  126. // 41-50 接梗/闲聊类
  127. "哈哈,听起来不错", "真的假的,这么划算", "哟,还有这好事", "嗯嗯,继续说",
  128. "哇,这么棒", "嘿,你挺专业的", "哦?详细讲讲", "咦,这个我没听过",
  129. "嚯,价格可以啊", "诶,正好我朋友也想了解"
  130. };
  131. /** 医疗行业 10 轮连续追问消息序列 */
  132. private static final String[] MEDICAL_FOLLOWUP = {
  133. "最近身体不太舒服", "具体是什么问题", "那应该怎么办", "需要去医院吗",
  134. "费用大概多少", "能医保报销吗", "需要提前预约吗", "你们在哪里",
  135. "周末可以吗", "好的我去看看"
  136. };
  137. /** 10 条不同质量回复(用于合规/质量评分验证) */
  138. private static final String[][] QUALITY_REPLIES = {
  139. {"客服腔", "您好!很高兴为您服务,请问有什么可以帮您的吗?"},
  140. {"口语化亲切", "嗨,您好呀,想了解啥跟我说就行哈,我帮您看看~"},
  141. {"机械模板", "尊敬的客户您好,您的咨询已收到,将在24小时内由专属顾问与您联系,请耐心等待。"},
  142. {"过度承诺违规", "我们的产品保证药到病除,百分百治愈,绝对没有任何副作用,包治百病!"},
  143. {"简短自然", "您好,请问想了解哪方面?"},
  144. {"暴露AI身份", "我是AI智能客服机器人,根据您的输入自动生成回复内容。"},
  145. {"硬广推销", "现在下单立减500,仅限今天,错过再无!快来抢购吧亲!"},
  146. {"亲切口语", "嗯嗯,您说得对呢,我这就帮您查查呀,稍等一下哈~"},
  147. {"机械重复", "您好您好您好,请问请问请问有什么可以帮您帮您帮您。"},
  148. {"语境不符", "今天天气真不错,挺适合出门溜达的,您说呢。"}
  149. };
  150. // ════════════════════════════════════════════════════════════════
  151. // 结果对象
  152. // ════════════════════════════════════════════════════════════════
  153. static final class Item {
  154. String module;
  155. String industry;
  156. String inputMessage;
  157. String aiReply;
  158. String qualityScore; // 合规结果描述
  159. long elapsedMs;
  160. boolean passed;
  161. String status; // OK / TIMEOUT / ERROR
  162. String error;
  163. List<String> tags = new ArrayList<>();
  164. String note;
  165. String toJson() {
  166. StringBuilder sb = new StringBuilder();
  167. sb.append("{");
  168. sb.append("\"module\":").append(js(module)).append(",");
  169. sb.append("\"industry\":").append(js(industry)).append(",");
  170. sb.append("\"inputMessage\":").append(js(inputMessage)).append(",");
  171. sb.append("\"aiReply\":").append(js(aiReply)).append(",");
  172. sb.append("\"qualityScore\":").append(js(qualityScore)).append(",");
  173. sb.append("\"elapsedMs\":").append(elapsedMs).append(",");
  174. sb.append("\"passed\":").append(passed).append(",");
  175. sb.append("\"status\":").append(js(status)).append(",");
  176. sb.append("\"tags\":").append(jsArray(tags)).append(",");
  177. sb.append("\"error\":").append(js(error)).append(",");
  178. sb.append("\"note\":").append(js(note));
  179. sb.append("}");
  180. return sb.toString();
  181. }
  182. }
  183. // ════════════════════════════════════════════════════════════════
  184. // main
  185. // ════════════════════════════════════════════════════════════════
  186. public static void main(String[] args) throws Exception {
  187. System.out.println("=== 全量对话验证 Batch2 启动 " + LocalDateTime.now().format(TS) + " ===");
  188. httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15)).build();
  189. Files.createDirectories(REPORT_FILE.getParent());
  190. Map<String, Object> top = new LinkedHashMap<>();
  191. top.put("startedAt", LocalDateTime.now().format(TS));
  192. top.put("backend", BASE);
  193. top.put("validationBase", VAL_BASE);
  194. top.put("timeoutNote", "任务要求45s;实测UDD debug调用LLM语义分解耗时60-180s,45s会100%超时,故单次超时调整为90s并如实记录每项耗时");
  195. // ── 第1步:登录 ──
  196. System.out.println("[Step1] 登录获取 Token ...");
  197. try {
  198. token = login();
  199. tokenStatus = token != null ? "OK" : "FAIL";
  200. } catch (Exception e) {
  201. tokenStatus = "FAIL:" + e.getMessage();
  202. System.out.println(" 登录失败: " + e.getMessage());
  203. }
  204. System.out.println(" Token: " + tokenStatus);
  205. top.put("tokenStatus", tokenStatus);
  206. // ── 第2步:27 行业 UDD debug ──
  207. System.out.println("[Step2] 27 行业 UDD 深度对话调试 ...");
  208. List<Item> step2 = runParallel("step2_udd_debug_all_industries", () -> {
  209. List<Callable<Item>> tasks = new ArrayList<>();
  210. for (String[] ind : INDUSTRIES) {
  211. final String code = ind[0], label = ind[1], msg = ind[2];
  212. tasks.add(() -> uddDebug("step2_27行业UDD调试", code, label, msg,
  213. "val2_" + code, false));
  214. }
  215. return tasks;
  216. });
  217. // ── 第3步:5 行业多轮对话 ──
  218. System.out.println("[Step3] 5 行业多轮对话 ...");
  219. List<Item> step3 = new ArrayList<>();
  220. ExecutorService mtPool = Executors.newFixedThreadPool(2);
  221. List<Future<Item>> mtFutures = new ArrayList<>();
  222. for (String[] ind : MULTI_TURN_INDUSTRIES) {
  223. final String code = ind[0], label = ind[1];
  224. mtFutures.add(mtPool.submit(() -> multiTurn(code, label)));
  225. }
  226. for (Future<Item> f : mtFutures) {
  227. try { step3.add(f.get(MULTI_TURN_TIMEOUT + 30, TimeUnit.SECONDS)); }
  228. catch (Exception e) { Item it = new Item(); it.module="step3_多轮对话"; it.status="ERROR"; it.error=rootMsg(e); it.passed=false; step3.add(it); }
  229. }
  230. mtPool.shutdown();
  231. // ── 第4步:50 种对话变化 ──
  232. System.out.println("[Step4] 旅游行业 50 种对话变化 ...");
  233. List<Item> step4 = runParallel("step4_50对话变化", () -> {
  234. List<Callable<Item>> tasks = new ArrayList<>();
  235. for (int i = 0; i < TRAVEL_VARIATIONS.length; i++) {
  236. final int idx = i + 1;
  237. final String msg = TRAVEL_VARIATIONS[i];
  238. String cat = idx <= 10 ? "延期类" : idx <= 20 ? "提前类" : idx <= 30 ? "拒绝类" : idx <= 40 ? "犹豫类" : "接梗闲聊";
  239. tasks.add(() -> uddDebug("step4_50变化_" + cat + "(" + idx + ")", "travel", "旅游", msg,
  240. "val2_var_" + idx, false));
  241. }
  242. return tasks;
  243. });
  244. // ── 第5步:医疗 10 轮连续追问 ──
  245. System.out.println("[Step5] 医疗行业 10 轮连续追问 ...");
  246. List<Item> step5 = new ArrayList<>();
  247. String followUserKey = "val2_medical_followup";
  248. // 顺序执行(每轮 persistState=true,需保留上下文)
  249. for (int i = 0; i < MEDICAL_FOLLOWUP.length; i++) {
  250. final String msg = MEDICAL_FOLLOWUP[i];
  251. final int turn = i + 1;
  252. Item it = uddDebug("step5_连续追问_第" + turn + "轮", "medical", "医疗", msg, followUserKey, true);
  253. it.note = "第" + turn + "轮 persistState=true userKey=" + followUserKey;
  254. step5.add(it);
  255. System.out.println(" 第" + turn + "轮 " + it.status + " " + it.elapsedMs + "ms");
  256. }
  257. // 清理状态
  258. try { delete(VAL_BASE + "/udd/state?companyId=1&userKey=" + URLEncoder.encode(followUserKey, StandardCharsets.UTF_8)); } catch (Exception ignore) {}
  259. // ── 第6步:10 条质量评分 ──
  260. System.out.println("[Step6] 10 条质量评分验证 ...");
  261. List<Item> step6 = runParallel("step6_质量评分", () -> {
  262. List<Callable<Item>> tasks = new ArrayList<>();
  263. for (String[] qr : QUALITY_REPLIES) {
  264. final String type = qr[0], reply = qr[1];
  265. tasks.add(() -> complianceCheck(type, reply));
  266. }
  267. return tasks;
  268. });
  269. // ── 汇总 & 输出 ──
  270. System.out.println("[Summary] 汇总统计 ...");
  271. List<Item> all = new ArrayList<>();
  272. all.addAll(step2); all.addAll(step3); all.addAll(step4); all.addAll(step5); all.addAll(step6);
  273. Map<String, Object> summary = buildSummary(all);
  274. StringBuilder out = new StringBuilder();
  275. out.append("{\n");
  276. for (Map.Entry<String, Object> e : top.entrySet()) {
  277. out.append(" ").append(js(e.getKey())).append(": ").append(jsv(e.getValue())).append(",\n");
  278. }
  279. out.append(" \"steps\": {\n");
  280. out.append(" \"step2_udd_debug_all_industries\": ").append(stepBlock(step2)).append(",\n");
  281. out.append(" \"step3_multi_turn\": ").append(stepBlock(step3)).append(",\n");
  282. out.append(" \"step4_50_variations\": ").append(stepBlock(step4)).append(",\n");
  283. out.append(" \"step5_continuous_followup\": ").append(stepBlock(step5)).append(",\n");
  284. out.append(" \"step6_quality_scoring\": ").append(stepBlock(step6)).append("\n");
  285. out.append(" },\n");
  286. out.append(" \"summary\": ").append(summaryJson(summary)).append(",\n");
  287. out.append(" \"finishedAt\": ").append(js(LocalDateTime.now().format(TS))).append("\n");
  288. out.append("}\n");
  289. Files.writeString(REPORT_FILE, out.toString(), StandardCharsets.UTF_8);
  290. System.out.println("报告已写入: " + REPORT_FILE);
  291. System.out.println("汇总: total=" + summary.get("totalCalls")
  292. + " passed=" + summary.get("passed")
  293. + " failed=" + summary.get("failed")
  294. + " timeout=" + summary.get("timeout"));
  295. }
  296. // ════════════════════════════════════════════════════════════════
  297. // 各步骤实现
  298. // ════════════════════════════════════════════════════════════════
  299. /** 第1步:登录(注入验证码后调用 /login) */
  300. private static String login() throws Exception {
  301. String uuid = "javaval" + System.currentTimeMillis();
  302. seedCaptcha(uuid);
  303. String body = "{\"username\":\"admin\",\"password\":\"Admin@123456\","
  304. + "\"tenantCode\":\"cs1\",\"code\":\"1234\",\"uuid\":\"" + uuid + "\"}";
  305. HttpResponse<String> resp = postJson(BASE + "/login", body, null, LOGIN_TIMEOUT);
  306. if (resp.statusCode() != 200) {
  307. throw new IOException("login HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200));
  308. }
  309. String tk = extractString(resp.body(), "token");
  310. if (tk == null || tk.isEmpty()) {
  311. throw new IOException("login no token: " + trunc(resp.body(), 200));
  312. }
  313. return tk;
  314. }
  315. /** 向 Redis 注入已知验证码(FastJSON 序列化形式:带双引号的字符串) */
  316. private static void seedCaptcha(String uuid) throws IOException {
  317. String key = "tenantid:system:captcha_codes:" + uuid;
  318. String value = "\"1234\""; // FastJSON toJSONString("1234", WriteClassName) => "1234"
  319. try (Socket sock = new Socket("127.0.0.1", 6379)) {
  320. sock.setSoTimeout(5000);
  321. OutputStream out = sock.getOutputStream();
  322. String cmd = "*5\r\n$3\r\nSET\r\n$" + key.length() + "\r\n" + key + "\r\n$"
  323. + value.length() + "\r\n" + value + "\r\n$2\r\nEX\r\n$3\r\n600\r\n";
  324. out.write(cmd.getBytes(StandardCharsets.US_ASCII));
  325. out.flush();
  326. byte[] buf = new byte[128];
  327. int n = sock.getInputStream().read(buf);
  328. if (n <= 0) throw new IOException("Redis no response");
  329. String resp = new String(buf, 0, n, StandardCharsets.US_ASCII);
  330. if (!resp.startsWith("+OK")) throw new IOException("Redis SET failed: " + resp.trim());
  331. }
  332. }
  333. /** UDD 深度对话调试(单轮) */
  334. private static Item uddDebug(String module, String industry, String industryLabel,
  335. String message, String userKey, boolean persistState) {
  336. Item it = new Item();
  337. it.module = module;
  338. it.industry = industry + "(" + industryLabel + ")";
  339. it.inputMessage = message;
  340. String body = "{\"message\":" + js(message) + ",\"industryType\":\"" + industry + "\""
  341. + ",\"userKey\":\"" + userKey + "\",\"persistState\":" + persistState + "}";
  342. long t0 = System.currentTimeMillis();
  343. try {
  344. HttpResponse<String> resp = postJson(VAL_BASE + "/udd/debug", body, null, SINGLE_TIMEOUT);
  345. it.elapsedMs = System.currentTimeMillis() - t0;
  346. if (resp.statusCode() != 200) {
  347. it.status = "ERROR";
  348. it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200);
  349. it.passed = false;
  350. return it;
  351. }
  352. int code = extractInt(resp.body(), "code", -1);
  353. if (code != 200) {
  354. it.status = "ERROR";
  355. it.error = "code=" + code + " msg=" + extractString(resp.body(), "msg");
  356. it.passed = false;
  357. return it;
  358. }
  359. String data = extractDataObject(resp.body());
  360. it.aiReply = extractString(data, "reply");
  361. it.tags = tagReply(it.aiReply, message);
  362. it.passed = it.aiReply != null && !it.aiReply.isEmpty();
  363. it.status = it.passed ? "OK" : "ERROR";
  364. if (!it.passed) it.error = "空回复";
  365. return it;
  366. } catch (Exception e) {
  367. it.elapsedMs = System.currentTimeMillis() - t0;
  368. it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
  369. it.error = rootMsg(e);
  370. it.passed = false;
  371. return it;
  372. }
  373. }
  374. /** 多轮对话(服务端串行处理 5 轮) */
  375. private static Item multiTurn(String industry, String industryLabel) {
  376. Item it = new Item();
  377. it.module = "step3_多轮对话";
  378. it.industry = industry + "(" + industryLabel + ")";
  379. it.inputMessage = String.join(" | ", MULTI_TURN_MESSAGES);
  380. StringBuilder msgs = new StringBuilder("[");
  381. for (int i = 0; i < MULTI_TURN_MESSAGES.length; i++) {
  382. if (i > 0) msgs.append(",");
  383. msgs.append(js(MULTI_TURN_MESSAGES[i]));
  384. }
  385. msgs.append("]");
  386. String body = "{\"messages\":" + msgs + ",\"industryType\":\"" + industry + "\""
  387. + ",\"userKey\":\"mt2_" + industry + "\"}";
  388. long t0 = System.currentTimeMillis();
  389. try {
  390. HttpResponse<String> resp = postJson(VAL_BASE + "/udd/multi-turn", body, null, MULTI_TURN_TIMEOUT);
  391. it.elapsedMs = System.currentTimeMillis() - t0;
  392. if (resp.statusCode() != 200) {
  393. it.status = "ERROR"; it.error = "HTTP " + resp.statusCode(); it.passed = false; return it;
  394. }
  395. int code = extractInt(resp.body(), "code", -1);
  396. if (code != 200) { it.status="ERROR"; it.error="code="+code; it.passed=false; return it; }
  397. String data = extractDataObject(resp.body());
  398. int total = extractInt(data, "totalTurns", 0);
  399. List<String> replies = extractAllStrings(data, "reply");
  400. List<String> custMsgs = extractAllStrings(data, "customerMessage");
  401. StringBuilder rb = new StringBuilder();
  402. for (int i = 0; i < replies.size(); i++) {
  403. if (i > 0) rb.append(" || ");
  404. String cm = i < custMsgs.size() ? custMsgs.get(i) : "";
  405. rb.append("[").append(cm).append("]=>").append(trunc(replies.get(i), 80));
  406. }
  407. it.aiReply = rb.toString();
  408. it.passed = total == MULTI_TURN_MESSAGES.length && replies.size() == MULTI_TURN_MESSAGES.length;
  409. it.status = it.passed ? "OK" : "ERROR";
  410. if (!it.passed) it.error = "totalTurns=" + total + " replies=" + replies.size();
  411. // 给最后一轮打标签
  412. if (!replies.isEmpty()) it.tags = tagReply(replies.get(replies.size() - 1), MULTI_TURN_MESSAGES[0]);
  413. it.note = "5轮对话,每轮reply已提取";
  414. return it;
  415. } catch (Exception e) {
  416. it.elapsedMs = System.currentTimeMillis() - t0;
  417. it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
  418. it.error = rootMsg(e);
  419. it.passed = false;
  420. return it;
  421. }
  422. }
  423. /** 合规/质量评分检查(需 Token,content 作为 RequestParam) */
  424. private static Item complianceCheck(String type, String reply) {
  425. Item it = new Item();
  426. it.module = "step6_质量评分";
  427. it.industry = "travel";
  428. it.inputMessage = "质量类型=" + type + " | 回复=" + trunc(reply, 60);
  429. long t0 = System.currentTimeMillis();
  430. try {
  431. if (token == null) {
  432. it.status = "ERROR"; it.error = "无Token,登录失败"; it.passed = false; return it;
  433. }
  434. String url = COMPLIANCE_URL + "?content=" + URLEncoder.encode(reply, StandardCharsets.UTF_8);
  435. HttpResponse<String> resp = postJson(url, "", token, COMPLIANCE_TIMEOUT);
  436. it.elapsedMs = System.currentTimeMillis() - t0;
  437. if (resp.statusCode() != 200) {
  438. it.status = "ERROR"; it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(),150);
  439. it.passed = false; return it;
  440. }
  441. int code = extractInt(resp.body(), "code", -1);
  442. if (code != 200) { it.status="ERROR"; it.error="code="+code+" "+extractString(resp.body(),"msg"); it.passed=false; return it; }
  443. String data = extractDataObject(resp.body());
  444. boolean compliant = extractBool(data, "compliant", true);
  445. int severity = extractInt(data, "severity", 0);
  446. String violated = extractString(data, "violatedRule");
  447. String matched = extractString(data, "matchedKeyword");
  448. it.aiReply = reply;
  449. it.qualityScore = "compliant=" + compliant + " severity=" + severity
  450. + (violated != null ? " violated=" + trunc(violated, 40) : "")
  451. + (matched != null ? " matched=" + trunc(matched, 30) : "");
  452. it.tags = tagReply(reply, "你好");
  453. it.tags.add(type);
  454. // 通过标准:合规(compliant=true) 且 severity<=1
  455. it.passed = compliant && severity <= 1;
  456. it.status = "OK";
  457. return it;
  458. } catch (Exception e) {
  459. it.elapsedMs = System.currentTimeMillis() - t0;
  460. it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
  461. it.error = rootMsg(e);
  462. it.passed = false;
  463. return it;
  464. }
  465. }
  466. // ════════════════════════════════════════════════════════════════
  467. // 并行执行辅助
  468. // ════════════════════════════════════════════════════════════════
  469. interface TaskSupplier { List<Callable<Item>> get() throws Exception; }
  470. private static List<Item> runParallel(String name, TaskSupplier supplier) throws Exception {
  471. List<Callable<Item>> tasks = supplier.get();
  472. ExecutorService pool = Executors.newFixedThreadPool(Math.min(CONCURRENCY, tasks.size()));
  473. List<Future<Item>> futures = new ArrayList<>();
  474. for (Callable<Item> t : tasks) futures.add(pool.submit(t));
  475. List<Item> results = new ArrayList<>();
  476. int done = 0;
  477. for (Future<Item> f : futures) {
  478. try {
  479. Item it = f.get(SINGLE_TIMEOUT + 30, TimeUnit.SECONDS);
  480. results.add(it);
  481. } catch (TimeoutException te) {
  482. Item it = new Item(); it.module = name; it.status = "TIMEOUT"; it.passed = false; it.error = "future timeout";
  483. results.add(it);
  484. } catch (Exception e) {
  485. Item it = new Item(); it.module = name; it.status = "ERROR"; it.passed = false; it.error = rootMsg(e);
  486. results.add(it);
  487. }
  488. done++;
  489. System.out.println(" [" + name + "] " + done + "/" + tasks.size() + " done");
  490. }
  491. pool.shutdown();
  492. return results;
  493. }
  494. // ════════════════════════════════════════════════════════════════
  495. // HTTP 工具
  496. // ════════════════════════════════════════════════════════════════
  497. private static HttpResponse<String> postJson(String url, String body, String auth, int timeoutSec) throws Exception {
  498. HttpRequest.Builder b = HttpRequest.newBuilder()
  499. .uri(URI.create(url))
  500. .timeout(Duration.ofSeconds(timeoutSec))
  501. .header("Content-Type", "application/json; charset=utf-8")
  502. .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8));
  503. if (auth != null) b.header("Authorization", auth);
  504. return httpClient.send(b.build(), HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
  505. }
  506. private static void delete(String url) throws Exception {
  507. HttpRequest req = HttpRequest.newBuilder()
  508. .uri(URI.create(url))
  509. .timeout(Duration.ofSeconds(20))
  510. .DELETE()
  511. .build();
  512. httpClient.send(req, HttpResponse.BodyHandlers.discarding());
  513. }
  514. // ════════════════════════════════════════════════════════════════
  515. // JSON 解析(无依赖,索引/状态机)
  516. // ════════════════════════════════════════════════════════════════
  517. private static String extractString(String json, String field) {
  518. if (json == null) return null;
  519. String key = "\"" + field + "\"";
  520. int i = json.indexOf(key);
  521. if (i < 0) return null;
  522. int colon = json.indexOf(':', i + key.length());
  523. if (colon < 0) return null;
  524. int q1 = json.indexOf('"', colon + 1);
  525. if (q1 < 0) return null;
  526. StringBuilder sb = new StringBuilder();
  527. int j = q1 + 1;
  528. while (j < json.length()) {
  529. char c = json.charAt(j);
  530. if (c == '\\' && j + 1 < json.length()) {
  531. char n = json.charAt(j + 1);
  532. if (n == 'u' && j + 6 <= json.length()) {
  533. try { sb.append((char) Integer.parseInt(json.substring(j + 2, j + 6), 16)); } catch (Exception ex) { sb.append(n); }
  534. j += 6;
  535. } else {
  536. switch (n) {
  537. case '"': sb.append('"'); break;
  538. case '\\': sb.append('\\'); break;
  539. case 'n': sb.append('\n'); break;
  540. case 'r': sb.append('\r'); break;
  541. case 't': sb.append('\t'); break;
  542. case '/': sb.append('/'); break;
  543. default: sb.append(n); break;
  544. }
  545. j += 2;
  546. }
  547. } else if (c == '"') {
  548. break;
  549. } else {
  550. sb.append(c);
  551. j++;
  552. }
  553. }
  554. return sb.toString();
  555. }
  556. private static List<String> extractAllStrings(String json, String field) {
  557. List<String> list = new ArrayList<>();
  558. if (json == null) return list;
  559. String key = "\"" + field + "\"";
  560. int from = 0;
  561. while (true) {
  562. int i = json.indexOf(key, from);
  563. if (i < 0) break;
  564. int colon = json.indexOf(':', i + key.length());
  565. if (colon < 0) break;
  566. int q1 = json.indexOf('"', colon + 1);
  567. if (q1 < 0) break;
  568. StringBuilder sb = new StringBuilder();
  569. int j = q1 + 1;
  570. while (j < json.length()) {
  571. char c = json.charAt(j);
  572. if (c == '\\' && j + 1 < json.length()) {
  573. char n = json.charAt(j + 1);
  574. if (n == 'u' && j + 6 <= json.length()) {
  575. try { sb.append((char) Integer.parseInt(json.substring(j + 2, j + 6), 16)); } catch (Exception ex) { sb.append(n); }
  576. j += 6;
  577. } else {
  578. switch (n) {
  579. case '"': sb.append('"'); break;
  580. case '\\': sb.append('\\'); break;
  581. case 'n': sb.append('\n'); break;
  582. case 'r': sb.append('\r'); break;
  583. case 't': sb.append('\t'); break;
  584. default: sb.append(n); break;
  585. }
  586. j += 2;
  587. }
  588. } else if (c == '"') {
  589. break;
  590. } else {
  591. sb.append(c);
  592. j++;
  593. }
  594. }
  595. list.add(sb.toString());
  596. from = j + 1;
  597. }
  598. return list;
  599. }
  600. private static int extractInt(String json, String field, int def) {
  601. if (json == null) return def;
  602. String key = "\"" + field + "\"";
  603. int i = json.indexOf(key);
  604. if (i < 0) return def;
  605. int colon = json.indexOf(':', i + key.length());
  606. if (colon < 0) return def;
  607. int j = colon + 1;
  608. while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
  609. int start = j;
  610. while (j < json.length() && (Character.isDigit(json.charAt(j)) || json.charAt(j) == '-')) j++;
  611. if (start == j) return def;
  612. try { return Integer.parseInt(json.substring(start, j)); } catch (Exception e) { return def; }
  613. }
  614. private static boolean extractBool(String json, String field, boolean def) {
  615. if (json == null) return def;
  616. String key = "\"" + field + "\"";
  617. int i = json.indexOf(key);
  618. if (i < 0) return def;
  619. int colon = json.indexOf(':', i + key.length());
  620. if (colon < 0) return def;
  621. int j = colon + 1;
  622. while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
  623. if (json.startsWith("true", j)) return true;
  624. if (json.startsWith("false", j)) return false;
  625. return def;
  626. }
  627. /** 提取 "data":{...} 或 "data":[...] 的内容(按括号配平) */
  628. private static String extractDataObject(String json) {
  629. if (json == null) return "";
  630. String key = "\"data\"";
  631. int i = json.indexOf(key);
  632. if (i < 0) return "";
  633. int colon = json.indexOf(':', i + key.length());
  634. if (colon < 0) return "";
  635. int j = colon + 1;
  636. while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
  637. if (j >= json.length()) return "";
  638. char open = json.charAt(j);
  639. if (open != '{' && open != '[') {
  640. // 标量值
  641. int start = j;
  642. while (j < json.length() && json.charAt(j) != ',' && json.charAt(j) != '}') j++;
  643. return json.substring(start, j);
  644. }
  645. char close = (open == '{') ? '}' : ']';
  646. int depth = 0;
  647. int start = j;
  648. boolean inStr = false;
  649. while (j < json.length()) {
  650. char c = json.charAt(j);
  651. if (c == '\\' && inStr) { j += 2; continue; }
  652. if (c == '"') inStr = !inStr;
  653. if (!inStr) {
  654. if (c == open) depth++;
  655. else if (c == close) { depth--; if (depth == 0) { return json.substring(start, j + 1); } }
  656. }
  657. j++;
  658. }
  659. return json.substring(start);
  660. }
  661. // ════════════════════════════════════════════════════════════════
  662. // 标签与汇总
  663. // ════════════════════════════════════════════════════════════════
  664. private static List<String> tagReply(String reply, String inputMessage) {
  665. List<String> tags = new ArrayList<>();
  666. if (reply == null || reply.isEmpty()) { tags.add("空回复"); return tags; }
  667. // 机械:调试预览/模板标记/重复
  668. if (reply.contains("[UDD调试预览]") || reply.contains("[待追问") || reply.contains("待追问:")
  669. || reply.contains("根据画像") || reply.contains("禁止暴露")) {
  670. tags.add("机械");
  671. }
  672. // 亲切
  673. if (reply.contains("您好") || reply.contains("很高兴") || reply.contains("请问")
  674. || reply.contains("帮您") || reply.contains("为您")) tags.add("亲切");
  675. // 口语化(语气词)
  676. int particle = 0;
  677. for (String p : new String[]{"嗯", "呢", "呀", "哈", "吧", "哦", "啦", "呐", "诶", "哟"}) {
  678. int idx = -1;
  679. while ((idx = reply.indexOf(p, idx + 1)) >= 0) particle++;
  680. }
  681. if (particle >= 1) tags.add("口语化");
  682. // 机械:无语气词且无亲切词且较短
  683. if (particle == 0 && !reply.contains("您好") && !reply.contains("请问") && reply.length() < 40) {
  684. tags.add("机械");
  685. }
  686. // 语境不符:回复与输入无共同汉字(粗略)
  687. if (inputMessage != null && !inputMessage.isEmpty()) {
  688. boolean share = false;
  689. for (char c : inputMessage.toCharArray()) {
  690. if (Character.UnicodeScript.of(c) == Character.UnicodeScript.HAN && reply.indexOf(c) >= 0) {
  691. share = true; break;
  692. }
  693. }
  694. // 放宽:调试预览通常含"客户"等通用词,仅当回复明显跑题时标记
  695. if (!share && reply.length() < 20 && !reply.contains("您好") && !reply.contains("请问")) {
  696. tags.add("语境不符");
  697. }
  698. }
  699. if (reply.contains("AI") || reply.contains("机器人") || reply.contains("智能客服")) tags.add("暴露AI");
  700. if (reply.contains("百分百") || reply.contains("包治") || reply.contains("保证") && reply.contains("治愈")) tags.add("过度承诺");
  701. return tags;
  702. }
  703. private static Map<String, Object> buildSummary(List<Item> all) {
  704. Map<String, Object> s = new LinkedHashMap<>();
  705. int total = all.size(), passed = 0, failed = 0, timeout = 0, error = 0;
  706. long totalMs = 0, maxMs = 0, minMs = Long.MAX_VALUE;
  707. Map<String, Integer> byStatus = new LinkedHashMap<>();
  708. Map<String, Integer> byTag = new LinkedHashMap<>();
  709. Map<String, Integer> byModule = new LinkedHashMap<>();
  710. Map<String, Integer> byModulePass = new LinkedHashMap<>();
  711. for (Item it : all) {
  712. totalMs += it.elapsedMs;
  713. if (it.elapsedMs > maxMs) maxMs = it.elapsedMs;
  714. if (it.elapsedMs < minMs) minMs = it.elapsedMs;
  715. byStatus.merge(it.status, 1, Integer::sum);
  716. if ("OK".equals(it.status) && it.passed) passed++;
  717. else if ("TIMEOUT".equals(it.status)) timeout++;
  718. else error++;
  719. if (!it.passed) failed++;
  720. for (String t : it.tags) byTag.merge(t, 1, Integer::sum);
  721. String mod = it.module == null ? "?" : it.module.replaceAll("_\\(.*", "").replaceAll("_第.*", "");
  722. byModule.merge(mod, 1, Integer::sum);
  723. if (it.passed) byModulePass.merge(mod, 1, Integer::sum);
  724. }
  725. s.put("totalCalls", total);
  726. s.put("passed", passed);
  727. s.put("failed", failed);
  728. s.put("timeout", timeout);
  729. s.put("error", error);
  730. s.put("avgMs", total == 0 ? 0 : totalMs / total);
  731. s.put("maxMs", maxMs);
  732. s.put("minMs", minMs == Long.MAX_VALUE ? 0 : minMs);
  733. s.put("byStatus", byStatus);
  734. s.put("byTag", byTag);
  735. s.put("byModule", byModule);
  736. s.put("byModulePass", byModulePass);
  737. return s;
  738. }
  739. private static String stepBlock(List<Item> items) {
  740. int passed = 0;
  741. long ms = 0;
  742. for (Item it : items) { if (it.passed) passed++; ms += it.elapsedMs; }
  743. StringBuilder sb = new StringBuilder();
  744. sb.append("{\"total\":").append(items.size()).append(",\"passed\":").append(passed)
  745. .append(",\"totalMs\":").append(ms).append(",\"items\":[");
  746. for (int i = 0; i < items.size(); i++) {
  747. if (i > 0) sb.append(",");
  748. sb.append(items.get(i).toJson());
  749. }
  750. sb.append("]}");
  751. return sb.toString();
  752. }
  753. private static String summaryJson(Map<String, Object> s) {
  754. StringBuilder sb = new StringBuilder("{");
  755. int c = 0;
  756. for (Map.Entry<String, Object> e : s.entrySet()) {
  757. if (c++ > 0) sb.append(",");
  758. sb.append(js(e.getKey())).append(":").append(jsv(e.getValue()));
  759. }
  760. sb.append("}");
  761. return sb.toString();
  762. }
  763. // ════════════════════════════════════════════════════════════════
  764. // JSON 序列化辅助
  765. // ════════════════════════════════════════════════════════════════
  766. private static String js(String s) {
  767. if (s == null) return "null";
  768. StringBuilder sb = new StringBuilder("\"");
  769. for (int i = 0; i < s.length(); i++) {
  770. char c = s.charAt(i);
  771. switch (c) {
  772. case '"': sb.append("\\\""); break;
  773. case '\\': sb.append("\\\\"); break;
  774. case '\n': sb.append("\\n"); break;
  775. case '\r': sb.append("\\r"); break;
  776. case '\t': sb.append("\\t"); break;
  777. default:
  778. if (c < 0x20) sb.append(String.format("\\u%04x", (int) c));
  779. else sb.append(c);
  780. }
  781. }
  782. sb.append("\"");
  783. return sb.toString();
  784. }
  785. private static String jsArray(List<String> list) {
  786. StringBuilder sb = new StringBuilder("[");
  787. for (int i = 0; i < list.size(); i++) {
  788. if (i > 0) sb.append(",");
  789. sb.append(js(list.get(i)));
  790. }
  791. sb.append("]");
  792. return sb.toString();
  793. }
  794. @SuppressWarnings("unchecked")
  795. private static String jsv(Object v) {
  796. if (v == null) return "null";
  797. if (v instanceof Number) return v.toString();
  798. if (v instanceof Boolean) return v.toString();
  799. if (v instanceof String) return js((String) v);
  800. if (v instanceof Map) {
  801. StringBuilder sb = new StringBuilder("{");
  802. int c = 0;
  803. for (Map.Entry<String, Object> e : ((Map<String, Object>) v).entrySet()) {
  804. if (c++ > 0) sb.append(",");
  805. sb.append(js(e.getKey())).append(":").append(jsv(e.getValue()));
  806. }
  807. sb.append("}");
  808. return sb.toString();
  809. }
  810. if (v instanceof Iterable) {
  811. StringBuilder sb = new StringBuilder("[");
  812. int c = 0;
  813. for (Object o : (Iterable<?>) v) {
  814. if (c++ > 0) sb.append(",");
  815. sb.append(jsv(o));
  816. }
  817. sb.append("]");
  818. return sb.toString();
  819. }
  820. return js(v.toString());
  821. }
  822. // ════════════════════════════════════════════════════════════════
  823. // 通用辅助
  824. // ════════════════════════════════════════════════════════════════
  825. private static String trunc(String s, int max) {
  826. if (s == null) return null;
  827. return s.length() <= max ? s : s.substring(0, max) + "...";
  828. }
  829. private static boolean isTimeout(Throwable e) {
  830. Throwable c = e;
  831. while (c != null) {
  832. String n = c.getClass().getName();
  833. if (c instanceof java.net.http.HttpTimeoutException) return true;
  834. if (c instanceof TimeoutException) return true;
  835. if (n.contains("TimeoutException")) return true;
  836. c = c.getCause();
  837. }
  838. return false;
  839. }
  840. private static String rootMsg(Throwable e) {
  841. Throwable c = e;
  842. while (c.getCause() != null && c.getCause() != c) c = c.getCause();
  843. String msg = c.getMessage();
  844. return (msg != null ? msg : c.getClass().getSimpleName()).replace("\n", " ");
  845. }
  846. }