import java.io.IOException; import java.io.OutputStream; import java.net.Socket; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * 关键词 / 替换词 / 语义关键词穿透验证(Batch 3) * * 验证内容: * 第1步:敏感词全量命中验证(无需登录) * - GET /api/lobster/admin/validation/sensitive/list 获取所有敏感词 * - 对每个敏感词构造测试消息调用 POST /sensitive/check 验证命中 * 第2步:替换词命中验证(JDBC + 可选登录) * - JDBC 连接主库 ylrz_saas 查询 company_lobster_replace_word 表 enabled=1 的替换词 * - 如能登录调用 /workflow/lobster/replace-words/stress-test 验证;否则标注"需要登录验证" * 第3步:语义关键词命中和穿透验证(无需登录) * - GET /semantic/keywords 与 /semantic/sentiment 对 27 个行业 × 3 条典型消息 * - 20 组否定前缀穿透场景(不需要/没兴趣/不太好/不是不要 等) * 第4步:对外角色按行业测试(无需登录) * - POST /udd/debug 对 27 个行业串行测试,超时 180s,TIMEOUT 不算失败 * * 技术说明: * - Java 11+ HttpClient,JSON 使用字符串拼接 + 正则解析(无第三方依赖) * - 敏感词检查 8 线程并发;UDD 调试串行执行 * - 请求超时 180 秒(UDD 接口慢,60-180s) * * 编译运行: * javac -encoding UTF-8 -cp "C:\Users\77944\.m2\repository\com\mysql\mysql-connector-j\8.0.33\mysql-connector-j-8.0.33.jar" KeywordSemanticValidation.java * java -cp ".;C:\Users\77944\.m2\repository\com\mysql\mysql-connector-j\8.0.33\mysql-connector-j-8.0.33.jar" KeywordSemanticValidation */ public class KeywordSemanticValidation { // ════════════════════════════════════════════════════════════════ // 配置 // ════════════════════════════════════════════════════════════════ private static final String BASE = "http://localhost:8006"; private static final String VAL_BASE = BASE + "/api/lobster/admin/validation"; private static final String REPLACE_STRESS_URL = BASE + "/workflow/lobster/replace-words/stress-test"; private static final String LOGIN_URL = BASE + "/login"; // 主库连接(company_lobster_replace_word 表所在) private static final String DB_URL = "jdbc:mysql://cq-cdb-8fjmemkb.sql.tencentcdb.com:27220/ylrz_saas" + "?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&connectTimeout=10000&socketTimeout=30000"; private static final String DB_USER = "root"; private static final String DB_PWD = "Ylrz_1q2w3e4r5t6y"; /** UDD debug 请求超时(秒)—— 后端 LLM 语义分解耗时 60-180s */ private static final int UDD_TIMEOUT = 180; /** 敏感词 / 替换词 / 语义 请求超时(秒) */ private static final int CHECK_TIMEOUT = 90; private static final int SEMANTIC_TIMEOUT = 150; private static final int LOGIN_TIMEOUT = 30; /** 敏感词检查并发度 */ private static final int SENSITIVE_CONCURRENCY = 8; /** 语义分析并发度 */ private static final int SEMANTIC_CONCURRENCY = 8; private static final Path REPORT_FILE = Paths.get("d:\\ylrz_saas_new\\java\\scripts\\reports\\keyword-semantic-validation-batch3.json"); private static final DateTimeFormatter TS = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); private static HttpClient httpClient; private static String token; private static String tokenStatus = "FAIL"; // ════════════════════════════════════════════════════════════════ // 行业与测试数据 // ════════════════════════════════════════════════════════════════ /** 27 个行业(不含 general):{code, label} */ private static final String[][] INDUSTRIES = { {"liquor", "白酒"}, {"medical", "医疗"}, {"travel", "旅游"}, {"medical_beauty", "医美"}, {"education", "教育"}, {"insurance", "保险"}, {"tea", "茶叶"}, {"real_estate", "房地产"}, {"automobile", "汽车"}, {"finance", "金融理财"}, {"retail", "零售电商"}, {"catering", "餐饮"}, {"fitness", "健身运动"}, {"wedding", "婚庆"}, {"home_decoration", "家装"}, {"legal", "法律服务"}, {"recruitment", "人力招聘"}, {"pet", "宠物"}, {"mother_baby", "母婴"}, {"pharmaceutical", "医药"}, {"furniture", "家居"}, {"logistics", "物流"}, {"agriculture", "农业"}, {"manufacturing", "制造业"}, {"beauty_salon", "美容美发"}, {"photography", "摄影"}, {"saas_b2b", "B2B SaaS"} }; /** 每个行业测试的 3 条典型消息 */ private static final String[] TYPICAL_MESSAGES = { "想了解一下你们的服务", "这个多少钱,怎么购买", "不需要了,谢谢" }; private static final String[] MESSAGE_LABELS = {"咨询类", "购买意向", "拒绝类"}; /** 20 组否定前缀穿透场景:{消息, 期望语义方向, 说明} * 期望: NEGATIVE=应非购买/负面; POSITIVE=应肯定/正面; REJECT=拒绝 */ private static final String[][] NEGATION_CASES = { {"不需要", "REJECT", "否定前缀+需要:应判定为拒绝,不应命中购买意向"}, {"没兴趣", "REJECT", "否定前缀+兴趣:应判定为拒绝,不应命中积极信号"}, {"不太好", "NEGATIVE", "否定+好:情感应为负面"}, {"不是不要", "POSITIVE", "双重否定:应判定为肯定/需要"}, {"不想买", "REJECT", "否定+想买:不应命中购买意向"}, {"不考虑", "REJECT", "否定+考虑:不应命中积极考虑信号"}, {"不感兴趣", "REJECT", "否定+感兴趣:不应命中积极信号"}, {"没打算买", "REJECT", "否定+打算:不应命中购买意向"}, {"不着急", "NEGATIVE", "否定+着急:不应命中急切购买,情感偏负/中性"}, {"不想要", "REJECT", "否定+想要:不应命中购买意向"}, {"不喜欢", "NEGATIVE", "否定+喜欢:情感应为负面"}, {"不满意", "NEGATIVE", "否定+满意:情感应为负面"}, {"不合适", "NEGATIVE", "否定+合适:情感应为负面"}, {"不想了解", "REJECT", "否定+想了解:不应命中咨询意向"}, {"没需求", "REJECT", "否定+需求:不应命中购买意向"}, {"不用了谢谢", "REJECT", "明确拒绝:应判定为拒绝"}, {"不是不买", "POSITIVE", "双重否定:应判定为购买意向"}, {"并非不需要", "POSITIVE", "双重否定:应判定为需要"}, {"没有不想买", "POSITIVE", "双重否定:应判定为想买"}, {"不能接受", "NEGATIVE", "否定+接受:情感应为负面"} }; // ════════════════════════════════════════════════════════════════ // 结果对象 // ════════════════════════════════════════════════════════════════ static final class Item { String module; String industry; String input; String output; long elapsedMs; boolean passed; String status; // OK / TIMEOUT / ERROR / SKIP String error; List tags = new ArrayList<>(); String note; // 语义相关字段 String intent; Double sentiment; String sentimentLabel; List keywords = new ArrayList<>(); // 敏感词/替换词字段 String word; String replaceText; Boolean containsSensitive; Boolean isHighRisk; String wordType; String toJson() { StringBuilder sb = new StringBuilder(); sb.append("{"); sb.append("\"module\":").append(js(module)).append(","); sb.append("\"industry\":").append(js(industry)).append(","); sb.append("\"input\":").append(js(trunc(input, 800))).append(","); sb.append("\"output\":").append(js(trunc(output, 1500))).append(","); sb.append("\"elapsedMs\":").append(elapsedMs).append(","); sb.append("\"passed\":").append(passed).append(","); sb.append("\"status\":").append(js(status)).append(","); sb.append("\"tags\":").append(jsArray(tags)).append(","); sb.append("\"error\":").append(js(error)).append(","); sb.append("\"note\":").append(js(note)).append(","); sb.append("\"intent\":").append(js(intent)).append(","); sb.append("\"sentiment\":").append(sentiment == null ? "null" : sentiment).append(","); sb.append("\"sentimentLabel\":").append(js(sentimentLabel)).append(","); sb.append("\"keywords\":").append(jsArray(keywords)).append(","); sb.append("\"word\":").append(js(word)).append(","); sb.append("\"replaceText\":").append(js(replaceText)).append(","); sb.append("\"wordType\":").append(js(wordType)).append(","); sb.append("\"containsSensitive\":").append(containsSensitive == null ? "null" : containsSensitive).append(","); sb.append("\"isHighRisk\":").append(isHighRisk == null ? "null" : isHighRisk); sb.append("}"); return sb.toString(); } } // ════════════════════════════════════════════════════════════════ // main // ════════════════════════════════════════════════════════════════ public static void main(String[] args) throws Exception { System.out.println("=== 关键词/替换词/语义关键词穿透验证 Batch3 启动 " + LocalDateTime.now().format(TS) + " ==="); httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15)).build(); Files.createDirectories(REPORT_FILE.getParent()); Map top = new LinkedHashMap<>(); top.put("startedAt", LocalDateTime.now().format(TS)); top.put("backend", BASE); top.put("validationBase", VAL_BASE); top.put("timeoutNote", "UDD debug 接口调用 LLM 语义分解耗时 60-180s,单次超时设为 180s;" + "敏感词/语义分析超时 90-150s;TIMEOUT 不计为失败"); long startAll = System.currentTimeMillis(); // ── 第1步:敏感词全量命中验证 ── System.out.println("\n[Step1] 敏感词全量命中验证 ..."); List step1 = step1SensitiveCheck(); // ── 第2步:替换词命中验证 ── System.out.println("\n[Step2] 替换词命中验证(JDBC + 可选登录)..."); // 尝试登录(用于替换词检查 API) try { token = login(); tokenStatus = token != null ? "OK" : "FAIL"; } catch (Exception e) { tokenStatus = "FAIL:" + rootMsg(e); System.out.println(" 登录失败: " + e.getMessage()); } System.out.println(" Token: " + tokenStatus); top.put("tokenStatus", tokenStatus); List step2 = step2ReplaceWordCheck(); // ── 第3步:语义关键词命中和穿透验证 ── System.out.println("\n[Step3] 语义关键词命中和穿透验证 ..."); List step3Keywords = step3IndustrySemantic(); List step3Negation = step3NegationPenetration(); // ── 第4步:对外角色按行业测试 ── System.out.println("\n[Step4] 对外角色按行业 UDD 测试(串行,180s 超时)..."); List step4 = step4UddByIndustry(); long totalMs = System.currentTimeMillis() - startAll; // ── 汇总 & 输出 ── System.out.println("\n[Summary] 汇总统计 ..."); List all = new ArrayList<>(); all.addAll(step1); all.addAll(step2); all.addAll(step3Keywords); all.addAll(step3Negation); all.addAll(step4); Map summary = buildSummary(all); StringBuilder out = new StringBuilder(); out.append("{\n"); for (Map.Entry e : top.entrySet()) { out.append(" ").append(js(e.getKey())).append(": ").append(jsv(e.getValue())).append(",\n"); } out.append(" \"steps\": {\n"); out.append(" \"step1_sensitive_check\": ").append(stepBlock(step1)).append(",\n"); out.append(" \"step2_replace_word\": ").append(stepBlock(step2)).append(",\n"); out.append(" \"step3_industry_semantic\": ").append(stepBlock(step3Keywords)).append(",\n"); out.append(" \"step3_negation_penetration\": ").append(stepBlock(step3Negation)).append(",\n"); out.append(" \"step4_udd_by_industry\": ").append(stepBlock(step4)).append("\n"); out.append(" },\n"); out.append(" \"summary\": ").append(summaryJson(summary)).append(",\n"); out.append(" \"totalElapsedMs\": ").append(totalMs).append(",\n"); out.append(" \"finishedAt\": ").append(js(LocalDateTime.now().format(TS))).append("\n"); out.append("}\n"); Files.writeString(REPORT_FILE, out.toString(), StandardCharsets.UTF_8); System.out.println("报告已写入: " + REPORT_FILE); System.out.println("汇总: total=" + summary.get("totalCalls") + " passed=" + summary.get("passed") + " failed=" + summary.get("failed") + " timeout=" + summary.get("timeout") + " skip=" + summary.get("skip")); System.out.println("总耗时: " + totalMs + " ms (" + (totalMs / 1000) + " s)"); } // ════════════════════════════════════════════════════════════════ // 第1步:敏感词全量命中验证 // ════════════════════════════════════════════════════════════════ private static List step1SensitiveCheck() throws Exception { List list = new ArrayList<>(); // 1.1 获取所有敏感词 Resp r = httpGet(VAL_BASE + "/sensitive/list?companyId=1", CHECK_TIMEOUT); Item listIt = new Item(); listIt.module = "step1_sensitive_list"; listIt.input = "GET /sensitive/list?companyId=1"; listIt.output = trunc(r.body, 4000); listIt.elapsedMs = r.elapsedMs; List words = new ArrayList<>(); if (r.error != null) { listIt.passed = false; listIt.status = "ERROR"; listIt.error = r.error; } else { int code = extractInt(r.body, "code", -1); words = extractAllStr(r.body, "word"); listIt.passed = (r.statusCode == 200 && code == 200 && !words.isEmpty()); listIt.status = listIt.passed ? "OK" : "ERROR"; if (!listIt.passed) { listIt.error = "http=" + r.statusCode + " code=" + code + " words=" + words.size(); } } list.add(listIt); System.out.println(" [sensitive-list] passed=" + listIt.passed + " words=" + words.size() + " elapsed=" + listIt.elapsedMs + "ms"); // 1.2 逐词构造测试消息并 check(8 线程并发) if (!words.isEmpty()) { System.out.println(" 开始逐词检测 " + words.size() + " 个敏感词,并发=" + SENSITIVE_CONCURRENCY + " ..."); ExecutorService pool = Executors.newFixedThreadPool(SENSITIVE_CONCURRENCY); List> futures = new ArrayList<>(); for (String w : words) { final String word = w; futures.add(pool.submit(() -> runSensitiveCheck(word))); } pool.shutdown(); long t0 = System.currentTimeMillis(); int done = 0; for (Future f : futures) { try { list.add(f.get(CHECK_TIMEOUT + 30, TimeUnit.SECONDS)); } catch (Exception e) { Item it = new Item(); it.module = "step1_sensitive_check"; it.status = "ERROR"; it.passed = false; it.error = "future-get-error: " + rootMsg(e); list.add(it); } done++; if (done % 20 == 0 || done == futures.size()) { long el = System.currentTimeMillis() - t0; System.out.println(" 敏感词进度: " + done + "/" + futures.size() + " 累计=" + (el / 1000) + "s"); } } pool.awaitTermination(5, TimeUnit.MINUTES); } int pass = 0; for (Item it : list) if (it.passed) pass++; System.out.println(" 敏感词完成: " + pass + "/" + list.size() + " 通过"); return list; } private static Item runSensitiveCheck(String word) { Item it = new Item(); it.module = "step1_sensitive_check"; it.word = word; // 构造包含该词的测试消息;API 实际读取 content 字段 String testMessage = "这是一个测试消息,包含敏感词:" + word; String body = "{\"content\":" + js(testMessage) + ",\"message\":" + js(testMessage) + ",\"industryType\":\"general\",\"companyId\":1}"; // 首次失败则重试一次(缓解后端偶发 500) Resp r = httpPost(VAL_BASE + "/sensitive/check", body, CHECK_TIMEOUT); String retryNote = null; if (isBadResp(r)) { Resp r2 = httpPost(VAL_BASE + "/sensitive/check", body, CHECK_TIMEOUT); retryNote = "retried; first=" + briefResp(r); r = r2; } it.input = body; it.output = trunc(r.body, 1500); it.elapsedMs = r.elapsedMs; if (r.error != null) { it.passed = false; it.status = isTimeoutStr(r.error) ? "TIMEOUT" : "ERROR"; it.error = (retryNote != null ? retryNote + "; " : "") + r.error; } else { int code = extractInt(r.body, "code", -1); Boolean hit = extractBool(r.body, "containsSensitive"); it.containsSensitive = hit; it.isHighRisk = extractBool(r.body, "isHighRisk"); // 命中判定:detectedWords 非空 或 containsSensitive=true boolean detected = r.body != null && r.body.contains("\"detectedWords\":[{"); it.passed = (r.statusCode == 200 && code == 200 && (detected || Boolean.TRUE.equals(hit))); it.status = it.passed ? "OK" : "ERROR"; if (!it.passed) { it.error = (retryNote != null ? retryNote + "; " : "") + "http=" + r.statusCode + " code=" + code + " detected=" + detected + " containsSensitive=" + hit; } else { it.tags.add("命中"); if (Boolean.TRUE.equals(it.isHighRisk)) it.tags.add("高风险"); } } return it; } // ════════════════════════════════════════════════════════════════ // 第2步:替换词命中验证(JDBC + 可选登录) // ════════════════════════════════════════════════════════════════ private static List step2ReplaceWordCheck() { List list = new ArrayList<>(); // 2.1 JDBC 查询主库替换词 List replaceWords = new ArrayList<>(); // {content, replace_text, word_type} Item dbIt = new Item(); dbIt.module = "step2_replace_word_db_query"; dbIt.input = "JDBC: SELECT content,replace_text,word_type FROM company_lobster_replace_word WHERE enabled=1"; long t0 = System.currentTimeMillis(); try { Class.forName("com.mysql.cj.jdbc.Driver"); try (Connection conn = DriverManager.getConnection(DB_URL, DB_USER, DB_PWD); PreparedStatement ps = conn.prepareStatement( "SELECT content, replace_text, word_type FROM company_lobster_replace_word WHERE enabled=1 ORDER BY id"); ResultSet rs = ps.executeQuery()) { while (rs.next()) { replaceWords.add(new String[]{ rs.getString("content"), rs.getString("replace_text"), rs.getString("word_type") }); } } dbIt.elapsedMs = System.currentTimeMillis() - t0; dbIt.output = "查询到 " + replaceWords.size() + " 条启用替换词"; dbIt.passed = !replaceWords.isEmpty(); dbIt.status = dbIt.passed ? "OK" : "ERROR"; if (!dbIt.passed) dbIt.error = "未查询到启用替换词"; System.out.println(" [JDBC] 查询到 " + replaceWords.size() + " 条替换词,耗时=" + dbIt.elapsedMs + "ms"); } catch (Exception e) { dbIt.elapsedMs = System.currentTimeMillis() - t0; dbIt.passed = false; dbIt.status = "ERROR"; dbIt.error = rootMsg(e); System.out.println(" [JDBC] 查询失败: " + rootMsg(e)); } list.add(dbIt); // 2.2 对每个替换词构造测试消息并验证 boolean canCallApi = token != null; System.out.println(" 开始验证 " + replaceWords.size() + " 个替换词" + (canCallApi ? "(已登录,调用 stress-test API)" : "(未登录,仅记录列表)") + " ..."); for (String[] rw : replaceWords) { Item it = new Item(); it.module = "step2_replace_word_check"; it.word = rw[0]; it.replaceText = rw[1]; it.wordType = rw[2]; String testMessage = "我们的产品是" + rw[0] + "的,效果很好"; it.input = testMessage; if (!canCallApi) { // 未登录:直接记录,标注"需要登录验证" it.status = "SKIP"; it.passed = false; it.note = "需要登录验证"; it.tags.add("需要登录验证"); it.elapsedMs = 0; list.add(it); continue; } // 已登录:调用替换词 stress-test API String body = "{\"content\":" + js(testMessage) + ",\"replaceMode\":\"enhanced\"}"; long tt0 = System.currentTimeMillis(); try { HttpResponse resp = postJson(REPLACE_STRESS_URL, body, token, CHECK_TIMEOUT); it.elapsedMs = System.currentTimeMillis() - tt0; it.output = trunc(resp.body(), 1500); if (resp.statusCode() != 200) { it.status = "ERROR"; it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200); it.passed = false; } else { int code = extractInt(resp.body(), "code", -1); if (code != 200) { it.status = "ERROR"; it.error = "code=" + code + " msg=" + extractStr(resp.body(), "msg"); it.passed = false; } else { // 检查响应是否包含原始词或替换后的文本 String data = extractDataObject(resp.body()); boolean hasReplace = data != null && (data.contains(rw[0]) || data.contains("replace")); it.passed = hasReplace || data != null; it.status = it.passed ? "OK" : "ERROR"; if (!it.passed) it.error = "响应未包含替换信息"; if (it.passed) it.tags.add("替换命中"); } } } catch (Exception e) { it.elapsedMs = System.currentTimeMillis() - tt0; it.status = isTimeout(e) ? "TIMEOUT" : "ERROR"; it.error = rootMsg(e); it.passed = false; } list.add(it); } int pass = 0; for (Item it : list) if (it.passed) pass++; System.out.println(" 替换词完成: " + pass + "/" + list.size() + " 通过"); return list; } // ════════════════════════════════════════════════════════════════ // 第3步:语义关键词命中(27 行业 × 3 消息) // ════════════════════════════════════════════════════════════════ private static List step3IndustrySemantic() throws Exception { int total = INDUSTRIES.length * TYPICAL_MESSAGES.length * 2; // keywords + sentiment System.out.println(" 27 行业 × 3 消息 × 2 接口(keywords+sentiment) = " + total + " 次调用,并发=" + SEMANTIC_CONCURRENCY); ExecutorService pool = Executors.newFixedThreadPool(SEMANTIC_CONCURRENCY); List> futures = new ArrayList<>(); for (String[] ind : INDUSTRIES) { final String code = ind[0], label = ind[1]; for (int i = 0; i < TYPICAL_MESSAGES.length; i++) { final String msg = TYPICAL_MESSAGES[i]; final String mlabel = MESSAGE_LABELS[i]; futures.add(pool.submit(() -> runIndustryKeyword(code, label, msg, mlabel))); futures.add(pool.submit(() -> runIndustrySentiment(code, label, msg, mlabel))); } } pool.shutdown(); List list = new ArrayList<>(); int done = 0; long t0 = System.currentTimeMillis(); for (Future f : futures) { try { list.add(f.get(SEMANTIC_TIMEOUT + 30, TimeUnit.SECONDS)); } catch (Exception e) { Item it = new Item(); it.module = "step3_industry_semantic"; it.status = "ERROR"; it.passed = false; it.error = "future-get-error: " + rootMsg(e); list.add(it); } done++; if (done % 40 == 0 || done == futures.size()) { long el = System.currentTimeMillis() - t0; System.out.println(" 语义进度: " + done + "/" + futures.size() + " 累计=" + (el / 1000) + "s"); } } pool.awaitTermination(5, TimeUnit.MINUTES); int pass = 0; for (Item it : list) if (it.passed) pass++; System.out.println(" 行业语义完成: " + pass + "/" + list.size() + " 通过"); return list; } private static Item runIndustryKeyword(String code, String label, String message, String mlabel) { Item it = new Item(); it.module = "step3_industry_keywords"; it.industry = code + "(" + label + ")"; it.input = "message=" + message + " | label=" + mlabel; String url = VAL_BASE + "/semantic/keywords?message=" + URLEncoder.encode(message, StandardCharsets.UTF_8) + "&companyId=1&industryType=" + code; Resp r = httpGet(url, SEMANTIC_TIMEOUT); it.output = trunc(r.body, 1500); it.elapsedMs = r.elapsedMs; if (r.error != null) { it.status = isTimeoutStr(r.error) ? "TIMEOUT" : "ERROR"; it.passed = false; it.error = r.error; } else { int rc = extractInt(r.body, "code", -1); // keywords 接口 data 是字符串数组(如 ["kw1","kw2"]),用专门方法提取 it.keywords = extractKeywordArray(r.body); it.passed = (r.statusCode == 200 && rc == 200 && !it.keywords.isEmpty()); it.status = it.passed ? "OK" : "ERROR"; if (!it.passed) it.error = "http=" + r.statusCode + " code=" + rc + " keywords=" + it.keywords.size(); else { it.tags.add(mlabel); if (message.contains("多少钱")) it.tags.add("购买意向"); if (message.contains("不需要")) it.tags.add("拒绝"); } } return it; } private static Item runIndustrySentiment(String code, String label, String message, String mlabel) { Item it = new Item(); it.module = "step3_industry_sentiment"; it.industry = code + "(" + label + ")"; it.input = "message=" + message + " | label=" + mlabel; String url = VAL_BASE + "/semantic/sentiment?message=" + URLEncoder.encode(message, StandardCharsets.UTF_8) + "&companyId=1&industryType=" + code; Resp r = httpGet(url, SEMANTIC_TIMEOUT); it.output = trunc(r.body, 1500); it.elapsedMs = r.elapsedMs; if (r.error != null) { it.status = isTimeoutStr(r.error) ? "TIMEOUT" : "ERROR"; it.passed = false; it.error = r.error; } else { int rc = extractInt(r.body, "code", -1); Double sent = extractDouble(r.body, "sentiment"); String slabel = extractStr(r.body, "label"); it.sentiment = sent; it.sentimentLabel = slabel; it.passed = (r.statusCode == 200 && rc == 200 && sent != null); it.status = it.passed ? "OK" : "ERROR"; if (!it.passed) it.error = "http=" + r.statusCode + " code=" + rc + " sentiment=" + sent; else { it.tags.add(mlabel); if (slabel != null) it.tags.add(slabel); } } return it; } // ════════════════════════════════════════════════════════════════ // 第3步补充:否定前缀穿透验证(20 组) // ════════════════════════════════════════════════════════════════ private static List step3NegationPenetration() throws Exception { System.out.println(" 20 组否定前缀穿透场景,并发=" + SEMANTIC_CONCURRENCY); ExecutorService pool = Executors.newFixedThreadPool(SEMANTIC_CONCURRENCY); List> futures = new ArrayList<>(); for (int i = 0; i < NEGATION_CASES.length; i++) { final int idx = i; futures.add(pool.submit(() -> runNegationCase(idx))); } pool.shutdown(); List list = new ArrayList<>(); for (Future f : futures) { try { list.add(f.get(SEMANTIC_TIMEOUT + 30, TimeUnit.SECONDS)); } catch (Exception e) { Item it = new Item(); it.module = "step3_negation_penetration"; it.status = "ERROR"; it.passed = false; it.error = "future-get-error: " + rootMsg(e); list.add(it); } } pool.awaitTermination(5, TimeUnit.MINUTES); int pass = 0; for (Item it : list) if (it.passed) pass++; System.out.println(" 否定穿透完成: " + pass + "/" + list.size() + " 通过"); return list; } private static Item runNegationCase(int idx) { String[] cas = NEGATION_CASES[idx]; String message = cas[0]; String expect = cas[1]; // REJECT / NEGATIVE / POSITIVE String desc = cas[2]; Item it = new Item(); it.module = "step3_negation_penetration"; it.industry = "general"; it.word = message; it.input = "message=" + message + " | expect=" + expect + " | " + desc; it.tags.add("否定穿透"); it.tags.add("场景" + (idx + 1)); // 同时调用 keywords 和 sentiment String kwUrl = VAL_BASE + "/semantic/keywords?message=" + URLEncoder.encode(message, StandardCharsets.UTF_8) + "&companyId=1"; String sentUrl = VAL_BASE + "/semantic/sentiment?message=" + URLEncoder.encode(message, StandardCharsets.UTF_8) + "&companyId=1"; long t0 = System.currentTimeMillis(); Resp kwResp = httpGet(kwUrl, SEMANTIC_TIMEOUT); Resp sentResp = httpGet(sentUrl, SEMANTIC_TIMEOUT); it.elapsedMs = System.currentTimeMillis() - t0; it.output = "keywords=" + trunc(kwResp.body, 500) + " | sentiment=" + trunc(sentResp.body, 500); if (kwResp.error != null || sentResp.error != null) { it.status = "ERROR"; it.passed = false; it.error = "kwErr=" + kwResp.error + " sentErr=" + sentResp.error; return it; } int kwCode = extractInt(kwResp.body, "code", -1); int sentCode = extractInt(sentResp.body, "code", -1); it.keywords = extractKeywordArray(kwResp.body); Double sentiment = extractDouble(sentResp.body, "sentiment"); String slabel = extractStr(sentResp.body, "label"); it.sentiment = sentiment; it.sentimentLabel = slabel; if (kwCode != 200 || sentCode != 200 || sentiment == null) { it.status = "ERROR"; it.passed = false; it.error = "kwCode=" + kwCode + " sentCode=" + sentCode + " sentiment=" + sentiment; return it; } // 穿透判定逻辑: // 1. REJECT/NEGATIVE(否定):sentiment 应 <= 0.1(非明显正面), // 且 keywords 不应包含明显的购买意向词(如"购买""下单""成交""购买意向") // 2. POSITIVE(双重否定肯定):sentiment 应 >= -0.1(非明显负面) boolean penetrationOk; String reason; if ("POSITIVE".equals(expect)) { // 双重否定应判为肯定:sentiment 不应明显负面 penetrationOk = sentiment >= -0.15; reason = "双重否定期望肯定, sentiment=" + sentiment + " threshold>=-0.15"; } else { // REJECT 或 NEGATIVE:不应命中购买意向,sentiment 不应明显正面 boolean hasPurchaseKw = hasPurchaseIntentKeyword(it.keywords); boolean sentimentNotPositive = sentiment <= 0.15; penetrationOk = !hasPurchaseKw && sentimentNotPositive; reason = "否定期望非购买, hasPurchaseKw=" + hasPurchaseKw + " sentiment=" + sentiment + " threshold<=0.15"; } it.passed = penetrationOk; it.status = it.passed ? "OK" : "ERROR"; it.note = desc + " | " + reason; if (!it.passed) it.error = reason; return it; } /** 判断关键词列表是否含购买意向词(用于否定穿透判定) */ private static boolean hasPurchaseIntentKeyword(List keywords) { if (keywords == null) return false; for (String kw : keywords) { if (kw == null) continue; for (String p : new String[]{"购买", "下单", "成交", "购买意向", "想买", "打算买", "订购", "付款"}) { if (kw.contains(p)) return true; } } return false; } // ════════════════════════════════════════════════════════════════ // 第4步:对外角色按行业 UDD 测试(串行) // ════════════════════════════════════════════════════════════════ private static List step4UddByIndustry() { List list = new ArrayList<>(); for (String[] ind : INDUSTRIES) { String code = ind[0], label = ind[1]; System.out.println(" UDD [" + code + "] " + label + " ..."); Item it = uddDebug(code, label); list.add(it); System.out.println(" " + it.status + " " + it.elapsedMs + "ms" + (it.passed ? " passed" : (it.error != null ? " err=" + trunc(it.error, 80) : ""))); } int pass = 0, timeout = 0; for (Item it : list) { if (it.passed) pass++; if ("TIMEOUT".equals(it.status)) timeout++; } System.out.println(" UDD 完成: " + pass + "/" + list.size() + " 通过, timeout=" + timeout + "(不算失败)"); return list; } private static Item uddDebug(String code, String label) { Item it = new Item(); it.module = "step4_udd_by_industry"; it.industry = code + "(" + label + ")"; String message = "你好,介绍一下你们的服务"; it.input = "message=" + message + " | industryType=" + code + " | userKey=role_" + code; String body = "{\"message\":" + js(message) + ",\"industryType\":\"" + code + "\"" + ",\"userKey\":\"role_" + code + "\"" + ",\"persistState\":false}"; long t0 = System.currentTimeMillis(); try { HttpResponse resp = postJson(VAL_BASE + "/udd/debug", body, null, UDD_TIMEOUT); it.elapsedMs = System.currentTimeMillis() - t0; it.output = trunc(resp.body(), 1500); if (resp.statusCode() != 200) { it.status = "ERROR"; it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200); it.passed = false; return it; } int rc = extractInt(resp.body(), "code", -1); if (rc != 200) { it.status = "ERROR"; it.error = "code=" + rc + " msg=" + extractStr(resp.body(), "msg"); it.passed = false; return it; } String data = extractDataObject(resp.body()); String reply = extractStr(data, "reply"); it.output = trunc(reply, 1500); it.tags = tagReply(reply, message); it.passed = reply != null && !reply.isEmpty(); it.status = it.passed ? "OK" : "ERROR"; if (!it.passed) it.error = "空回复"; return it; } catch (Exception e) { it.elapsedMs = System.currentTimeMillis() - t0; boolean timeout = isTimeout(e); it.status = timeout ? "TIMEOUT" : "ERROR"; it.error = rootMsg(e); // TIMEOUT 不算失败 it.passed = timeout; if (timeout) it.note = "超时但不算失败"; return it; } } // ════════════════════════════════════════════════════════════════ // 登录(注入验证码到 Redis 后调用 /login) // ════════════════════════════════════════════════════════════════ private static String login() throws Exception { String uuid = "ksval" + System.currentTimeMillis(); seedCaptcha(uuid); String body = "{\"username\":\"admin\",\"password\":\"Admin@123456\"," + "\"tenantCode\":\"cs1\",\"code\":\"1234\",\"uuid\":\"" + uuid + "\"}"; HttpResponse resp = postJson(LOGIN_URL, body, null, LOGIN_TIMEOUT); if (resp.statusCode() != 200) { throw new IOException("login HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200)); } String tk = extractStr(resp.body(), "token"); if (tk == null || tk.isEmpty()) { throw new IOException("login no token: " + trunc(resp.body(), 200)); } return tk; } /** 向 Redis 注入已知验证码(FastJSON 序列化形式:带双引号的字符串) */ private static void seedCaptcha(String uuid) throws IOException { String key = "tenantid:system:captcha_codes:" + uuid; String value = "\"1234\""; try (Socket sock = new Socket("127.0.0.1", 6379)) { sock.setSoTimeout(5000); OutputStream out = sock.getOutputStream(); String cmd = "*5\r\n$3\r\nSET\r\n$" + key.length() + "\r\n" + key + "\r\n$" + value.length() + "\r\n" + value + "\r\n$2\r\nEX\r\n$3\r\n600\r\n"; out.write(cmd.getBytes(StandardCharsets.US_ASCII)); out.flush(); byte[] buf = new byte[128]; int n = sock.getInputStream().read(buf); if (n <= 0) throw new IOException("Redis no response"); String resp = new String(buf, 0, n, StandardCharsets.US_ASCII); if (!resp.startsWith("+OK")) throw new IOException("Redis SET failed: " + resp.trim()); } } // ════════════════════════════════════════════════════════════════ // HTTP 工具 // ════════════════════════════════════════════════════════════════ static final class Resp { int statusCode; String body; long elapsedMs; String error; Resp(int s, String b, long e, String err) { statusCode = s; body = b; elapsedMs = e; error = err; } } private static Resp httpGet(String url, int timeoutSec) { long t0 = System.currentTimeMillis(); try { HttpRequest req = HttpRequest.newBuilder() .uri(URI.create(url)) .timeout(Duration.ofSeconds(timeoutSec)) .GET() .build(); HttpResponse resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); long el = System.currentTimeMillis() - t0; return new Resp(resp.statusCode(), resp.body(), el, null); } catch (Exception e) { return new Resp(0, "", System.currentTimeMillis() - t0, rootMsg(e)); } } private static Resp httpPost(String url, String body, int timeoutSec) { long t0 = System.currentTimeMillis(); try { HttpRequest req = HttpRequest.newBuilder() .uri(URI.create(url)) .timeout(Duration.ofSeconds(timeoutSec)) .header("Content-Type", "application/json; charset=utf-8") .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)) .build(); HttpResponse resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); long el = System.currentTimeMillis() - t0; return new Resp(resp.statusCode(), resp.body(), el, null); } catch (Exception e) { return new Resp(0, "", System.currentTimeMillis() - t0, rootMsg(e)); } } private static HttpResponse postJson(String url, String body, String auth, int timeoutSec) throws Exception { HttpRequest.Builder b = HttpRequest.newBuilder() .uri(URI.create(url)) .timeout(Duration.ofSeconds(timeoutSec)) .header("Content-Type", "application/json; charset=utf-8") .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)); if (auth != null) b.header("Authorization", auth); return httpClient.send(b.build(), HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); } private static boolean isBadResp(Resp r) { if (r.error != null) return true; int code = extractInt(r.body, "code", -1); return r.statusCode != 200 || code == 500; } private static String briefResp(Resp r) { if (r.error != null) return "err:" + r.error; return "http=" + r.statusCode + " code=" + extractInt(r.body, "code", -1); } // ════════════════════════════════════════════════════════════════ // JSON 解析(无依赖,正则/索引) // ════════════════════════════════════════════════════════════════ private static final Pattern STR_PAT = Pattern.compile("\"((?:\\\\.|[^\"\\\\])*)\""); private static int extractInt(String json, String field, int def) { if (json == null) return def; Matcher m = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*(-?\\d+)").matcher(json); return m.find() ? Integer.parseInt(m.group(1)) : def; } private static Double extractDouble(String json, String field) { if (json == null) return null; Matcher m = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*(-?\\d+(?:\\.\\d+)?(?:[Ee][-+]?\\d+)?)") .matcher(json); if (m.find()) { try { return Double.parseDouble(m.group(1)); } catch (Exception e) { return null; } } // 处理 null 字面量 Matcher nm = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*null").matcher(json); if (nm.find()) return null; return null; } private static Boolean extractBool(String json, String field) { if (json == null) return null; Matcher m = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*(true|false)").matcher(json); return m.find() ? Boolean.parseBoolean(m.group(1)) : null; } private static String extractStr(String json, String field) { if (json == null) return null; Matcher m = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*\"((?:\\\\.|[^\"\\\\])*)\"").matcher(json); if (m.find()) return unescape(m.group(1)); return null; } private static List extractAllStr(String json, String field) { List out = new ArrayList<>(); if (json == null) return out; Matcher m = Pattern.compile("\"" + Pattern.quote(field) + "\"\\s*:\\s*\"((?:\\\\.|[^\"\\\\])*)\"").matcher(json); while (m.find()) out.add(unescape(m.group(1))); return out; } /** 提取 keywords 接口 data 中的字符串数组(data 是 ["a","b"] 形式) */ private static List extractKeywordArray(String json) { List out = new ArrayList<>(); if (json == null) return out; String data = extractDataObject(json); if (data == null || data.isEmpty()) return out; Matcher m = STR_PAT.matcher(data); while (m.find()) out.add(unescape(m.group(1))); return out; } /** 提取 "data":{...} 或 "data":[...] 的内容(按括号配平) */ private static String extractDataObject(String json) { if (json == null) return ""; String key = "\"data\""; int i = json.indexOf(key); if (i < 0) return ""; int colon = json.indexOf(':', i + key.length()); if (colon < 0) return ""; int j = colon + 1; while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++; if (j >= json.length()) return ""; char open = json.charAt(j); if (open != '{' && open != '[') { int start = j; while (j < json.length() && json.charAt(j) != ',' && json.charAt(j) != '}') j++; return json.substring(start, j); } char close = (open == '{') ? '}' : ']'; int depth = 0; int start = j; boolean inStr = false; while (j < json.length()) { char c = json.charAt(j); if (c == '\\' && inStr) { j += 2; continue; } if (c == '"') inStr = !inStr; if (!inStr) { if (c == open) depth++; else if (c == close) { depth--; if (depth == 0) return json.substring(start, j + 1); } } j++; } return json.substring(start); } private static String unescape(String s) { if (s == null || s.indexOf('\\') < 0) return s; StringBuilder sb = new StringBuilder(s.length()); for (int i = 0; i < s.length(); i++) { char ch = s.charAt(i); if (ch == '\\' && i + 1 < s.length()) { char n = s.charAt(i + 1); switch (n) { case '"': sb.append('"'); i++; break; case '\\': sb.append('\\'); i++; break; case '/': sb.append('/'); i++; break; case 'n': sb.append('\n'); i++; break; case 'r': sb.append('\r'); i++; break; case 't': sb.append('\t'); i++; break; case 'u': if (i + 5 < s.length()) { try { sb.append((char) Integer.parseInt(s.substring(i + 2, i + 6), 16)); i += 5; } catch (NumberFormatException e) { sb.append(ch); } } else sb.append(ch); break; default: sb.append(ch); } } else sb.append(ch); } return sb.toString(); } // ════════════════════════════════════════════════════════════════ // 标签与汇总 // ════════════════════════════════════════════════════════════════ private static List tagReply(String reply, String inputMessage) { List tags = new ArrayList<>(); if (reply == null || reply.isEmpty()) { tags.add("空回复"); return tags; } if (reply.contains("[UDD调试预览]") || reply.contains("[待追问") || reply.contains("待追问:") || reply.contains("根据画像") || reply.contains("禁止暴露")) { tags.add("机械"); } if (reply.contains("您好") || reply.contains("很高兴") || reply.contains("请问") || reply.contains("帮您") || reply.contains("为您")) tags.add("亲切"); int particle = 0; for (String p : new String[]{"嗯", "呢", "呀", "哈", "吧", "哦", "啦", "呐", "诶", "哟"}) { int idx = -1; while ((idx = reply.indexOf(p, idx + 1)) >= 0) particle++; } if (particle >= 1) tags.add("口语化"); if (particle == 0 && !reply.contains("您好") && !reply.contains("请问") && reply.length() < 40) { tags.add("机械"); } if (reply.contains("AI") || reply.contains("机器人") || reply.contains("智能客服")) tags.add("暴露AI"); if (reply.contains("百分百") || reply.contains("包治") || (reply.contains("保证") && reply.contains("治愈"))) { tags.add("过度承诺"); } return tags; } private static Map buildSummary(List all) { Map s = new LinkedHashMap<>(); int total = all.size(), passed = 0, failed = 0, timeout = 0, error = 0, skip = 0; long totalMs = 0, maxMs = 0, minMs = Long.MAX_VALUE; Map byStatus = new LinkedHashMap<>(); Map byModule = new LinkedHashMap<>(); Map byModulePass = new LinkedHashMap<>(); Map byTag = new LinkedHashMap<>(); for (Item it : all) { totalMs += it.elapsedMs; if (it.elapsedMs > maxMs) maxMs = it.elapsedMs; if (it.elapsedMs < minMs) minMs = it.elapsedMs; byStatus.merge(it.status, 1, Integer::sum); if ("OK".equals(it.status) && it.passed) passed++; else if ("TIMEOUT".equals(it.status)) timeout++; else if ("SKIP".equals(it.status)) skip++; else error++; if (!it.passed && !"TIMEOUT".equals(it.status)) failed++; String mod = it.module == null ? "?" : it.module; byModule.merge(mod, 1, Integer::sum); if (it.passed) byModulePass.merge(mod, 1, Integer::sum); for (String t : it.tags) byTag.merge(t, 1, Integer::sum); } s.put("totalCalls", total); s.put("passed", passed); s.put("failed", failed); s.put("timeout", timeout); s.put("skip", skip); s.put("error", error); s.put("avgMs", total == 0 ? 0 : totalMs / total); s.put("maxMs", maxMs); s.put("minMs", minMs == Long.MAX_VALUE ? 0 : minMs); s.put("byStatus", byStatus); s.put("byModule", byModule); s.put("byModulePass", byModulePass); s.put("byTag", byTag); return s; } private static String stepBlock(List items) { int passed = 0; long ms = 0; for (Item it : items) { if (it.passed) passed++; ms += it.elapsedMs; } StringBuilder sb = new StringBuilder(); sb.append("{\"total\":").append(items.size()).append(",\"passed\":").append(passed) .append(",\"totalMs\":").append(ms).append(",\"items\":["); for (int i = 0; i < items.size(); i++) { if (i > 0) sb.append(","); sb.append(items.get(i).toJson()); } sb.append("]}"); return sb.toString(); } private static String summaryJson(Map s) { StringBuilder sb = new StringBuilder("{"); int c = 0; for (Map.Entry e : s.entrySet()) { if (c++ > 0) sb.append(","); sb.append(js(e.getKey())).append(":").append(jsv(e.getValue())); } sb.append("}"); return sb.toString(); } // ════════════════════════════════════════════════════════════════ // JSON 序列化辅助 // ════════════════════════════════════════════════════════════════ private static String js(String s) { if (s == null) return "null"; StringBuilder sb = new StringBuilder("\""); for (int i = 0; i < s.length(); i++) { char c = s.charAt(i); switch (c) { case '"': sb.append("\\\""); break; case '\\': sb.append("\\\\"); break; case '\n': sb.append("\\n"); break; case '\r': sb.append("\\r"); break; case '\t': sb.append("\\t"); break; default: if (c < 0x20) sb.append(String.format("\\u%04x", (int) c)); else sb.append(c); } } sb.append("\""); return sb.toString(); } private static String jsArray(List list) { StringBuilder sb = new StringBuilder("["); for (int i = 0; i < list.size(); i++) { if (i > 0) sb.append(","); sb.append(js(list.get(i))); } sb.append("]"); return sb.toString(); } @SuppressWarnings("unchecked") private static String jsv(Object v) { if (v == null) return "null"; if (v instanceof Number) return v.toString(); if (v instanceof Boolean) return v.toString(); if (v instanceof String) return js((String) v); if (v instanceof Map) { StringBuilder sb = new StringBuilder("{"); int c = 0; for (Map.Entry e : ((Map) v).entrySet()) { if (c++ > 0) sb.append(","); sb.append(js(e.getKey())).append(":").append(jsv(e.getValue())); } sb.append("}"); return sb.toString(); } if (v instanceof Iterable) { StringBuilder sb = new StringBuilder("["); int c = 0; for (Object o : (Iterable) v) { if (c++ > 0) sb.append(","); sb.append(jsv(o)); } sb.append("]"); return sb.toString(); } return js(v.toString()); } // ════════════════════════════════════════════════════════════════ // 通用辅助 // ════════════════════════════════════════════════════════════════ private static String trunc(String s, int max) { if (s == null) return null; return s.length() <= max ? s : s.substring(0, max) + "..."; } private static boolean isTimeout(Throwable e) { Throwable c = e; while (c != null) { if (c instanceof java.net.http.HttpTimeoutException) return true; if (c instanceof TimeoutException) return true; String n = c.getClass().getName(); if (n.contains("TimeoutException")) return true; c = c.getCause(); } return false; } private static boolean isTimeoutStr(String s) { return s != null && (s.contains("timed out") || s.contains("TimeoutException") || s.contains("HttpTimeoutException")); } private static String rootMsg(Throwable e) { Throwable c = e; while (c.getCause() != null && c.getCause() != c) c = c.getCause(); String msg = c.getMessage(); return (msg != null ? msg : c.getClass().getSimpleName()).replace("\n", " "); } }