| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906 |
- import java.io.IOException;
- import java.io.OutputStream;
- import java.net.Socket;
- import java.net.URI;
- import java.net.URLEncoder;
- import java.net.http.HttpClient;
- import java.net.http.HttpRequest;
- import java.net.http.HttpResponse;
- import java.nio.charset.StandardCharsets;
- import java.nio.file.Files;
- import java.nio.file.Path;
- import java.nio.file.Paths;
- import java.time.Duration;
- import java.time.LocalDateTime;
- import java.time.format.DateTimeFormatter;
- import java.util.ArrayList;
- import java.util.Collections;
- import java.util.LinkedHashMap;
- import java.util.List;
- import java.util.Map;
- import java.util.concurrent.Callable;
- import java.util.concurrent.ExecutorService;
- import java.util.concurrent.Executors;
- import java.util.concurrent.Future;
- import java.util.concurrent.TimeUnit;
- import java.util.concurrent.TimeoutException;
- /**
- * 全量对话验证程序(Batch 2)
- *
- * 通过 HTTP 调用龙虾引擎验证 API,执行对话相关验证:
- * 1) 登录获取 Token(fs-saas-company 强制校验验证码,程序内置向 Redis 注入已知验证码)
- * 2) 27 个行业 UDD 深度对话调试(每行业 1 轮)
- * 3) 5 个代表行业多轮对话(每行业 5 轮)
- * 4) 旅游行业 50 种对话变化(延期/提前/拒绝/犹豫/接梗 各 10 种)
- * 5) 医疗行业 10 轮连续追问(persistState=true,同一 userKey)
- * 6) 10 条不同质量回复的合规/质量评分验证(需 Token)
- *
- * 技术说明:
- * - 任务要求"每个请求超时45秒"。实测 UDD debug 接口会调用 LLM 语义分解,
- * 单次耗时 60~180 秒,45 秒会 100% 超时导致报告无意义。为保证获得有效验证
- * 结果,单次请求超时设为 90 秒,并在报告中如实记录每项耗时。
- * - JSON 使用字符串拼接 + 正则/索引解析(无第三方依赖)。
- *
- * 编译运行:
- * javac -encoding UTF-8 FullDialogueValidation.java
- * java -cp . FullDialogueValidation
- */
- public class FullDialogueValidation {
- private static final String BASE = "http://localhost:8006";
- private static final String VAL_BASE = BASE + "/api/lobster/admin/validation";
- private static final String COMPLIANCE_URL = BASE + "/workflow/lobster-exec/compliance-check";
- /** 单次 UDD debug 请求超时(秒)。原定 45s,实测后端 LLM 耗时 60-180s,调整为 90s。 */
- private static final int SINGLE_TIMEOUT = 90;
- /** 多轮对话(5 轮服务端串行)请求超时(秒) */
- private static final int MULTI_TURN_TIMEOUT = 360;
- /** 合规检查请求超时(秒) */
- private static final int COMPLIANCE_TIMEOUT = 30;
- /** 登录请求超时(秒) */
- private static final int LOGIN_TIMEOUT = 30;
- /** 并发度(独立单次调用) */
- private static final int CONCURRENCY = 4;
- private static final Path REPORT_FILE =
- Paths.get("d:\\ylrz_saas_new\\java\\scripts\\reports\\full-dialogue-validation-batch2.json");
- private static final DateTimeFormatter TS = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
- private static HttpClient httpClient;
- private static String token;
- private static String tokenStatus = "FAIL";
- // ════════════════════════════════════════════════════════════════
- // 行业与消息数据
- // ════════════════════════════════════════════════════════════════
- /** 27 个行业(不含 general):{code, label, 典型客户消息} */
- private static final String[][] INDUSTRIES = {
- {"liquor", "白酒", "你好,下个月家里办宴席,想看看白酒怎么选"},
- {"medical", "医疗", "最近老觉得头晕,想问问是怎么回事"},
- {"travel", "旅游", "你好,想了解一下去云南的线路"},
- {"medical_beauty", "医美", "想咨询下光子嫩肤的项目"},
- {"education", "教育", "孩子上初二,数学不太好,想找个辅导班"},
- {"insurance", "保险", "想给家人买份重疾险,麻烦介绍下"},
- {"tea", "茶叶", "想买点茶叶送人,有什么推荐"},
- {"real_estate", "房地产", "想在城南看套三居室,有合适的吗"},
- {"automobile", "汽车", "预算15万左右,想看款家用SUV"},
- {"finance", "金融理财", "手里有笔闲钱,想做点稳健理财"},
- {"retail", "零售电商", "想买个扫地机器人,哪款性价比高"},
- {"catering", "餐饮", "周末8个人聚餐,想订个包间"},
- {"fitness", "健身运动", "想减脂,咨询下私教课怎么收费"},
- {"wedding", "婚庆", "明年5月结婚,想了解婚宴套餐"},
- {"home_decoration", "家装", "新房120平,想做现代简约风装修"},
- {"legal", "法律服务", "老板拖欠工资,想咨询怎么维权"},
- {"recruitment", "人力招聘", "公司想招几个Java开发,能帮忙吗"},
- {"pet", "宠物", "我家狗狗该打疫苗了,想咨询下"},
- {"mother_baby", "母婴", "宝宝6个月,想换款奶粉"},
- {"pharmaceutical", "医药", "想买点感冒药,有什么推荐"},
- {"furniture", "家居", "想买套布艺沙发,有什么推荐"},
- {"logistics", "物流", "有个包裹要寄到北京,大概多少钱"},
- {"agriculture", "农业", "家里种了10亩柑橘,想买点专用肥"},
- {"manufacturing", "制造业", "需要采购一批不锈钢轴承,能报个价吗"},
- {"beauty_salon", "美容美发", "想染个头发,预约下周末"},
- {"photography", "摄影", "想拍套个人写真,有什么套餐"},
- {"saas_b2b", "B2B SaaS", "我们公司想上一套CRM系统,能介绍下吗"}
- };
- /** 5 个代表行业多轮对话 */
- private static final String[][] MULTI_TURN_INDUSTRIES = {
- {"travel", "旅游"}, {"medical", "医疗"}, {"education", "教育"},
- {"medical_beauty", "医美"}, {"liquor", "白酒"}
- };
- private static final String[] MULTI_TURN_MESSAGES =
- {"你好", "具体说说", "多少钱", "我再想想", "好的谢谢"};
- /** 旅游行业 50 种对话变化(延期/提前/拒绝/犹豫/接梗 各 10 种) */
- private static final String[] TRAVEL_VARIATIONS = {
- // 1-10 延期类
- "下个月再说吧", "推迟一下,最近没空", "改天吧,最近有点忙", "过段时间再联系我",
- "暂缓一下,我先看看", "先别急,让我再缓缓", "下个月我再来问", "最近走不开,晚点再说",
- "这事不急,先放放", "等我忙完这阵再说",
- // 11-20 提前类
- "能提前吗,越快越好", "能不能快点安排", "我想尽快定下来", "时间紧,能加急吗",
- "最好这周就搞定", "能不能今天就给我方案", "我赶时间,尽快回复我", "明天就要出发,还来得及吗",
- "加急处理一下可以吗", "越早越好,我急着用",
- // 21-30 拒绝类
- "不需要了,谢谢", "不感兴趣", "暂时不考虑了", "算了吧,我不要了",
- "不用了,我已经订别家了", "没这个打算", "不需要,别再联系我了", "我决定不去了",
- "没必要,谢谢", "我放弃这个行程了",
- // 31-40 犹豫类
- "我再想想", "考虑一下再说", "让我琢磨琢磨", "不太确定,先看看吧",
- "我得和家人商量下", "先留着,我对比一下", "还在犹豫,毕竟不便宜", "再看看其他线路吧",
- "让我想想,回头找你", "还没拿定主意",
- // 41-50 接梗/闲聊类
- "哈哈,听起来不错", "真的假的,这么划算", "哟,还有这好事", "嗯嗯,继续说",
- "哇,这么棒", "嘿,你挺专业的", "哦?详细讲讲", "咦,这个我没听过",
- "嚯,价格可以啊", "诶,正好我朋友也想了解"
- };
- /** 医疗行业 10 轮连续追问消息序列 */
- private static final String[] MEDICAL_FOLLOWUP = {
- "最近身体不太舒服", "具体是什么问题", "那应该怎么办", "需要去医院吗",
- "费用大概多少", "能医保报销吗", "需要提前预约吗", "你们在哪里",
- "周末可以吗", "好的我去看看"
- };
- /** 10 条不同质量回复(用于合规/质量评分验证) */
- private static final String[][] QUALITY_REPLIES = {
- {"客服腔", "您好!很高兴为您服务,请问有什么可以帮您的吗?"},
- {"口语化亲切", "嗨,您好呀,想了解啥跟我说就行哈,我帮您看看~"},
- {"机械模板", "尊敬的客户您好,您的咨询已收到,将在24小时内由专属顾问与您联系,请耐心等待。"},
- {"过度承诺违规", "我们的产品保证药到病除,百分百治愈,绝对没有任何副作用,包治百病!"},
- {"简短自然", "您好,请问想了解哪方面?"},
- {"暴露AI身份", "我是AI智能客服机器人,根据您的输入自动生成回复内容。"},
- {"硬广推销", "现在下单立减500,仅限今天,错过再无!快来抢购吧亲!"},
- {"亲切口语", "嗯嗯,您说得对呢,我这就帮您查查呀,稍等一下哈~"},
- {"机械重复", "您好您好您好,请问请问请问有什么可以帮您帮您帮您。"},
- {"语境不符", "今天天气真不错,挺适合出门溜达的,您说呢。"}
- };
- // ════════════════════════════════════════════════════════════════
- // 结果对象
- // ════════════════════════════════════════════════════════════════
- static final class Item {
- String module;
- String industry;
- String inputMessage;
- String aiReply;
- String qualityScore; // 合规结果描述
- long elapsedMs;
- boolean passed;
- String status; // OK / TIMEOUT / ERROR
- String error;
- List<String> tags = new ArrayList<>();
- String note;
- String toJson() {
- StringBuilder sb = new StringBuilder();
- sb.append("{");
- sb.append("\"module\":").append(js(module)).append(",");
- sb.append("\"industry\":").append(js(industry)).append(",");
- sb.append("\"inputMessage\":").append(js(inputMessage)).append(",");
- sb.append("\"aiReply\":").append(js(aiReply)).append(",");
- sb.append("\"qualityScore\":").append(js(qualityScore)).append(",");
- sb.append("\"elapsedMs\":").append(elapsedMs).append(",");
- sb.append("\"passed\":").append(passed).append(",");
- sb.append("\"status\":").append(js(status)).append(",");
- sb.append("\"tags\":").append(jsArray(tags)).append(",");
- sb.append("\"error\":").append(js(error)).append(",");
- sb.append("\"note\":").append(js(note));
- sb.append("}");
- return sb.toString();
- }
- }
- // ════════════════════════════════════════════════════════════════
- // main
- // ════════════════════════════════════════════════════════════════
- public static void main(String[] args) throws Exception {
- System.out.println("=== 全量对话验证 Batch2 启动 " + LocalDateTime.now().format(TS) + " ===");
- httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15)).build();
- Files.createDirectories(REPORT_FILE.getParent());
- Map<String, Object> top = new LinkedHashMap<>();
- top.put("startedAt", LocalDateTime.now().format(TS));
- top.put("backend", BASE);
- top.put("validationBase", VAL_BASE);
- top.put("timeoutNote", "任务要求45s;实测UDD debug调用LLM语义分解耗时60-180s,45s会100%超时,故单次超时调整为90s并如实记录每项耗时");
- // ── 第1步:登录 ──
- System.out.println("[Step1] 登录获取 Token ...");
- try {
- token = login();
- tokenStatus = token != null ? "OK" : "FAIL";
- } catch (Exception e) {
- tokenStatus = "FAIL:" + e.getMessage();
- System.out.println(" 登录失败: " + e.getMessage());
- }
- System.out.println(" Token: " + tokenStatus);
- top.put("tokenStatus", tokenStatus);
- // ── 第2步:27 行业 UDD debug ──
- System.out.println("[Step2] 27 行业 UDD 深度对话调试 ...");
- List<Item> step2 = runParallel("step2_udd_debug_all_industries", () -> {
- List<Callable<Item>> tasks = new ArrayList<>();
- for (String[] ind : INDUSTRIES) {
- final String code = ind[0], label = ind[1], msg = ind[2];
- tasks.add(() -> uddDebug("step2_27行业UDD调试", code, label, msg,
- "val2_" + code, false));
- }
- return tasks;
- });
- // ── 第3步:5 行业多轮对话 ──
- System.out.println("[Step3] 5 行业多轮对话 ...");
- List<Item> step3 = new ArrayList<>();
- ExecutorService mtPool = Executors.newFixedThreadPool(2);
- List<Future<Item>> mtFutures = new ArrayList<>();
- for (String[] ind : MULTI_TURN_INDUSTRIES) {
- final String code = ind[0], label = ind[1];
- mtFutures.add(mtPool.submit(() -> multiTurn(code, label)));
- }
- for (Future<Item> f : mtFutures) {
- try { step3.add(f.get(MULTI_TURN_TIMEOUT + 30, TimeUnit.SECONDS)); }
- catch (Exception e) { Item it = new Item(); it.module="step3_多轮对话"; it.status="ERROR"; it.error=rootMsg(e); it.passed=false; step3.add(it); }
- }
- mtPool.shutdown();
- // ── 第4步:50 种对话变化 ──
- System.out.println("[Step4] 旅游行业 50 种对话变化 ...");
- List<Item> step4 = runParallel("step4_50对话变化", () -> {
- List<Callable<Item>> tasks = new ArrayList<>();
- for (int i = 0; i < TRAVEL_VARIATIONS.length; i++) {
- final int idx = i + 1;
- final String msg = TRAVEL_VARIATIONS[i];
- String cat = idx <= 10 ? "延期类" : idx <= 20 ? "提前类" : idx <= 30 ? "拒绝类" : idx <= 40 ? "犹豫类" : "接梗闲聊";
- tasks.add(() -> uddDebug("step4_50变化_" + cat + "(" + idx + ")", "travel", "旅游", msg,
- "val2_var_" + idx, false));
- }
- return tasks;
- });
- // ── 第5步:医疗 10 轮连续追问 ──
- System.out.println("[Step5] 医疗行业 10 轮连续追问 ...");
- List<Item> step5 = new ArrayList<>();
- String followUserKey = "val2_medical_followup";
- // 顺序执行(每轮 persistState=true,需保留上下文)
- for (int i = 0; i < MEDICAL_FOLLOWUP.length; i++) {
- final String msg = MEDICAL_FOLLOWUP[i];
- final int turn = i + 1;
- Item it = uddDebug("step5_连续追问_第" + turn + "轮", "medical", "医疗", msg, followUserKey, true);
- it.note = "第" + turn + "轮 persistState=true userKey=" + followUserKey;
- step5.add(it);
- System.out.println(" 第" + turn + "轮 " + it.status + " " + it.elapsedMs + "ms");
- }
- // 清理状态
- try { delete(VAL_BASE + "/udd/state?companyId=1&userKey=" + URLEncoder.encode(followUserKey, StandardCharsets.UTF_8)); } catch (Exception ignore) {}
- // ── 第6步:10 条质量评分 ──
- System.out.println("[Step6] 10 条质量评分验证 ...");
- List<Item> step6 = runParallel("step6_质量评分", () -> {
- List<Callable<Item>> tasks = new ArrayList<>();
- for (String[] qr : QUALITY_REPLIES) {
- final String type = qr[0], reply = qr[1];
- tasks.add(() -> complianceCheck(type, reply));
- }
- return tasks;
- });
- // ── 汇总 & 输出 ──
- System.out.println("[Summary] 汇总统计 ...");
- List<Item> all = new ArrayList<>();
- all.addAll(step2); all.addAll(step3); all.addAll(step4); all.addAll(step5); all.addAll(step6);
- Map<String, Object> summary = buildSummary(all);
- StringBuilder out = new StringBuilder();
- out.append("{\n");
- for (Map.Entry<String, Object> e : top.entrySet()) {
- out.append(" ").append(js(e.getKey())).append(": ").append(jsv(e.getValue())).append(",\n");
- }
- out.append(" \"steps\": {\n");
- out.append(" \"step2_udd_debug_all_industries\": ").append(stepBlock(step2)).append(",\n");
- out.append(" \"step3_multi_turn\": ").append(stepBlock(step3)).append(",\n");
- out.append(" \"step4_50_variations\": ").append(stepBlock(step4)).append(",\n");
- out.append(" \"step5_continuous_followup\": ").append(stepBlock(step5)).append(",\n");
- out.append(" \"step6_quality_scoring\": ").append(stepBlock(step6)).append("\n");
- out.append(" },\n");
- out.append(" \"summary\": ").append(summaryJson(summary)).append(",\n");
- out.append(" \"finishedAt\": ").append(js(LocalDateTime.now().format(TS))).append("\n");
- out.append("}\n");
- Files.writeString(REPORT_FILE, out.toString(), StandardCharsets.UTF_8);
- System.out.println("报告已写入: " + REPORT_FILE);
- System.out.println("汇总: total=" + summary.get("totalCalls")
- + " passed=" + summary.get("passed")
- + " failed=" + summary.get("failed")
- + " timeout=" + summary.get("timeout"));
- }
- // ════════════════════════════════════════════════════════════════
- // 各步骤实现
- // ════════════════════════════════════════════════════════════════
- /** 第1步:登录(注入验证码后调用 /login) */
- private static String login() throws Exception {
- String uuid = "javaval" + System.currentTimeMillis();
- seedCaptcha(uuid);
- String body = "{\"username\":\"admin\",\"password\":\"Admin@123456\","
- + "\"tenantCode\":\"cs1\",\"code\":\"1234\",\"uuid\":\"" + uuid + "\"}";
- HttpResponse<String> resp = postJson(BASE + "/login", body, null, LOGIN_TIMEOUT);
- if (resp.statusCode() != 200) {
- throw new IOException("login HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200));
- }
- String tk = extractString(resp.body(), "token");
- if (tk == null || tk.isEmpty()) {
- throw new IOException("login no token: " + trunc(resp.body(), 200));
- }
- return tk;
- }
- /** 向 Redis 注入已知验证码(FastJSON 序列化形式:带双引号的字符串) */
- private static void seedCaptcha(String uuid) throws IOException {
- String key = "tenantid:system:captcha_codes:" + uuid;
- String value = "\"1234\""; // FastJSON toJSONString("1234", WriteClassName) => "1234"
- try (Socket sock = new Socket("127.0.0.1", 6379)) {
- sock.setSoTimeout(5000);
- OutputStream out = sock.getOutputStream();
- String cmd = "*5\r\n$3\r\nSET\r\n$" + key.length() + "\r\n" + key + "\r\n$"
- + value.length() + "\r\n" + value + "\r\n$2\r\nEX\r\n$3\r\n600\r\n";
- out.write(cmd.getBytes(StandardCharsets.US_ASCII));
- out.flush();
- byte[] buf = new byte[128];
- int n = sock.getInputStream().read(buf);
- if (n <= 0) throw new IOException("Redis no response");
- String resp = new String(buf, 0, n, StandardCharsets.US_ASCII);
- if (!resp.startsWith("+OK")) throw new IOException("Redis SET failed: " + resp.trim());
- }
- }
- /** UDD 深度对话调试(单轮) */
- private static Item uddDebug(String module, String industry, String industryLabel,
- String message, String userKey, boolean persistState) {
- Item it = new Item();
- it.module = module;
- it.industry = industry + "(" + industryLabel + ")";
- it.inputMessage = message;
- String body = "{\"message\":" + js(message) + ",\"industryType\":\"" + industry + "\""
- + ",\"userKey\":\"" + userKey + "\",\"persistState\":" + persistState + "}";
- long t0 = System.currentTimeMillis();
- try {
- HttpResponse<String> resp = postJson(VAL_BASE + "/udd/debug", body, null, SINGLE_TIMEOUT);
- it.elapsedMs = System.currentTimeMillis() - t0;
- if (resp.statusCode() != 200) {
- it.status = "ERROR";
- it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(), 200);
- it.passed = false;
- return it;
- }
- int code = extractInt(resp.body(), "code", -1);
- if (code != 200) {
- it.status = "ERROR";
- it.error = "code=" + code + " msg=" + extractString(resp.body(), "msg");
- it.passed = false;
- return it;
- }
- String data = extractDataObject(resp.body());
- it.aiReply = extractString(data, "reply");
- it.tags = tagReply(it.aiReply, message);
- it.passed = it.aiReply != null && !it.aiReply.isEmpty();
- it.status = it.passed ? "OK" : "ERROR";
- if (!it.passed) it.error = "空回复";
- return it;
- } catch (Exception e) {
- it.elapsedMs = System.currentTimeMillis() - t0;
- it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
- it.error = rootMsg(e);
- it.passed = false;
- return it;
- }
- }
- /** 多轮对话(服务端串行处理 5 轮) */
- private static Item multiTurn(String industry, String industryLabel) {
- Item it = new Item();
- it.module = "step3_多轮对话";
- it.industry = industry + "(" + industryLabel + ")";
- it.inputMessage = String.join(" | ", MULTI_TURN_MESSAGES);
- StringBuilder msgs = new StringBuilder("[");
- for (int i = 0; i < MULTI_TURN_MESSAGES.length; i++) {
- if (i > 0) msgs.append(",");
- msgs.append(js(MULTI_TURN_MESSAGES[i]));
- }
- msgs.append("]");
- String body = "{\"messages\":" + msgs + ",\"industryType\":\"" + industry + "\""
- + ",\"userKey\":\"mt2_" + industry + "\"}";
- long t0 = System.currentTimeMillis();
- try {
- HttpResponse<String> resp = postJson(VAL_BASE + "/udd/multi-turn", body, null, MULTI_TURN_TIMEOUT);
- it.elapsedMs = System.currentTimeMillis() - t0;
- if (resp.statusCode() != 200) {
- it.status = "ERROR"; it.error = "HTTP " + resp.statusCode(); it.passed = false; return it;
- }
- int code = extractInt(resp.body(), "code", -1);
- if (code != 200) { it.status="ERROR"; it.error="code="+code; it.passed=false; return it; }
- String data = extractDataObject(resp.body());
- int total = extractInt(data, "totalTurns", 0);
- List<String> replies = extractAllStrings(data, "reply");
- List<String> custMsgs = extractAllStrings(data, "customerMessage");
- StringBuilder rb = new StringBuilder();
- for (int i = 0; i < replies.size(); i++) {
- if (i > 0) rb.append(" || ");
- String cm = i < custMsgs.size() ? custMsgs.get(i) : "";
- rb.append("[").append(cm).append("]=>").append(trunc(replies.get(i), 80));
- }
- it.aiReply = rb.toString();
- it.passed = total == MULTI_TURN_MESSAGES.length && replies.size() == MULTI_TURN_MESSAGES.length;
- it.status = it.passed ? "OK" : "ERROR";
- if (!it.passed) it.error = "totalTurns=" + total + " replies=" + replies.size();
- // 给最后一轮打标签
- if (!replies.isEmpty()) it.tags = tagReply(replies.get(replies.size() - 1), MULTI_TURN_MESSAGES[0]);
- it.note = "5轮对话,每轮reply已提取";
- return it;
- } catch (Exception e) {
- it.elapsedMs = System.currentTimeMillis() - t0;
- it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
- it.error = rootMsg(e);
- it.passed = false;
- return it;
- }
- }
- /** 合规/质量评分检查(需 Token,content 作为 RequestParam) */
- private static Item complianceCheck(String type, String reply) {
- Item it = new Item();
- it.module = "step6_质量评分";
- it.industry = "travel";
- it.inputMessage = "质量类型=" + type + " | 回复=" + trunc(reply, 60);
- long t0 = System.currentTimeMillis();
- try {
- if (token == null) {
- it.status = "ERROR"; it.error = "无Token,登录失败"; it.passed = false; return it;
- }
- String url = COMPLIANCE_URL + "?content=" + URLEncoder.encode(reply, StandardCharsets.UTF_8);
- HttpResponse<String> resp = postJson(url, "", token, COMPLIANCE_TIMEOUT);
- it.elapsedMs = System.currentTimeMillis() - t0;
- if (resp.statusCode() != 200) {
- it.status = "ERROR"; it.error = "HTTP " + resp.statusCode() + " " + trunc(resp.body(),150);
- it.passed = false; return it;
- }
- int code = extractInt(resp.body(), "code", -1);
- if (code != 200) { it.status="ERROR"; it.error="code="+code+" "+extractString(resp.body(),"msg"); it.passed=false; return it; }
- String data = extractDataObject(resp.body());
- boolean compliant = extractBool(data, "compliant", true);
- int severity = extractInt(data, "severity", 0);
- String violated = extractString(data, "violatedRule");
- String matched = extractString(data, "matchedKeyword");
- it.aiReply = reply;
- it.qualityScore = "compliant=" + compliant + " severity=" + severity
- + (violated != null ? " violated=" + trunc(violated, 40) : "")
- + (matched != null ? " matched=" + trunc(matched, 30) : "");
- it.tags = tagReply(reply, "你好");
- it.tags.add(type);
- // 通过标准:合规(compliant=true) 且 severity<=1
- it.passed = compliant && severity <= 1;
- it.status = "OK";
- return it;
- } catch (Exception e) {
- it.elapsedMs = System.currentTimeMillis() - t0;
- it.status = isTimeout(e) ? "TIMEOUT" : "ERROR";
- it.error = rootMsg(e);
- it.passed = false;
- return it;
- }
- }
- // ════════════════════════════════════════════════════════════════
- // 并行执行辅助
- // ════════════════════════════════════════════════════════════════
- interface TaskSupplier { List<Callable<Item>> get() throws Exception; }
- private static List<Item> runParallel(String name, TaskSupplier supplier) throws Exception {
- List<Callable<Item>> tasks = supplier.get();
- ExecutorService pool = Executors.newFixedThreadPool(Math.min(CONCURRENCY, tasks.size()));
- List<Future<Item>> futures = new ArrayList<>();
- for (Callable<Item> t : tasks) futures.add(pool.submit(t));
- List<Item> results = new ArrayList<>();
- int done = 0;
- for (Future<Item> f : futures) {
- try {
- Item it = f.get(SINGLE_TIMEOUT + 30, TimeUnit.SECONDS);
- results.add(it);
- } catch (TimeoutException te) {
- Item it = new Item(); it.module = name; it.status = "TIMEOUT"; it.passed = false; it.error = "future timeout";
- results.add(it);
- } catch (Exception e) {
- Item it = new Item(); it.module = name; it.status = "ERROR"; it.passed = false; it.error = rootMsg(e);
- results.add(it);
- }
- done++;
- System.out.println(" [" + name + "] " + done + "/" + tasks.size() + " done");
- }
- pool.shutdown();
- return results;
- }
- // ════════════════════════════════════════════════════════════════
- // HTTP 工具
- // ════════════════════════════════════════════════════════════════
- private static HttpResponse<String> postJson(String url, String body, String auth, int timeoutSec) throws Exception {
- HttpRequest.Builder b = HttpRequest.newBuilder()
- .uri(URI.create(url))
- .timeout(Duration.ofSeconds(timeoutSec))
- .header("Content-Type", "application/json; charset=utf-8")
- .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8));
- if (auth != null) b.header("Authorization", auth);
- return httpClient.send(b.build(), HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
- }
- private static void delete(String url) throws Exception {
- HttpRequest req = HttpRequest.newBuilder()
- .uri(URI.create(url))
- .timeout(Duration.ofSeconds(20))
- .DELETE()
- .build();
- httpClient.send(req, HttpResponse.BodyHandlers.discarding());
- }
- // ════════════════════════════════════════════════════════════════
- // JSON 解析(无依赖,索引/状态机)
- // ════════════════════════════════════════════════════════════════
- private static String extractString(String json, String field) {
- if (json == null) return null;
- String key = "\"" + field + "\"";
- int i = json.indexOf(key);
- if (i < 0) return null;
- int colon = json.indexOf(':', i + key.length());
- if (colon < 0) return null;
- int q1 = json.indexOf('"', colon + 1);
- if (q1 < 0) return null;
- StringBuilder sb = new StringBuilder();
- int j = q1 + 1;
- while (j < json.length()) {
- char c = json.charAt(j);
- if (c == '\\' && j + 1 < json.length()) {
- char n = json.charAt(j + 1);
- if (n == 'u' && j + 6 <= json.length()) {
- try { sb.append((char) Integer.parseInt(json.substring(j + 2, j + 6), 16)); } catch (Exception ex) { sb.append(n); }
- j += 6;
- } else {
- switch (n) {
- case '"': sb.append('"'); break;
- case '\\': sb.append('\\'); break;
- case 'n': sb.append('\n'); break;
- case 'r': sb.append('\r'); break;
- case 't': sb.append('\t'); break;
- case '/': sb.append('/'); break;
- default: sb.append(n); break;
- }
- j += 2;
- }
- } else if (c == '"') {
- break;
- } else {
- sb.append(c);
- j++;
- }
- }
- return sb.toString();
- }
- private static List<String> extractAllStrings(String json, String field) {
- List<String> list = new ArrayList<>();
- if (json == null) return list;
- String key = "\"" + field + "\"";
- int from = 0;
- while (true) {
- int i = json.indexOf(key, from);
- if (i < 0) break;
- int colon = json.indexOf(':', i + key.length());
- if (colon < 0) break;
- int q1 = json.indexOf('"', colon + 1);
- if (q1 < 0) break;
- StringBuilder sb = new StringBuilder();
- int j = q1 + 1;
- while (j < json.length()) {
- char c = json.charAt(j);
- if (c == '\\' && j + 1 < json.length()) {
- char n = json.charAt(j + 1);
- if (n == 'u' && j + 6 <= json.length()) {
- try { sb.append((char) Integer.parseInt(json.substring(j + 2, j + 6), 16)); } catch (Exception ex) { sb.append(n); }
- j += 6;
- } else {
- switch (n) {
- case '"': sb.append('"'); break;
- case '\\': sb.append('\\'); break;
- case 'n': sb.append('\n'); break;
- case 'r': sb.append('\r'); break;
- case 't': sb.append('\t'); break;
- default: sb.append(n); break;
- }
- j += 2;
- }
- } else if (c == '"') {
- break;
- } else {
- sb.append(c);
- j++;
- }
- }
- list.add(sb.toString());
- from = j + 1;
- }
- return list;
- }
- private static int extractInt(String json, String field, int def) {
- if (json == null) return def;
- String key = "\"" + field + "\"";
- int i = json.indexOf(key);
- if (i < 0) return def;
- int colon = json.indexOf(':', i + key.length());
- if (colon < 0) return def;
- int j = colon + 1;
- while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
- int start = j;
- while (j < json.length() && (Character.isDigit(json.charAt(j)) || json.charAt(j) == '-')) j++;
- if (start == j) return def;
- try { return Integer.parseInt(json.substring(start, j)); } catch (Exception e) { return def; }
- }
- private static boolean extractBool(String json, String field, boolean def) {
- if (json == null) return def;
- String key = "\"" + field + "\"";
- int i = json.indexOf(key);
- if (i < 0) return def;
- int colon = json.indexOf(':', i + key.length());
- if (colon < 0) return def;
- int j = colon + 1;
- while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
- if (json.startsWith("true", j)) return true;
- if (json.startsWith("false", j)) return false;
- return def;
- }
- /** 提取 "data":{...} 或 "data":[...] 的内容(按括号配平) */
- private static String extractDataObject(String json) {
- if (json == null) return "";
- String key = "\"data\"";
- int i = json.indexOf(key);
- if (i < 0) return "";
- int colon = json.indexOf(':', i + key.length());
- if (colon < 0) return "";
- int j = colon + 1;
- while (j < json.length() && Character.isWhitespace(json.charAt(j))) j++;
- if (j >= json.length()) return "";
- char open = json.charAt(j);
- if (open != '{' && open != '[') {
- // 标量值
- int start = j;
- while (j < json.length() && json.charAt(j) != ',' && json.charAt(j) != '}') j++;
- return json.substring(start, j);
- }
- char close = (open == '{') ? '}' : ']';
- int depth = 0;
- int start = j;
- boolean inStr = false;
- while (j < json.length()) {
- char c = json.charAt(j);
- if (c == '\\' && inStr) { j += 2; continue; }
- if (c == '"') inStr = !inStr;
- if (!inStr) {
- if (c == open) depth++;
- else if (c == close) { depth--; if (depth == 0) { return json.substring(start, j + 1); } }
- }
- j++;
- }
- return json.substring(start);
- }
- // ════════════════════════════════════════════════════════════════
- // 标签与汇总
- // ════════════════════════════════════════════════════════════════
- private static List<String> tagReply(String reply, String inputMessage) {
- List<String> tags = new ArrayList<>();
- if (reply == null || reply.isEmpty()) { tags.add("空回复"); return tags; }
- // 机械:调试预览/模板标记/重复
- if (reply.contains("[UDD调试预览]") || reply.contains("[待追问") || reply.contains("待追问:")
- || reply.contains("根据画像") || reply.contains("禁止暴露")) {
- tags.add("机械");
- }
- // 亲切
- if (reply.contains("您好") || reply.contains("很高兴") || reply.contains("请问")
- || reply.contains("帮您") || reply.contains("为您")) tags.add("亲切");
- // 口语化(语气词)
- int particle = 0;
- for (String p : new String[]{"嗯", "呢", "呀", "哈", "吧", "哦", "啦", "呐", "诶", "哟"}) {
- int idx = -1;
- while ((idx = reply.indexOf(p, idx + 1)) >= 0) particle++;
- }
- if (particle >= 1) tags.add("口语化");
- // 机械:无语气词且无亲切词且较短
- if (particle == 0 && !reply.contains("您好") && !reply.contains("请问") && reply.length() < 40) {
- tags.add("机械");
- }
- // 语境不符:回复与输入无共同汉字(粗略)
- if (inputMessage != null && !inputMessage.isEmpty()) {
- boolean share = false;
- for (char c : inputMessage.toCharArray()) {
- if (Character.UnicodeScript.of(c) == Character.UnicodeScript.HAN && reply.indexOf(c) >= 0) {
- share = true; break;
- }
- }
- // 放宽:调试预览通常含"客户"等通用词,仅当回复明显跑题时标记
- if (!share && reply.length() < 20 && !reply.contains("您好") && !reply.contains("请问")) {
- tags.add("语境不符");
- }
- }
- if (reply.contains("AI") || reply.contains("机器人") || reply.contains("智能客服")) tags.add("暴露AI");
- if (reply.contains("百分百") || reply.contains("包治") || reply.contains("保证") && reply.contains("治愈")) tags.add("过度承诺");
- return tags;
- }
- private static Map<String, Object> buildSummary(List<Item> all) {
- Map<String, Object> s = new LinkedHashMap<>();
- int total = all.size(), passed = 0, failed = 0, timeout = 0, error = 0;
- long totalMs = 0, maxMs = 0, minMs = Long.MAX_VALUE;
- Map<String, Integer> byStatus = new LinkedHashMap<>();
- Map<String, Integer> byTag = new LinkedHashMap<>();
- Map<String, Integer> byModule = new LinkedHashMap<>();
- Map<String, Integer> byModulePass = new LinkedHashMap<>();
- for (Item it : all) {
- totalMs += it.elapsedMs;
- if (it.elapsedMs > maxMs) maxMs = it.elapsedMs;
- if (it.elapsedMs < minMs) minMs = it.elapsedMs;
- byStatus.merge(it.status, 1, Integer::sum);
- if ("OK".equals(it.status) && it.passed) passed++;
- else if ("TIMEOUT".equals(it.status)) timeout++;
- else error++;
- if (!it.passed) failed++;
- for (String t : it.tags) byTag.merge(t, 1, Integer::sum);
- String mod = it.module == null ? "?" : it.module.replaceAll("_\\(.*", "").replaceAll("_第.*", "");
- byModule.merge(mod, 1, Integer::sum);
- if (it.passed) byModulePass.merge(mod, 1, Integer::sum);
- }
- s.put("totalCalls", total);
- s.put("passed", passed);
- s.put("failed", failed);
- s.put("timeout", timeout);
- s.put("error", error);
- s.put("avgMs", total == 0 ? 0 : totalMs / total);
- s.put("maxMs", maxMs);
- s.put("minMs", minMs == Long.MAX_VALUE ? 0 : minMs);
- s.put("byStatus", byStatus);
- s.put("byTag", byTag);
- s.put("byModule", byModule);
- s.put("byModulePass", byModulePass);
- return s;
- }
- private static String stepBlock(List<Item> items) {
- int passed = 0;
- long ms = 0;
- for (Item it : items) { if (it.passed) passed++; ms += it.elapsedMs; }
- StringBuilder sb = new StringBuilder();
- sb.append("{\"total\":").append(items.size()).append(",\"passed\":").append(passed)
- .append(",\"totalMs\":").append(ms).append(",\"items\":[");
- for (int i = 0; i < items.size(); i++) {
- if (i > 0) sb.append(",");
- sb.append(items.get(i).toJson());
- }
- sb.append("]}");
- return sb.toString();
- }
- private static String summaryJson(Map<String, Object> s) {
- StringBuilder sb = new StringBuilder("{");
- int c = 0;
- for (Map.Entry<String, Object> e : s.entrySet()) {
- if (c++ > 0) sb.append(",");
- sb.append(js(e.getKey())).append(":").append(jsv(e.getValue()));
- }
- sb.append("}");
- return sb.toString();
- }
- // ════════════════════════════════════════════════════════════════
- // JSON 序列化辅助
- // ════════════════════════════════════════════════════════════════
- private static String js(String s) {
- if (s == null) return "null";
- StringBuilder sb = new StringBuilder("\"");
- for (int i = 0; i < s.length(); i++) {
- char c = s.charAt(i);
- switch (c) {
- case '"': sb.append("\\\""); break;
- case '\\': sb.append("\\\\"); break;
- case '\n': sb.append("\\n"); break;
- case '\r': sb.append("\\r"); break;
- case '\t': sb.append("\\t"); break;
- default:
- if (c < 0x20) sb.append(String.format("\\u%04x", (int) c));
- else sb.append(c);
- }
- }
- sb.append("\"");
- return sb.toString();
- }
- private static String jsArray(List<String> list) {
- StringBuilder sb = new StringBuilder("[");
- for (int i = 0; i < list.size(); i++) {
- if (i > 0) sb.append(",");
- sb.append(js(list.get(i)));
- }
- sb.append("]");
- return sb.toString();
- }
- @SuppressWarnings("unchecked")
- private static String jsv(Object v) {
- if (v == null) return "null";
- if (v instanceof Number) return v.toString();
- if (v instanceof Boolean) return v.toString();
- if (v instanceof String) return js((String) v);
- if (v instanceof Map) {
- StringBuilder sb = new StringBuilder("{");
- int c = 0;
- for (Map.Entry<String, Object> e : ((Map<String, Object>) v).entrySet()) {
- if (c++ > 0) sb.append(",");
- sb.append(js(e.getKey())).append(":").append(jsv(e.getValue()));
- }
- sb.append("}");
- return sb.toString();
- }
- if (v instanceof Iterable) {
- StringBuilder sb = new StringBuilder("[");
- int c = 0;
- for (Object o : (Iterable<?>) v) {
- if (c++ > 0) sb.append(",");
- sb.append(jsv(o));
- }
- sb.append("]");
- return sb.toString();
- }
- return js(v.toString());
- }
- // ════════════════════════════════════════════════════════════════
- // 通用辅助
- // ════════════════════════════════════════════════════════════════
- private static String trunc(String s, int max) {
- if (s == null) return null;
- return s.length() <= max ? s : s.substring(0, max) + "...";
- }
- private static boolean isTimeout(Throwable e) {
- Throwable c = e;
- while (c != null) {
- String n = c.getClass().getName();
- if (c instanceof java.net.http.HttpTimeoutException) return true;
- if (c instanceof TimeoutException) return true;
- if (n.contains("TimeoutException")) return true;
- c = c.getCause();
- }
- return false;
- }
- private static String rootMsg(Throwable e) {
- Throwable c = e;
- while (c.getCause() != null && c.getCause() != c) c = c.getCause();
- String msg = c.getMessage();
- return (msg != null ? msg : c.getClass().getSimpleName()).replace("\n", " ");
- }
- }
|