ApplyIndustryCorpusReseed.java 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. import com.fs.company.service.workflow.config.LobsterIndustryCatalog;
  2. import com.fs.company.service.workflow.corpus.LobsterIndustryCorpusSeedCatalog;
  3. import java.sql.Connection;
  4. import java.sql.DriverManager;
  5. import java.sql.PreparedStatement;
  6. import java.sql.ResultSet;
  7. import java.sql.SQLException;
  8. import java.sql.Statement;
  9. import java.util.ArrayList;
  10. import java.util.List;
  11. /**
  12. * Re-insert industry matrix seed corpus after garbled cleanup.
  13. * Requires fs-service compiled classes on classpath.
  14. * Usage: java -cp mysql.jar;../fs-service/target/classes;. ApplyIndustryCorpusReseed [tenantDbName]
  15. */
  16. public class ApplyIndustryCorpusReseed {
  17. private static final String MASTER_HOST = getenv("DB_HOST", "cq-cdb-8fjmemkb.sql.tencentcdb.com");
  18. private static final int MASTER_PORT = Integer.parseInt(getenv("DB_PORT", "27220"));
  19. private static final String MASTER_USER = getenv("DB_USER", "root");
  20. private static final String MASTER_PASSWORD = getenv("DB_PASSWORD", "Ylrz_1q2w3e4r5t6y");
  21. private static final String MASTER_DB = getenv("DB_NAME", "ylrz_saas");
  22. public static void main(String[] args) throws Exception {
  23. Class.forName("com.mysql.cj.jdbc.Driver");
  24. try (Connection master = open(MASTER_HOST, MASTER_PORT, MASTER_DB, MASTER_USER, MASTER_PASSWORD)) {
  25. List<Long> tenantIds = loadActiveTenantIds(master);
  26. if (args.length > 0) {
  27. tenantIds.clear();
  28. for (String db : args) {
  29. reseedTenantDb(master, db);
  30. }
  31. return;
  32. }
  33. for (Long tenantId : tenantIds) {
  34. TenantDb tenant = loadTenant(master, tenantId);
  35. if (tenant == null) {
  36. continue;
  37. }
  38. reseedConn(master, tenantId, tenant);
  39. }
  40. }
  41. }
  42. private static void reseedTenantDb(Connection master, String dbName) throws Exception {
  43. try (Statement st = master.createStatement();
  44. ResultSet rs = st.executeQuery(
  45. "SELECT id FROM tenant_info WHERE status=1 AND db_url LIKE '%/" + dbName + "%' LIMIT 1")) {
  46. if (!rs.next()) {
  47. System.out.println("[SKIP] no tenant for db " + dbName);
  48. return;
  49. }
  50. long tenantId = rs.getLong(1);
  51. TenantDb tenant = loadTenant(master, tenantId);
  52. if (tenant != null) {
  53. reseedConn(master, tenantId, tenant);
  54. }
  55. }
  56. }
  57. private static void reseedConn(Connection master, long tenantId, TenantDb tenant) throws Exception {
  58. System.out.println("\n[TENANT " + tenantId + "] " + tenant.database);
  59. try (Connection conn = open(tenant.host, tenant.port, tenant.database, tenant.user, tenant.password)) {
  60. if (!columnExists(conn, "lobster_sales_corpus", "customer_question")) {
  61. System.out.println(" [SKIP] lobster_sales_corpus schema not ready");
  62. return;
  63. }
  64. Long companyId = loadPrimaryCompanyId(conn, tenantId, master);
  65. if (companyId == null) {
  66. System.out.println(" [SKIP] no company_id");
  67. return;
  68. }
  69. int total = 0;
  70. if (!columnExists(conn, "lobster_sales_corpus", "industry_type")) {
  71. deleteAllIndustrySeeds(conn, companyId);
  72. }
  73. for (LobsterIndustryCatalog.IndustryPack pack : LobsterIndustryCatalog.allPacks()) {
  74. String code = pack.code;
  75. String label = LobsterIndustryCatalog.labelZh(code);
  76. if (columnExists(conn, "lobster_sales_corpus", "industry_type")) {
  77. deleteExistingSeeds(conn, companyId, code);
  78. }
  79. List<LobsterIndustryCorpusSeedCatalog.SeedEntry> seeds =
  80. LobsterIndustryCorpusSeedCatalog.entriesFor(code, label);
  81. for (LobsterIndustryCorpusSeedCatalog.SeedEntry seed : seeds) {
  82. insertSeed(conn, companyId, code, seed);
  83. total++;
  84. }
  85. System.out.println(" industry=" + code + " seeds=" + seeds.size());
  86. }
  87. System.out.println(" inserted_total=" + total);
  88. }
  89. }
  90. private static void deleteExistingSeeds(Connection conn, long companyId, String industryCode) throws SQLException {
  91. if (columnExists(conn, "lobster_sales_corpus", "industry_type")) {
  92. String sql = "DELETE FROM lobster_sales_corpus WHERE company_id=? AND industry_type=? "
  93. + "AND source_type='industry_seed'";
  94. try (PreparedStatement ps = conn.prepareStatement(sql)) {
  95. ps.setLong(1, companyId);
  96. ps.setString(2, industryCode);
  97. ps.executeUpdate();
  98. }
  99. return;
  100. }
  101. String sql = "DELETE FROM lobster_sales_corpus WHERE company_id=? AND source_type='industry_seed' "
  102. + "AND scenario=?";
  103. try (PreparedStatement ps = conn.prepareStatement(sql)) {
  104. ps.setLong(1, companyId);
  105. ps.setString(2, industryCode);
  106. ps.executeUpdate();
  107. }
  108. }
  109. private static void deleteAllIndustrySeeds(Connection conn, long companyId) throws SQLException {
  110. String sql = "DELETE FROM lobster_sales_corpus WHERE company_id=? AND source_type='industry_seed'";
  111. try (PreparedStatement ps = conn.prepareStatement(sql)) {
  112. ps.setLong(1, companyId);
  113. ps.executeUpdate();
  114. }
  115. }
  116. private static void insertSeed(Connection conn, long companyId, String industryCode,
  117. LobsterIndustryCorpusSeedCatalog.SeedEntry seed) throws SQLException {
  118. String dialog = "\u5ba2\u6237: " + seed.customerQuestion + "\n\u9500\u552e: " + seed.salesAnswer;
  119. boolean hasIndustryType = columnExists(conn, "lobster_sales_corpus", "industry_type");
  120. boolean hasSalesperson = columnExists(conn, "lobster_sales_corpus", "salesperson_name");
  121. StringBuilder cols = new StringBuilder("company_id");
  122. StringBuilder vals = new StringBuilder("?");
  123. List<Object> params = new ArrayList<>();
  124. params.add(companyId);
  125. if (hasIndustryType) {
  126. cols.append(", industry_type");
  127. vals.append(", ?");
  128. params.add(industryCode);
  129. }
  130. if (hasSalesperson) {
  131. cols.append(", salesperson_name");
  132. vals.append(", ?");
  133. params.add("\u884c\u4e1a\u8bed\u6599");
  134. }
  135. cols.append(", customer_question, sales_answer, dialog_text, scenario, source_type");
  136. vals.append(", ?, ?, ?, ?, ?");
  137. params.add(seed.customerQuestion);
  138. params.add(seed.salesAnswer);
  139. params.add(dialog);
  140. params.add(seed.scenario);
  141. params.add("industry_seed");
  142. if (columnExists(conn, "lobster_sales_corpus", "status")) {
  143. cols.append(", status, quality_status");
  144. vals.append(", ?, ?");
  145. params.add("analyzed");
  146. params.add("qualified");
  147. }
  148. if (columnExists(conn, "lobster_sales_corpus", "score")) {
  149. cols.append(", score");
  150. vals.append(", ?");
  151. params.add(new java.math.BigDecimal("85"));
  152. }
  153. if (columnExists(conn, "lobster_sales_corpus", "kb_written")) {
  154. cols.append(", kb_written");
  155. vals.append(", ?");
  156. params.add(0);
  157. }
  158. if (columnExists(conn, "lobster_sales_corpus", "enabled")) {
  159. cols.append(", enabled");
  160. vals.append(", ?");
  161. params.add(1);
  162. }
  163. cols.append(", create_by, create_time");
  164. vals.append(", ?, NOW()");
  165. params.add("system");
  166. String sql = "INSERT INTO lobster_sales_corpus (" + cols + ") VALUES (" + vals + ")";
  167. try (PreparedStatement ps = conn.prepareStatement(sql)) {
  168. for (int i = 0; i < params.size(); i++) {
  169. ps.setObject(i + 1, params.get(i));
  170. }
  171. ps.executeUpdate();
  172. }
  173. }
  174. private static Long loadPrimaryCompanyId(Connection conn, long tenantId, Connection master) throws SQLException {
  175. try (PreparedStatement ps = master.prepareStatement(
  176. "SELECT company_id FROM tenant_info WHERE id=? AND status=1 LIMIT 1")) {
  177. ps.setLong(1, tenantId);
  178. try (ResultSet rs = ps.executeQuery()) {
  179. if (rs.next() && rs.getObject("company_id") != null) {
  180. return rs.getLong("company_id");
  181. }
  182. }
  183. } catch (SQLException ignored) {
  184. }
  185. if (tableExists(conn, "lobster_sales_corpus")) {
  186. try (Statement st = conn.createStatement();
  187. ResultSet rs = st.executeQuery(
  188. "SELECT company_id FROM lobster_sales_corpus WHERE company_id IS NOT NULL "
  189. + "ORDER BY id DESC LIMIT 1")) {
  190. if (rs.next()) {
  191. return rs.getLong(1);
  192. }
  193. }
  194. }
  195. String[] tables = {"company_info", "sys_company"};
  196. String[] cols = {"company_id", "id"};
  197. for (int i = 0; i < tables.length; i++) {
  198. if (!tableExists(conn, tables[i])) {
  199. continue;
  200. }
  201. try (Statement st = conn.createStatement();
  202. ResultSet rs = st.executeQuery(
  203. "SELECT " + cols[i] + " FROM " + tables[i] + " ORDER BY " + cols[i] + " LIMIT 1")) {
  204. if (rs.next()) {
  205. return rs.getLong(1);
  206. }
  207. } catch (SQLException ignored) {
  208. }
  209. }
  210. return null;
  211. }
  212. private static boolean columnExists(Connection conn, String table, String column) throws SQLException {
  213. try (ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), null, table, column)) {
  214. return rs.next();
  215. }
  216. }
  217. private static boolean tableExists(Connection conn, String table) throws SQLException {
  218. try (ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), null, table, null)) {
  219. return rs.next();
  220. }
  221. }
  222. private static List<Long> loadActiveTenantIds(Connection master) throws SQLException {
  223. List<Long> ids = new ArrayList<>();
  224. try (Statement st = master.createStatement();
  225. ResultSet rs = st.executeQuery("SELECT id FROM tenant_info WHERE status = 1 ORDER BY id")) {
  226. while (rs.next()) {
  227. ids.add(rs.getLong(1));
  228. }
  229. }
  230. return ids;
  231. }
  232. private static TenantDb loadTenant(Connection master, long tenantId) throws SQLException {
  233. try (PreparedStatement ps = master.prepareStatement(
  234. "SELECT db_url, db_account, db_pwd FROM tenant_info WHERE id = ? AND status = 1 LIMIT 1")) {
  235. ps.setLong(1, tenantId);
  236. try (ResultSet rs = ps.executeQuery()) {
  237. if (!rs.next()) {
  238. return null;
  239. }
  240. TenantDb t = new TenantDb();
  241. String url = rs.getString("db_url");
  242. String body = url.substring("jdbc:mysql://".length());
  243. int slash = body.indexOf('/');
  244. String hostPort = body.substring(0, slash);
  245. String db = body.substring(slash + 1).replaceAll("\\?.*", "");
  246. int colon = hostPort.indexOf(':');
  247. t.host = colon >= 0 ? hostPort.substring(0, colon) : hostPort;
  248. t.port = colon >= 0 ? Integer.parseInt(hostPort.substring(colon + 1)) : 3306;
  249. t.database = db;
  250. t.user = rs.getString("db_account");
  251. t.password = rs.getString("db_pwd");
  252. return t;
  253. }
  254. }
  255. }
  256. private static Connection open(String host, int port, String db, String user, String password)
  257. throws SQLException {
  258. return DriverManager.getConnection(
  259. "jdbc:mysql://" + host + ":" + port + "/" + db
  260. + "?useUnicode=true&characterEncoding=utf8&serverTimezone=GMT%2B8",
  261. user, password);
  262. }
  263. private static String getenv(String key, String def) {
  264. String v = System.getenv(key);
  265. return v != null && !v.isBlank() ? v : def;
  266. }
  267. private static final class TenantDb {
  268. String host;
  269. int port;
  270. String database;
  271. String user;
  272. String password;
  273. }
  274. }