package com.fs.fssync.listener;

import com.fs.fssync.config.FlinkConfig;
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.PipelineOptionsInternal;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.graph.StreamGraph;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;

import javax.annotation.PreDestroy;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

/**
 * MySQL事件监听
 *
 * @author jokey
 * @since 2024-3-14
 */
@Slf4j
@Component
public class MySqlEventListener implements CommandLineRunner {

    @Autowired
    private FlinkConfig flinkConfig;

    private static final String JOB_ID = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d";
    private static final String CHECKPOINT_DIR = "file:///home/software/fs-sync/data/flink/checkpoints";
    // 文件操作用的路径（不含协议前缀）
    private static final String CHECKPOINT_LOCAL_PATH = "/home/software/fs-sync/data/flink/checkpoints";

    @Override
    public void run(String... args) throws Exception {
        // 查找最新的有效检查点
        String latestCheckpoint = findLatestCheckpoint();
        boolean hasValidCheckpoint = latestCheckpoint != null;

        log.info("检查点查找结果: {}", hasValidCheckpoint ? "找到有效检查点" : "未找到有效检查点");

        // 创建环境并配置
        final StreamExecutionEnvironment env = configureEnvironment(latestCheckpoint);

        FlinkConfig.MySqlConfig mysqlConfig = flinkConfig.getCdc().getMysql();

        // Debezium 配置
        Properties debeziumProps = createDebeziumProperties();

        // 创建 MySQL CDC 源 - 根据是否有检查点决定启动模式
        MySqlSource<String> mySqlSource = createMySqlSource(mysqlConfig, debeziumProps, hasValidCheckpoint);

        // 创建数据流
        DataStreamSource<String> streamSource = env.fromSource(
                mySqlSource,
                WatermarkStrategy.noWatermarks(),
                "MySQL Source"
        ).setParallelism(flinkConfig.getParallelism().getSource());

        // 添加自定义接收器
        CustomSink customSink = new CustomSink();
        streamSource.addSink(customSink)
                .name("syncToRedis")
                .setParallelism(flinkConfig.getParallelism().getSink());

        // 修改执行方法，捕获更多信息
        StreamGraph streamGraph = env.getStreamGraph();
        streamGraph.setJobName("MySQL-CDC-Sync-Job");

        // 执行前记录更多信息
        log.info("准备执行CDC任务，作业名称: {}, 作业ID: {}, 检查点目录: {}, 启动模式: {}",
                streamGraph.getJobName(), JOB_ID, CHECKPOINT_DIR,
                hasValidCheckpoint ? "从检查点恢复" : mysqlConfig.getStartupOptions());

        // 执行作业
        env.execute(streamGraph);
    }

    private Properties createDebeziumProperties() {
        Map<String, String> configMap = new HashMap<>();
        configMap.put("decimal.handling.mode", "string");
        configMap.put("bigint.unsigned.handling.mode", "long");
        configMap.put("time.precision.mode", "adaptive_time_microseconds");
        configMap.put("datetime.handling.mode", "string");
        configMap.put("binary.handling.mode", "bytes");
        configMap.put("json.handling.mode", "string");
        configMap.put("geometry.handling.mode", "string");
        configMap.put("include.unknown.datatypes", "false");

        Properties props = new Properties();
        props.putAll(configMap);
        return props;
    }

    private MySqlSource<String> createMySqlSource(
            FlinkConfig.MySqlConfig mysqlConfig,
            Properties debeziumProps,
            boolean hasValidCheckpoint) {

        // 如果有有效检查点，使用LATEST模式避免重复处理数据
        // 注意：当使用检查点恢复时，Flink会自动从上次的位置继续，
        // 此时应使用LATEST启动选项以避免再次读取已处理的数据
        StartupOptions startupOptions = hasValidCheckpoint
            ? StartupOptions.latest()
            : getStartupOptions(mysqlConfig.getStartupOptions());

        log.info("MySqlSource 启动模式: {}", hasValidCheckpoint ? "LATEST (从检查点恢复)" : startupOptions);

        return MySqlSource.<String>builder()
                .debeziumProperties(debeziumProps)
                .hostname(mysqlConfig.getHostname())
                .port(mysqlConfig.getPort())
                .databaseList(mysqlConfig.getDatabaseList())
                .tableList(mysqlConfig.getTableList())
                .username(mysqlConfig.getUsername())
                .password(mysqlConfig.getPassword())
                .deserializer(new JsonDebeziumDeserializationSchema())
                .startupOptions(startupOptions)
                .serverTimeZone(mysqlConfig.getServerTimeZone())
                .build();
    }

    private StreamExecutionEnvironment configureEnvironment(String latestCheckpoint) {
        Configuration configuration = new Configuration();
        configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, JOB_ID);

        // 如果找到有效检查点，设置恢复路径
        if (latestCheckpoint != null) {
            configuration.setString("execution.savepoint.path", latestCheckpoint);
            log.info("将从检查点恢复: {}", latestCheckpoint);
        }

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration);

        // 设置状态后端
        try {
            // 使用HashMap状态后端 (内存中计算，但检查点存储在文件系统)
            env.setStateBackend(new HashMapStateBackend());
            env.getCheckpointConfig().setCheckpointStorage(new FileSystemCheckpointStorage(CHECKPOINT_DIR));
            log.info("设置状态后端为: {}", CHECKPOINT_DIR);
        } catch (Exception e) {
            log.error("设置状态后端失败", e);
        }

        // 检查点配置
        env.enableCheckpointing(flinkConfig.getCheckpoint().getInterval());
        CheckpointConfig checkpointConfig = env.getCheckpointConfig();
        checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        checkpointConfig.setMinPauseBetweenCheckpoints(500);
        checkpointConfig.setCheckpointTimeout(60000);
        checkpointConfig.setMaxConcurrentCheckpoints(1);
        checkpointConfig.setTolerableCheckpointFailureNumber(3);
        checkpointConfig.setExternalizedCheckpointCleanup(
                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        checkpointConfig.enableUnalignedCheckpoints();

        // 设置重启策略
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.seconds(10)));

        log.info("Flink环境配置完成，检查点间隔: {}ms, 作业ID: {}",
                flinkConfig.getCheckpoint().getInterval(), JOB_ID);

        return env;
    }

    private String findLatestCheckpoint() {
        try {
            // 使用Java NIO API进行更可靠的文件操作
            Path checkpointsRootDir = Paths.get(CHECKPOINT_LOCAL_PATH);
            if (!Files.exists(checkpointsRootDir) || !Files.isDirectory(checkpointsRootDir)) {
                log.warn("检查点根目录不存在: {}", CHECKPOINT_LOCAL_PATH);
                return null;
            }
            // 查找Job ID目录
            Path jobDir = checkpointsRootDir.resolve(JOB_ID);
            if (!Files.exists(jobDir) || !Files.isDirectory(jobDir)) {
                log.warn("作业检查点目录不存在: {}", jobDir);
                return null;
            }
            // 在作业目录下查找所有chk-开头的检查点目录
            List<Path> checkpoints = Files.list(jobDir)
                    .filter(p -> Files.isDirectory(p) && p.getFileName().toString().startsWith("chk-"))
                    .sorted((p1, p2) -> {
                        try {
                            // 提取检查点ID进行比较
                            int id1 = Integer.parseInt(p1.getFileName().toString().substring(4));
                            int id2 = Integer.parseInt(p2.getFileName().toString().substring(4));
                            // 降序排列，最新的检查点优先
                            return Integer.compare(id2, id1);
                        } catch (Exception e) {
                            log.warn("比较检查点目录时出错: {}", e.getMessage());
                            return 0;
                        }
                    })
                    .collect(Collectors.toList());
            if (checkpoints.isEmpty()) {
                log.warn("在作业目录下未找到检查点: {}", jobDir);
                return null;
            }
            // 遍历检查点目录，查找有效的检查点
            for (Path checkpoint : checkpoints) {
                // 检查_metadata文件是否存在
                Path metadata = checkpoint.resolve("_metadata");
                if (Files.exists(metadata) && Files.isRegularFile(metadata)) {
                    // 构建Flink可识别的检查点路径（注意检查点是相对于Job ID目录的）
                    String checkpointPath = "file://" + checkpoint.toString();
                    log.info("找到有效检查点: {}", checkpointPath);
                    // 记录详细信息用于调试
                    log.info("检查点ID: {}, 检查点目录: {}, 元数据文件: {}",
                            checkpoint.getFileName().toString().substring(4),
                            checkpoint,
                            metadata);
                    return checkpointPath;
                }
            }
            log.warn("未找到有效的检查点");
            return null;
        } catch (Exception e) {
            log.error("查找检查点时出错", e);
            return null;
        }
    }
    // 从检查点目录名中提取检查点ID
    private long extractCheckpointId(String directoryName) {
        try {
            // 针对不同格式的检查点目录名进行处理
            if (directoryName.startsWith("chk-")) {
                return Long.parseLong(directoryName.substring(4));
            } else {
                // 尝试直接解析整个名称
                return Long.parseLong(directoryName);
            }
        } catch (NumberFormatException e) {
            log.warn("无法从目录名提取检查点ID: {}", directoryName);
            return 0;
        }
    }

    private StartupOptions getStartupOptions(String option) {
        log.info("将使用配置的启动模式: {}", option);
        if (option == null) {
            return StartupOptions.latest();
        }
        switch (option.toLowerCase()) {
            case "initial": return StartupOptions.initial();
            case "latest": return StartupOptions.latest();
            case "earliest": return StartupOptions.earliest();
            default: return StartupOptions.latest();
        }
    }
}
