package com.spider.quartz.task;

import com.microsoft.playwright.*;
import com.microsoft.playwright.options.LoadState;
import com.microsoft.playwright.options.WaitUntilState;
import com.spider.quartz.config.BaiduAiTranslationService;
import com.spider.quartz.domain.JapanUnqualifiedGoods;
import com.spider.quartz.domain.JapanUnqualifiedGoodsZh;
import com.spider.quartz.mapper.JapanUnqualifiedGoodsMapper;
import com.spider.quartz.mapper.JapanUnqualifiedGoodsZhMapper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * 日本不合格商品爬取任务 - 优化版（Java 8兼容）
 * 网站：https://www.recall.caa.go.jp/
 */
@Component
public class JapanTask {

    @Autowired
    private JapanUnqualifiedGoodsMapper japanUnqualifiedGoodsMapper;

    @Autowired
    private JapanUnqualifiedGoodsZhMapper japanUnqualifiedGoodsZhMapper;

    @Autowired
    private BaiduAiTranslationService baiduTranslationService; // 注入翻译服务

    // ===================== 配置常量 =====================
    private static final String BASE_URL = "https://www.recall.caa.go.jp";
    private static final String RECALL_LIST_URL = "https://www.recall.caa.go.jp/result/index.php?screenkbn=03";
    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";

    // 日期格式化器
    private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    private final SimpleDateFormat recallDateFormat = new SimpleDateFormat("yyyy-MM-dd");
    private final SimpleDateFormat japanDateFormat = new SimpleDateFormat("yyyy年MM月dd日");
    private final SimpleDateFormat japanDateFormat2 = new SimpleDateFormat("yyyy/MM/dd");

    // 网络配置
    private static final int DETAIL_PAGE_TIMEOUT = 30000;
    private static final int MAX_RETRIES = 3;
    private static final int MAX_PAGES = 100;
    private static final int ITEMS_PER_PAGE = 15;

    // 添加以下两个字段到类中（放在其他字段后面）
    private List<JapanUnqualifiedGoods> pendingGoodsList = new ArrayList<>(); // 待处理的数据
    private List<JapanUnqualifiedGoods> failedGoodsList = new ArrayList<>();  // 失败的数据
    // 统计信息
    private int recallCount = 0;
    private int totalPagesProcessed = 0;
    private int successCount = 0;
    private int failureCount = 0;
    private long startTime = 0;

    // 浏览器实例管理
    private Playwright playwright;
    private Browser browser;

    /**
     * 定时任务调用的方法
     */
    public void startCrawlBySchedule() {
        try {
            log("========== 定时任务开始执行日本不合格商品数据爬取 ==========");
            log("执行时间: " + dateFormat.format(new Date()));

            startCrawl();

            log("========== 定时任务执行完成 ==========");
        } catch (Exception e) {
            error("定时任务执行异常", e);
        }
    }

    /**
     * 启动爬取任务
     */
    public void startCrawl() {
        startTime = System.currentTimeMillis();
        CrawlConditions conditions = null;

        try {
            // 1. 获取爬取条件
            conditions = getCrawlConditions();
            log("开始爬取，最小召回日期: " + recallDateFormat.format(conditions.minRecallDate));
            log("数据模式: " + (conditions.hasExistingData ? "增量爬取" : "全量爬取"));
            log("目标年份: " + conditions.targetYear);

            // 2. 初始化Playwright和浏览器
            initPlaywright();

            // 3. 爬取所有页面的召回列表
            crawlAllRecallPages(conditions);

            // 4. 批量保存成功记录到数据库
            batchSavePendingGoods();

            // 5. 批量保存失败记录到数据库
            batchSaveFailedGoods();

            // 6. 重试失败记录
            retryFailedGoods();

            // 4. 输出统计信息
            printStatistics();

        } catch (Exception e) {
            error("爬取过程异常", e);
        } finally {
            cleanupResources();
            // 清理集合
            pendingGoodsList.clear();
            failedGoodsList.clear();
        }
    }

    /**
     * 获取爬取条件
     */
    private CrawlConditions getCrawlConditions() {
        try {
            Date latestRecallTime = japanUnqualifiedGoodsMapper.selectLatestRecallTime();
            Date minRecallDate;
            boolean hasExistingData;

            if (latestRecallTime != null) {
                minRecallDate = latestRecallTime;
                hasExistingData = true;
                log("使用最新召回时间: " + dateFormat.format(latestRecallTime));
            } else {
                minRecallDate = getCurrentYearStartDate();
                hasExistingData = false;
                log("数据库中无数据，从当年开始爬取: " + recallDateFormat.format(minRecallDate));
            }

            Calendar cal = Calendar.getInstance();
            cal.setTime(minRecallDate);
            int targetYear = cal.get(Calendar.YEAR);

            return new CrawlConditions(minRecallDate, hasExistingData, targetYear);

        } catch (Exception e) {
            error("获取爬取条件失败", e);
            Date minRecallDate = getCurrentYearStartDate();
            return new CrawlConditions(minRecallDate, false, Calendar.getInstance().get(Calendar.YEAR));
        }
    }

    /**
     * 爬取所有分页的召回列表
     */
    private void crawlAllRecallPages(CrawlConditions conditions) {
        BrowserContext mainContext = null;
        Page mainPage = null;

        try {
            mainContext = browser.newContext();
            mainPage = mainContext.newPage();

            configureBrowser(mainPage);

            // 导航到列表页
            if (!navigateToListPage(mainPage)) {
                return;
            }

            int currentPageNum = 1;
            boolean shouldContinue = true;

            while (shouldContinue && !Thread.currentThread().isInterrupted() && currentPageNum <= MAX_PAGES) {
                log("\n=== 正在处理第 " + currentPageNum + " 页 ===");

                // 获取当前页面内容
                Document listDoc = getPageDocument(mainPage);
                if (listDoc == null) {
                    break;
                }

                // 提取召回项目
                Elements recallItems = extractRecallItems(listDoc);
                if (recallItems.isEmpty()) {
                    log("第 " + currentPageNum + " 页未找到召回记录");
                    if (currentPageNum == 1) {
                        log("第一页无记录，网站可能没有数据");
                    }
                    break;
                }

                log("第 " + currentPageNum + " 页共找到 " + recallItems.size() + " 条记录");

                // 处理当前页面的所有召回记录
                PageProcessResult result = processPageRecallItems(recallItems, conditions, recallCount);

                totalPagesProcessed++;
                recallCount += recallItems.size();

                log("第 " + currentPageNum + " 页处理完成，成功: " + successCount + " 条，失败: " + failureCount + " 条");

                // 检查是否应该停止爬取
                if (result.shouldStopCrawling) {
                    shouldContinue = false;
                    log("满足停止条件，爬取结束: " + result.stopReason);
                    break;
                }

                // 检查并翻页
                if (!hasMorePages(mainPage)) {
                    log("没有更多页面，爬取结束");
                    break;
                }

                if (!navigateToNextPage(mainPage, currentPageNum)) {
                    log("无法翻到下一页，爬取结束");
                    break;
                }

                currentPageNum++;
            }

            log("\n最终处理页数: " + (currentPageNum - 1));

        } catch (Exception e) {
            error("主页面处理异常", e);
        } finally {
            closeResources(mainPage, mainContext);
        }
    }

    /**
     * 导航到列表页
     */
    private boolean navigateToListPage(Page page) {
        try {
            log("正在导航到列表页: " + RECALL_LIST_URL);
            page.navigate(RECALL_LIST_URL, new Page.NavigateOptions()
                    .setTimeout(120000)
                    .setWaitUntil(WaitUntilState.NETWORKIDLE));

            waitForPageStable(page);

            // 验证页面是否正常加载
            return verifyPageContent(page);

        } catch (Exception e) {
            error("导航到列表页失败", e);
            return false;
        }
    }

    /**
     * 翻页方法 - 优化版（第二页特殊处理，其他页面通用处理）
     */
    private boolean navigateToNextPage(Page page, int currentPageNum) {
        try {
            log("尝试翻到第 " + (currentPageNum + 1) + " 页");

            // 特别处理第二页（现有逻辑，不需要修改）
            if (currentPageNum == 1) {
                log("特别注意：正在翻到第二页，将采用特殊处理策略");
                return navigateToSecondPageWithRefresh(page);
            }

            // 从第二页开始往后翻页的新逻辑
            return navigateFromSecondPage(page, currentPageNum + 1);

        } catch (Exception e) {
            error("翻页失败", e);
            return false;
        }
    }

    /**
     * 从第二页开始往后的翻页逻辑
     */
    private boolean navigateFromSecondPage(Page page, int targetPage) {
        final int MAX_RETRIES = 5;

        for (int retry = 1; retry <= MAX_RETRIES; retry++) {
            try {
                log("第 " + retry + " 次尝试翻到第 " + targetPage + " 页");

                // 1. 获取翻页前状态
                PageState beforeState = safeCapturePageState(page);
                log("翻页前状态 - 当前页: " + beforeState.currentActivePage +
                        ", 显示范围: " + beforeState.displayRange +
                        ", 行数: " + beforeState.rowCount);

                // 2. 检查是否已经在目标页
                if (beforeState.currentActivePage == targetPage) {
                    log("✓ 已经在第 " + targetPage + " 页，无需翻页");
                    return true;
                }

                // 3. 检查当前显示范围是否已经是目标页
                if (isExpectedRangeForPage(beforeState.displayRange, targetPage)) {
                    log("✓ 当前显示范围已经是第 " + targetPage + " 页的范围");
                    return true;
                }

                // 4. 执行翻页操作
                boolean pageLoaded = performPageNavigation(page, targetPage, retry, beforeState);

                if (pageLoaded) {
                    log("✓ 翻到第 " + targetPage + " 页成功！");
                    return true;
                }

                // 5. 如果失败，增加等待时间并重试
                int waitTime = 30000 + (retry * 15000); // 每次增加15秒
                log("翻页未成功，等待 " + (waitTime/1000) + " 秒后重试...");
                safeSleep(waitTime);

                // 6. 重试前刷新页面
                if (retry < MAX_RETRIES) {
                    log("重试前刷新页面...");
                    try {
                        page.reload(new Page.ReloadOptions()
                                .setWaitUntil(WaitUntilState.DOMCONTENTLOADED)
                                .setTimeout(30000));
                        safeSleep(5000);
                    } catch (Exception e) {
                        log("刷新页面失败: " + e.getMessage());
                    }
                }

            } catch (Exception e) {
                error("第 " + retry + " 次尝试翻到第 " + targetPage + " 页失败", e);
                if (retry < MAX_RETRIES) {
                    int waitTime = 30000 + (retry * 15000);
                    log("等待 " + (waitTime/1000) + " 秒后重试...");
                    safeSleep(waitTime);
                }
            }
        }

        log("翻到第 " + targetPage + " 页失败，已达到最大重试次数 " + MAX_RETRIES);
        return false;
    }

    /**
     * 执行翻页操作
     */
    private boolean performPageNavigation(Page page, int targetPage, int retryCount, PageState beforeState) {
        try {
            boolean clickPerformed = false;
            long startTime = System.currentTimeMillis();

            // 方法1：优先尝试直接点击页码链接
            String pageSelector = "ul.pagination a:has-text('" + targetPage + "')";
            ElementHandle pageLink = page.querySelector(pageSelector);

            if (pageLink != null && pageLink.isVisible()) {
                log("找到页码" + targetPage + "链接，准备点击");
                pageLink.scrollIntoViewIfNeeded();
                safeSleep(1000);

                try {
                    // 使用更可靠的点击方式
                    pageLink.click(new ElementHandle.ClickOptions()
                            .setDelay(200)
                            .setForce(true)  // 使用强制点击
                            .setClickCount(1));
                    clickPerformed = true;
                    log("已点击页码链接");
                } catch (Exception e) {
                    log("点击失败，尝试JavaScript点击: " + e.getMessage());
                    // 使用JavaScript点击
                    page.evaluate("(link) => link.click()", pageLink);
                    clickPerformed = true;
                }
            }

            // 方法2：如果没有找到链接，使用JavaScript翻页
            if (!clickPerformed) {
                try {
                    log("未找到页码链接，尝试使用JavaScript翻页: goPageNum(" + (targetPage-1) + ")");
                    Object result = page.evaluate("(function() { " +
                            "  try { " +
                            "    goPageNum(" + (targetPage-1) + "); " +
                            "    return true; " +
                            "  } catch(e) { " +
                            "    return false; " +
                            "  } " +
                            "})()");
                    if (Boolean.TRUE.equals(result)) {
                        clickPerformed = true;
                        log("JavaScript翻页调用完成");
                    }
                } catch (Exception e) {
                    log("JavaScript翻页失败: " + e.getMessage());
                }
            }

            // 方法3：尝试点击"次へ"链接
            if (!clickPerformed) {
                ElementHandle nextLink = findNextLink(page);
                if (nextLink != null && nextLink.isVisible()) {
                    log("找到'次へ'链接，准备点击");
                    nextLink.scrollIntoViewIfNeeded();
                    safeSleep(1000);

                    try {
                        nextLink.click(new ElementHandle.ClickOptions()
                                .setDelay(200)
                                .setForce(true)
                                .setClickCount(1));
                        clickPerformed = true;
                        log("已点击'次へ'链接");
                    } catch (Exception e) {
                        log("点击失败，尝试JavaScript点击: " + e.getMessage());
                        page.evaluate("(link) => link.click()", nextLink);
                        clickPerformed = true;
                    }
                }
            }

            if (!clickPerformed) {
                log("无法执行任何翻页操作");
                return false;
            }

            // 点击后立即刷新页面（关键修改）
            log("翻页点击完成，立即刷新页面以确保加载正确数据...");
            safeSleep(2000); // 等待2秒让翻页请求开始

            try {
                // 刷新页面
                page.reload(new Page.ReloadOptions()
                        .setWaitUntil(WaitUntilState.NETWORKIDLE)
                        .setTimeout(60000));

                // 等待页面稳定
                safeSleep(5000);

                // 检查页面是否加载了正确的内容
                PageState afterState = safeCapturePageState(page);
                log("刷新后状态 - 当前页: " + afterState.currentActivePage +
                        ", 显示范围: " + afterState.displayRange +
                        ", 行数: " + afterState.rowCount);

                // 检查翻页是否成功
                if (checkIfPageNavigationSuccessful(afterState, beforeState, targetPage)) {
                    log("翻页成功！已翻到第 " + targetPage + " 页");
                    return true;
                } else {
                    log("翻页后状态不正确，可能需要额外等待");
                }

            } catch (Exception e) {
                log("刷新页面失败: " + e.getMessage());
            }

            // 如果刷新后状态不正确，等待一段时间再检查
            return waitForPageLoadAfterClick(page, retryCount, beforeState, targetPage);

        } catch (Exception e) {
            error("执行翻页操作时出错", e);
            return false;
        }
    }
    /**
     * 查找下一页链接
     */
    private ElementHandle findNextLink(Page page) {
        try {
            // 尝试多种选择器
            String[] selectors = {
                    "ul.pagination a:has-text('次へ')",
                    "ul.pagination a:has-text('次')",
                    "a.next",
                    ".pagination a:contains('次')",
                    "a:has-text('>')",
                    "a:has-text('›')"
            };

            for (String selector : selectors) {
                ElementHandle link = page.querySelector(selector);
                if (link != null && link.isVisible()) {
                    return link;
                }
            }
        } catch (Exception e) {
            // 静默处理
        }
        return null;
    }

    /**
     * 尝试其他翻页方法
     */
    private boolean tryAlternativePageNavigation(Page page, int targetPage) {
        try {
            // 方法1：使用JavaScript点击翻页按钮
            String jsClick = String.format(
                    "var links = document.querySelectorAll('ul.pagination a'); " +
                            "for (var i = 0; i < links.length; i++) { " +
                            "  if (links[i].textContent.trim() === '%d') { " +
                            "    links[i].click(); " +
                            "    return true; " +
                            "  } " +
                            "} " +
                            "return false;", targetPage);

            Object result = page.evaluate(jsClick);
            if (Boolean.TRUE.equals(result)) {
                log("通过JavaScript点击页码成功");
                return true;
            }

            // 方法2：模拟键盘翻页（Page Down）
            page.keyboard().press("PageDown");
            safeSleep(1000);
            log("模拟PageDown按键");

            // 方法3：尝试通过表格判断是否已经翻页
            Elements rows = Jsoup.parse(page.content()).select("div.search_result_main table tbody tr");
            if (rows.size() > 0) {
                // 检查第一条记录的日期是否变化
                String firstDate = extractFirstRowDate(rows);
                if (firstDate != null && !firstDate.isEmpty()) {
                    log("表格中有" + rows.size() + "行数据，第一条记录日期: " + firstDate);
                    return true;
                }
            }

        } catch (Exception e) {
            log("尝试其他翻页方法失败: " + e.getMessage());
        }
        return false;
    }

    /**
     * 翻页后等待页面加载
     */
    private boolean waitForPageLoadAfterClick(Page page, int retryCount, PageState beforeState, int targetPage) {
        try {
            // 增加初始等待时间
            safeSleep(5000);

            int maxChecks = 15; // 减少检查次数
            int checkInterval = 2000; // 增加检查间隔

            for (int i = 0; i < maxChecks; i++) {
                try {
                    log("等待检查(" + (i+1) + "/" + maxChecks + ") - 检查翻页状态");

                    // 每次检查前等待
                    safeSleep(checkInterval);

                    // 尝试获取页面状态，如果失败则跳过
                    PageState currentState;
                    try {
                        currentState = safeCapturePageState(page);
                    } catch (Exception e) {
                        log("获取页面状态失败，继续等待: " + e.getMessage());
                        continue;
                    }

                    log("等待检查 - 当前页: " + currentState.currentActivePage +
                            ", 显示范围: " + currentState.displayRange +
                            ", 行数: " + currentState.rowCount);

                    // 检查是否翻页成功
                    if (checkIfPageNavigationSuccessful(currentState, beforeState, targetPage)) {
                        log("✓ 翻页成功检测到正确状态");
                        return true;
                    }

                    // 如果状态完全空白，尝试刷新页面
                    if (currentState.rowCount == 0 && currentState.displayRange == null) {
                        log("页面状态空白，尝试刷新...");
                        try {
                            page.reload(new Page.ReloadOptions()
                                    .setWaitUntil(WaitUntilState.DOMCONTENTLOADED)
                                    .setTimeout(30000));
                            safeSleep(3000);
                        } catch (Exception e) {
                            log("刷新失败: " + e.getMessage());
                        }
                    }

                } catch (Exception e) {
                    log("等待检查时出错: " + e.getMessage());
                }
            }

            log("等待" + (maxChecks * checkInterval / 1000) + "秒后仍未检测到翻页成功");
            return false;

        } catch (Exception e) {
            error("等待页面加载时出错", e);
            return false;
        }
    }

    /**
     * 检查翻页是否成功
     */
    private boolean checkIfPageNavigationSuccessful(PageState currentState, PageState beforeState, int targetPage) {
        // 1. 检查是否有数据
        if (currentState.rowCount <= 0) {
            log("检查失败：翻页后无数据");
            return false;
        }

        // 2. 检查是否已经在目标页
        if (currentState.currentActivePage == targetPage) {
            log("✓ 页码正确，翻页成功");
            return true;
        }

        // 3. 检查显示范围是否为目标页范围
        if (isExpectedRangeForPage(currentState.displayRange, targetPage)) {
            log("✓ 显示范围正确，翻页成功");
            return true;
        }

        // 4. 检查数据行数是否正确（最后一页可能不满15条）
        if (targetPage == getTotalPages(currentState.displayRange)) {
            // 最后一页，行数可以小于ITEMS_PER_PAGE
            log("目标页是最后一页，接受任意行数: " + currentState.rowCount);
            if (currentState.rowCount > 0) {
                log("✓ 最后一页有数据，翻页成功");
                return true;
            }
        } else if (currentState.rowCount == ITEMS_PER_PAGE) {
            log("✓ 数据行数正确，翻页成功");
            return true;
        }

        // 5. 检查显示范围是否变化
        if (currentState.displayRange != null && beforeState.displayRange != null) {
            if (!currentState.displayRange.equals(beforeState.displayRange)) {
                log("✓ 显示范围已变化，翻页成功");
                return true;
            }
        }

        // 6. 检查第一条记录的日期是否变化（额外验证）
        if (currentState.firstRowDate != null && beforeState.firstRowDate != null) {
            if (!currentState.firstRowDate.equals(beforeState.firstRowDate)) {
                log("✓ 第一条记录日期已变化，翻页成功");
                return true;
            }
        }

        log("检查失败：所有条件都不满足");
        return false;
    }

    /**
     * 从显示范围获取总页数
     */
    private int getTotalPages(String displayRange) {
        try {
            if (displayRange != null && displayRange.contains("件中")) {
                java.util.regex.Matcher matcher = java.util.regex.Pattern
                        .compile("(\\d+)件中")
                        .matcher(displayRange);
                if (matcher.find()) {
                    int totalItems = Integer.parseInt(matcher.group(1));
                    return (int) Math.ceil((double) totalItems / ITEMS_PER_PAGE);
                }
            }
        } catch (Exception e) {
            // 忽略错误
        }
        return 0;
    }

    /**
     * 安全的页面状态捕获方法
     */
    private PageState safeCapturePageState(Page page) {
        PageState state = new PageState();

        try {
            // 先等待页面基本加载
            try {
                page.waitForLoadState(LoadState.DOMCONTENTLOADED, new Page.WaitForLoadStateOptions().setTimeout(5000));
            } catch (Exception e) {
                // 忽略超时，继续
            }

            // 使用更安全的方式获取页面内容
            String html;
            try {
                html = page.content();
            } catch (Exception e) {
                // 如果获取内容失败，尝试等待后重试
                safeSleep(2000);
                html = page.content();
            }

            Document doc = Jsoup.parse(html);

            // 获取表格行数
            Elements tableRows = doc.select("div.search_result_main table tbody tr");
            state.rowCount = tableRows.size();

            // 获取显示范围
            Element displayText = doc.selectFirst("div.result_nav_box p");
            if (displayText == null) {
                displayText = doc.selectFirst(".search_result_bottom p");
            }
            if (displayText != null) {
                state.displayRange = displayText.text().trim();
            }

            // 获取当前页
            state.currentActivePage = getCurrentPageFromDocument(doc);

            // 获取第一条记录的日期
            if (state.rowCount > 0) {
                Element firstRow = tableRows.first();
                Element dateElement = firstRow.selectFirst("span.result_list_start_date");
                if (dateElement != null) {
                    state.firstRowDate = dateElement.text().trim();
                }
            }

        } catch (Exception e) {
            log("安全捕获页面状态时出错: " + e.getMessage());
            // 返回基本状态对象
        }

        return state;
    }

    /**
     * 从文档中获取当前页码
     */
    private int getCurrentPageFromDocument(Document doc) {
        try {
            // 方法1：查找激活的页码
            Element activePageElement = doc.selectFirst("span.pn_active");
            if (activePageElement != null) {
                try {
                    return Integer.parseInt(activePageElement.text().trim());
                } catch (NumberFormatException e) {
                    // 继续尝试其他方法
                }
            }

            // 方法2：从显示范围推断
            return inferPageFromDocument(doc);

        } catch (Exception e) {
            // 忽略错误
        }

        return 1; // 默认返回第一页
    }

    /**
     * 检查显示范围是否符合目标页
     */
    private boolean isExpectedRangeForPage(String displayRange, int targetPage) {
        if (displayRange == null || displayRange.isEmpty()) {
            return false;
        }

        // 预期范围：第1页: 1-15, 第2页: 16-30, 第3页: 31-45, 第4页: 46-60
        int expectedStart = (targetPage - 1) * ITEMS_PER_PAGE + 1;
        int expectedEnd = targetPage * ITEMS_PER_PAGE;
        String expectedStartStr = String.valueOf(expectedStart);

        return displayRange.contains(expectedStartStr);
    }

    /**
     * 检查是否有分页元素
     */
    private boolean hasPaginationElement(Page page) {
        try {
            String html = page.content();
            Document doc = Jsoup.parse(html);
            return doc.selectFirst("ul.pagination") != null;
        } catch (Exception e) {
            return false;
        }
    }

    /**
     * 最终页面检查（等待超时后调用）
     */
    private boolean finalPageCheck(Page page, PageState beforeState, int targetPage) {
        try {
            log("执行最终检查...");

            // 刷新页面获取最新状态
            page.reload(new Page.ReloadOptions()
                    .setWaitUntil(WaitUntilState.DOMCONTENTLOADED)
                    .setTimeout(30000));

            safeSleep(3000);

            // 获取刷新后的状态
            PageState finalState = safeCapturePageState(page);

            log("最终检查状态 - 当前页: " + finalState.currentActivePage +
                    ", 显示范围: " + finalState.displayRange +
                    ", 行数: " + finalState.rowCount);

            // 检查是否已经翻页成功
            if (finalState.rowCount > 0) {
                if (isExpectedRangeForPage(finalState.displayRange, targetPage)) {
                    log("✓ 最终检查：显示范围正确，翻页成功");
                    return true;
                }

                if (finalState.currentActivePage == targetPage) {
                    log("✓ 最终检查：页码正确，翻页成功");
                    return true;
                }

                if (finalState.rowCount == ITEMS_PER_PAGE) {
                    log("✓ 最终检查：数据行数正确，翻页成功");
                    return true;
                }

                // 检查显示范围是否变化
                if (!finalState.displayRange.equals(beforeState.displayRange)) {
                    log("✓ 最终检查：显示范围已变化，翻页成功");
                    return true;
                }
            }

            log("最终检查：翻页可能失败");
            return false;

        } catch (Exception e) {
            error("最终检查时出错", e);
            return false;
        }
    }

    /**
     * 验证翻页是否成功
     */
    private boolean verifyPageNavigation(PageState beforeState, PageState afterState, int targetPage) {
        try {
            log("验证翻页结果 - 期望: 第" + targetPage + "页，实际: 第" + afterState.currentActivePage + "页");

            // 1. 检查是否有数据
            if (afterState.rowCount <= 0) {
                log("验证失败：翻页后无数据");
                return false;
            }

            // 2. 检查显示范围是否合理
            if (afterState.displayRange != null && !afterState.displayRange.isEmpty()) {
                // 示例: "98件中　46-60件を表示中"
                if (afterState.displayRange.contains("件中") && afterState.displayRange.contains("表示中")) {
                    log("显示范围验证通过: " + afterState.displayRange);

                    // 检查是否为期望的范围
                    String expectedRange = calculateExpectedRange(targetPage);
                    if (afterState.displayRange.contains(expectedRange.split("-")[0])) {
                        log("✓ 显示范围匹配期望值: " + expectedRange);
                        return true;
                    }
                }
            }

            // 3. 检查页码是否正确
            if (afterState.currentActivePage == targetPage) {
                log("✓ 页码验证通过: " + afterState.currentActivePage);
                return true;
            }

            // 4. 即使页码不正确，但有15条数据也算成功
            if (afterState.rowCount == ITEMS_PER_PAGE) {
                log("✓ 数据行数正确(" + ITEMS_PER_PAGE + "条)，翻页成功");
                return true;
            }

            // 5. 显示范围发生变化也算成功
            if (!beforeState.displayRange.equals(afterState.displayRange)) {
                log("✓ 显示范围发生变化，翻页成功");
                return true;
            }

            log("翻页验证失败");
            return false;

        } catch (Exception e) {
            error("验证翻页时出错", e);
            return false;
        }
    }

    /**
     * 提取第一行记录的日期（用于验证数据是否更新）
     */
    private String extractFirstRowDate(Elements rows) {
        try {
            if (!rows.isEmpty()) {
                Element firstRow = rows.first();
                Element dateElement = firstRow.selectFirst("span.result_list_start_date");
                return dateElement != null ? dateElement.text().trim() : "";
            }
        } catch (Exception e) {
            // 静默处理
        }
        return "";
    }
    /**
     * 针对第二页开始往后的翻页验证方法
     */
    private boolean verifyPageChangeForSecondPageOnwards(PageState beforeState, PageState afterState, int expectedPage, int retryCount) {
        try {
            log("验证翻页 - 期望页: " + expectedPage + ", 实际页: " + afterState.currentActivePage +
                    ", 显示范围: " + afterState.displayRange + ", 行数: " + afterState.rowCount);

            // 1. 检查是否有数据
            if (afterState.rowCount <= 0) {
                log("验证失败：页面无数据");
                return false;
            }

            // 2. 检查显示范围是否变化
            if (afterState.displayRange != null && beforeState.displayRange != null) {
                if (afterState.displayRange.equals(beforeState.displayRange)) {
                    log("验证失败：显示范围未变化，可能翻页失败");
                    return false;
                }

                // 检查显示范围是否符合预期页码
                // 根据每页15条记录计算，第1页：1-15，第2页：16-30，第3页：31-45，第4页：46-60
                String expectedRange = calculateExpectedRange(expectedPage);
                if (afterState.displayRange.contains(expectedRange)) {
                    log("显示范围验证成功：" + afterState.displayRange + " 符合期望范围：" + expectedRange);
                    return true;
                } else {
                    log("显示范围验证：实际范围=" + afterState.displayRange + "，期望范围应包含=" + expectedRange);
                }
            }

            // 3. 检查页码是否正确
            if (afterState.currentActivePage == expectedPage) {
                log("页码验证成功：当前页=" + afterState.currentActivePage);
                return true;
            } else {
                log("页码验证失败：当前页=" + afterState.currentActivePage + "，期望页=" + expectedPage);

                // 即使页码不正确，但如果显示范围正确且有数据，也可以认为是成功
                if (afterState.rowCount == ITEMS_PER_PAGE) {
                    log("行数正确（" + ITEMS_PER_PAGE + "条），即使页码显示不正确也认为是成功翻页");
                    return true;
                }
            }

            return false;

        } catch (Exception e) {
            error("验证翻页失败", e);
            return false;
        }
    }

    /**
     * 计算期望的显示范围
     */
    private String calculateExpectedRange(int pageNumber) {
        // 每页15条记录
        int start = (pageNumber - 1) * ITEMS_PER_PAGE + 1;
        int end = pageNumber * ITEMS_PER_PAGE;
        return start + "-" + end;
    }

    /**
     * 特殊处理：翻到第二页（带重试机制，需要刷新页面）
     */
    private boolean navigateToSecondPageWithRefresh(Page page) throws InterruptedException {
        final int MAX_RETRIES = 5;
        final int BASE_WAIT_TIME = 60000; // 1分钟基础等待

        for (int retry = 1; retry <= MAX_RETRIES; retry++) {
            try {
                log("第 " + retry + " 次尝试翻到第二页");

                // 1. 获取点击前状态
                int beforePage = getCurrentActivePage(page);
                log("翻页前状态 - 当前页: " + beforePage);

                // 2. 尝试点击页码"2"
                boolean clickSuccess = false;

                // 先尝试点击页码2
                String selector = "ul.pagination a:has-text('2')";
                ElementHandle pageLink = page.querySelector(selector);
                if (pageLink != null && pageLink.isVisible()) {
                    log("找到页码2链接，准备点击");
                    pageLink.scrollIntoViewIfNeeded();
                    page.waitForTimeout(1000);

                    pageLink.click(new ElementHandle.ClickOptions()
                            .setDelay(100)
                            .setForce(false));
                    clickSuccess = true;
                } else {
                    // 尝试点击"次へ"链接
                    ElementHandle nextLink = page.querySelector("ul.pagination a:has-text('次')");
                    if (nextLink != null && nextLink.isVisible()) {
                        log("找到'次へ'链接，准备点击");
                        nextLink.scrollIntoViewIfNeeded();
                        page.waitForTimeout(1000);

                        nextLink.click(new ElementHandle.ClickOptions()
                                .setDelay(100)
                                .setForce(false));
                        clickSuccess = true;
                    }
                }

                if (!clickSuccess) {
                    log("未找到翻页链接");
                    continue;
                }

                // 3. 点击后等待（每次重试增加30秒等待时间）
                int waitTime = BASE_WAIT_TIME + (retry * 30000);
                log("点击完成，等待 " + (waitTime/1000) + " 秒让页面加载...");
                Thread.sleep(waitTime);

                // 4. 强制重新加载页面（刷新）
                log("强制重新加载页面以获取最新状态...");
                page.reload(new Page.ReloadOptions()
                        .setWaitUntil(WaitUntilState.NETWORKIDLE)
                        .setTimeout(60000));

                // 等待页面稳定
                page.waitForTimeout(3000);

                // 5. 获取重新加载后的页面状态
                int afterPage = getCurrentActivePage(page);
                log("重新加载后状态 - 当前页: " + afterPage);

                // 6. 验证翻页是否成功
                if (afterPage == 2) {
                    // 再检查是否有数据
                    PageState state = capturePageState(page);
                    if (state.rowCount > 0) {
                        log("翻页验证成功！已翻到第 2 页，找到 " + state.rowCount + " 条记录");
                        return true;
                    } else {
                        log("页码正确但无数据，翻页可能失败");
                    }
                } else {
                    log("翻页失败：期望第 2 页，实际第 " + afterPage + " 页");
                }

            } catch (Exception e) {
                error("第 " + retry + " 次尝试翻到第二页失败", e);
                if (retry < MAX_RETRIES) {
                    int waitTime = BASE_WAIT_TIME + (retry * 30000);
                    log("等待 " + (waitTime/1000) + " 秒后重试...");
                    Thread.sleep(waitTime);
                }
            }
        }

        log("翻到第二页失败，已达到最大重试次数 " + MAX_RETRIES);
        return false;
    }

    /**
     * 翻到其他页面（第3页及以后）的方法
     */
    private boolean navigateToOtherPage(Page page, int targetPage) throws InterruptedException {
        final int MAX_RETRIES = 5;
        final int BASE_WAIT_TIME = 60000; // 1分钟基础等待

        for (int retry = 1; retry <= MAX_RETRIES; retry++) {
            try {
                log("第 " + retry + " 次尝试翻到第 " + targetPage + " 页");

                // 1. 获取点击前状态
                int beforePage = getCurrentActivePage(page);
                log("翻页前状态 - 当前页: " + beforePage);

                // 2. 查找并点击页码链接
                boolean clickSuccess = false;
                String selector = "ul.pagination a:has-text('" + targetPage + "')";
                ElementHandle pageLink = page.querySelector(selector);

                if (pageLink != null && pageLink.isVisible()) {
                    log("找到页码" + targetPage + "链接，准备点击");
                    pageLink.scrollIntoViewIfNeeded();
                    page.waitForTimeout(1000);

                    pageLink.click(new ElementHandle.ClickOptions()
                            .setDelay(100)
                            .setForce(false));
                    clickSuccess = true;
                } else {
                    // 如果找不到页码链接，尝试点击"次へ"链接
                    ElementHandle nextLink = page.querySelector("ul.pagination a:has-text('次')");
                    if (nextLink != null && nextLink.isVisible()) {
                        log("找到'次へ'链接，准备点击");
                        nextLink.scrollIntoViewIfNeeded();
                        page.waitForTimeout(1000);

                        nextLink.click(new ElementHandle.ClickOptions()
                                .setDelay(100)
                                .setForce(false));
                        clickSuccess = true;
                    }
                }

                if (!clickSuccess) {
                    log("未找到翻页链接");
                    continue;
                }

                // 3. 点击后等待（每次重试增加30秒等待时间）
                int waitTime = BASE_WAIT_TIME + (retry * 30000);
                log("点击完成，等待 " + (waitTime/1000) + " 秒让页面加载...");
                Thread.sleep(waitTime);

                // 4. 获取点击后的页面状态（不重新加载页面）
                int afterPage = getCurrentActivePage(page);
                log("点击后状态 - 当前页: " + afterPage);

                // 5. 验证翻页是否成功
                if (afterPage == targetPage) {
                    // 再检查是否有数据
                    PageState state = capturePageState(page);
                    if (state.rowCount > 0) {
                        log("翻页验证成功！已翻到第 " + targetPage + " 页，找到 " + state.rowCount + " 条记录");
                        return true;
                    } else {
                        log("页码正确但无数据，翻页可能失败");
                    }
                } else {
                    log("翻页失败：期望第 " + targetPage + " 页，实际第 " + afterPage + " 页");
                }

            } catch (Exception e) {
                error("第 " + retry + " 次尝试翻到第 " + targetPage + " 页失败", e);
                if (retry < MAX_RETRIES) {
                    int waitTime = BASE_WAIT_TIME + (retry * 30000);
                    log("等待 " + (waitTime/1000) + " 秒后重试...");
                    Thread.sleep(waitTime);
                }
            }
        }

        log("翻到第 " + targetPage + " 页失败，已达到最大重试次数 " + MAX_RETRIES);
        return false;
    }

    /**
     * 获取当前激活页码
     */
    private int getCurrentActivePage(Page page) {
        try {
            // 等待页面稳定
            page.waitForLoadState(LoadState.DOMCONTENTLOADED);
            page.waitForTimeout(1000);

            String html = page.content();
            Document doc = Jsoup.parse(html);

            Element activePageElement = doc.selectFirst("span.pn_active");
            if (activePageElement != null) {
                try {
                    return Integer.parseInt(activePageElement.text().trim());
                } catch (NumberFormatException e) {
                    return 1;
                }
            }
            return 1;
        } catch (Exception e) {
            error("获取当前激活页码失败", e);
            return 1;
        }
    }

    /**
     * 简化的捕获页面状态方法
     */
    /**
     * 改进的捕获页面状态方法（针对第二页及以后）
     */
    private PageState capturePageState(Page page) {
        try {
            PageState state = new PageState();

            // 等待页面稳定
            page.waitForLoadState(LoadState.DOMCONTENTLOADED);
            page.waitForTimeout(1000);

            // 获取页面HTML
            String html = page.content();
            Document doc = Jsoup.parse(html);

            // 1. 获取当前激活的页码
            Element activePageElement = doc.selectFirst("span.pn_active");
            if (activePageElement != null) {
                try {
                    state.currentActivePage = Integer.parseInt(activePageElement.text().trim());
                } catch (NumberFormatException e) {
                    // 从显示范围推断页码（使用Document版本）
                    state.currentActivePage = inferPageFromDocument(doc);
                }
            } else {
                // 如果没有找到激活页码，从显示范围推断
                state.currentActivePage = inferPageFromDocument(doc);
            }

            // 2. 获取显示范围文本
            Element displayText = doc.selectFirst("div.result_nav_box p");
            if (displayText == null) {
                displayText = doc.selectFirst(".search_result_bottom p");
            }
            if (displayText != null) {
                state.displayRange = displayText.text().trim();
            }

            // 3. 获取当前表格行数
            Elements tableRows = doc.select("div.search_result_main table tbody tr");
            state.rowCount = tableRows.size();

            // 4. 获取第一条记录的日期（用于验证数据是否更新）
            if (state.rowCount > 0) {
                Element firstRow = tableRows.first();
                Element dateElement = firstRow.selectFirst("span.result_list_start_date");
                if (dateElement != null) {
                    state.firstRowDate = dateElement.text().trim();
                }
            }

            log("捕获页面状态 - 当前页: " + state.currentActivePage +
                    ", 显示范围: " + state.displayRange +
                    ", 行数: " + state.rowCount);

            return state;

        } catch (Exception e) {
            error("捕获页面状态失败", e);
            return new PageState();
        }
    }

    /**
     * 从Document对象推断页码
     */
    private int inferPageFromDocument(Document doc) {
        try {
            // 从显示范围文本推断
            Element displayText = doc.selectFirst("div.result_nav_box p");
            if (displayText == null) {
                displayText = doc.selectFirst(".search_result_bottom p");
            }

            if (displayText != null) {
                String displayRange = displayText.text().trim();
                return inferPageFromDisplayRange(displayRange);
            }
        } catch (Exception e) {
            // 忽略错误
        }
        return 1; // 默认返回第一页
    }

    /**
     * 从显示范围推断页码
     */
    private int inferPageFromDisplayRange(String displayRange) {
        try {
            if (displayRange != null && !displayRange.isEmpty()) {
                // 示例文本："98件中　46-60件を表示中"
                if (displayRange.contains("件中") && displayRange.contains("表示中")) {
                    try {
                        String[] parts = displayRange.split("　");
                        if (parts.length >= 2) {
                            String rangePart = parts[1].replace("件を表示中", "");
                            String[] rangeNumbers = rangePart.split("-");
                            if (rangeNumbers.length == 2) {
                                int startItem = Integer.parseInt(rangeNumbers[0]);
                                // 计算页码：起始项/每页数量 + 1
                                int calculatedPage = (startItem - 1) / ITEMS_PER_PAGE + 1;
                                return calculatedPage;
                            }
                        }
                    } catch (Exception e) {
                        // 忽略解析错误
                    }
                }
            }
        } catch (Exception e) {
            // 忽略错误
        }
        return 1; // 默认返回第一页
    }

    /**
     * 检查是否还有更多页面（优化版）
     */
    private boolean hasMorePages(Page page) {
        try {
            // 直接捕获当前页面状态
            PageState state = capturePageState(page);

            log("检查是否有更多页面 - 当前页: " + state.currentActivePage +
                    ", 显示范围: " + state.displayRange +
                    ", 行数: " + state.rowCount);

            // 从显示范围提取总记录数
            if (state.displayRange != null && !state.displayRange.isEmpty()) {
                java.util.regex.Matcher matcher = java.util.regex.Pattern
                        .compile("(\\d+)件中")
                        .matcher(state.displayRange);

                if (matcher.find()) {
                    try {
                        int totalItems = Integer.parseInt(matcher.group(1));
                        int totalPages = (int) Math.ceil((double) totalItems / ITEMS_PER_PAGE);

                        boolean hasMore = state.currentActivePage < totalPages;
                        log("总记录数: " + totalItems + ", 总页数: " + totalPages +
                                ", 当前页: " + state.currentActivePage +
                                ", 还有更多页面: " + hasMore);

                        return hasMore;
                    } catch (NumberFormatException e) {
                        log("无法从显示范围提取数字: " + state.displayRange);
                    }
                }
            }

            // 备选方案：检查下一页链接
            try {
                ElementHandle nextLink = findNextLink(page);
                if (nextLink != null && nextLink.isVisible()) {
                    log("找到下一页链接");
                    return true;
                }
            } catch (Exception e) {
                log("检查下一页链接失败: " + e.getMessage());
            }

            // 保守处理：如果当前页有数据，假设可能还有更多页面
            if (state.rowCount == ITEMS_PER_PAGE) {
                log("当前页有" + ITEMS_PER_PAGE + "条数据，假设还有更多页面");
                return true;
            }

            log("没有更多页面的证据");
            return false;

        } catch (Exception e) {
            error("检查是否有更多页面时出错", e);
            return true; // 出错时保守处理，返回true继续尝试
        }
    }

    /**
     * 单挑数据爬取，爬取成功则更新数据
     * @param detailsUrl 详情页URL
     * @param id 数据库记录ID
     * @return 是否爬取成功
     */
    public boolean crawlSingleItem(String detailsUrl, Long id) {
        if (detailsUrl == null || detailsUrl.isEmpty() || id == null) {
            log("单条数据爬取失败：参数无效 - URL: " + detailsUrl + ", ID: " + id);
            return false;
        }

        log("========== 开始单条数据爬取 ==========");
        log("记录ID: " + id);
        log("详情页URL: " + detailsUrl);
        long startTime = System.currentTimeMillis();

        BrowserContext context = null;
        Page page = null;

        try {
            // 1. 检查并初始化浏览器，每次都初始化一次浏览器，确保页面正常加载
//            if (playwright == null || browser == null) {
                initPlaywright();
//            }

            // 2. 创建浏览器上下文和页面
            context = browser.newContext();
            page = context.newPage();

            // 3. 配置页面
            Map<String, String> headers = new HashMap<>();
            headers.put("User-Agent", USER_AGENT);
            headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            headers.put("Accept-Language", "ja-JP,ja;q=0.9,en;q=0.8");
            headers.put("Referer", RECALL_LIST_URL);
            page.setExtraHTTPHeaders(headers);

            page.addInitScript("() => {\n" +
                    "  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });\n" +
                    "  delete navigator.__proto__.webdriver;\n" +
                    "})");

            // 4. 导航到详情页
            log("正在导航到详情页...");
            page.navigate(detailsUrl, new Page.NavigateOptions()
                    .setTimeout(60000)
                    .setWaitUntil(WaitUntilState.LOAD));

            // 5. 等待页面加载完成
            page.waitForLoadState(LoadState.NETWORKIDLE);
            safeSleep(3000);

            // 6. 获取页面内容并解析
            String html = page.content();
            Document detailDoc = Jsoup.parse(html);

            // 7. 验证页面是否有效
            boolean hasDetailBox = detailDoc.selectFirst("div.detail_box_left") != null ||
                    detailDoc.selectFirst("div.detail_box_bottom") != null;
            boolean hasDetailCap = detailDoc.selectFirst("span.detail_cap") != null;

            if (!hasDetailBox || !hasDetailCap) {
                log("详情页无效或结构异常");

                // 更新数据库状态为失败
                JapanUnqualifiedGoods failedGoods = new JapanUnqualifiedGoods();
                failedGoods.setId(id);
                failedGoods.setStatus(1L); // 1: 爬取失败
                failedGoods.setSpiderTime(new Date());
                japanUnqualifiedGoodsMapper.updateJapanUnqualifiedGoods(failedGoods);

                return false;
            }

            // 8. 提取商品名称
            String productName = extractProductNameFromDetailPage(detailDoc);
            if ("未提供".equals(productName) || "提取失败".equals(productName)) {
                // 尝试从数据库中获取现有商品名称
                JapanUnqualifiedGoods existingGoods = japanUnqualifiedGoodsMapper.selectJapanUnqualifiedGoodsById(id);
                if (existingGoods != null && existingGoods.getGoodsName() != null) {
                    productName = existingGoods.getGoodsName();
                }
            }

            // 9. 创建新的商品对象并提取数据
            JapanUnqualifiedGoods goods = new JapanUnqualifiedGoods();

            // 设置商品名称
            goods.setGoodsName(cleanProductName(productName));

            // 提取所有详情数据
            try {
                // 获取所有detail_cap元素
                Elements detailCaps = detailDoc.select("span.detail_cap");
                log("找到 " + detailCaps.size() + " 个detail_cap元素");

                for (int i = 0; i < detailCaps.size(); i++) {
                    Element cap = detailCaps.get(i);
                    String label = cap.text().trim();
                    Element parentLi = cap.parent();
                    String value = "";

                    if (parentLi != null) {
                        // 尝试从Quill编辑器中提取
                        Element quillEditor = parentLi.selectFirst(".ql-editor");
                        if (quillEditor != null) {
                            value = quillEditor.text().trim();
                        } else {
                            // 尝试从detail_text中提取
                            Element detailText = parentLi.selectFirst("span.detail_text");
                            if (detailText != null) {
                                value = detailText.text().trim();
                            } else {
                                // 提取所有文本，排除标签部分
                                String fullText = parentLi.text().trim();
                                if (fullText.contains(" ")) {
                                    String[] parts = fullText.split("\\s+", 2);
                                    if (parts.length > 1) {
                                        value = parts[1].trim();
                                    }
                                }
                            }
                        }
                    }

                    log("字段 - 标签: " + label + ", 值: " +
                            (value != null ? value.substring(0, Math.min(50, value.length())) + "..." : "null"));

                    // 根据标签设置对应的字段
                    if (value == null || value.trim().isEmpty()) continue;

                    switch (label) {
                        case "商品名":
                            goods.setGoodsName(cleanProductName(value));
                            break;
                        case "連絡先":
                            goods.setPhone(cleanFieldValue(value));
                            break;
                        case "対応方法":
                            goods.setCopingMethod(cleanFieldValue(value));
                            break;
                        case "対応開始日":
                            try {
                                Date actionStartDate = japanDateFormat.parse(value);
                                goods.setRecallTime(actionStartDate);
                            } catch (ParseException e) {
                                // 尝试其他格式
                                try {
                                    Date actionStartDate = japanDateFormat2.parse(value);
                                    goods.setRecallTime(actionStartDate);
                                } catch (ParseException e2) {
                                    warn("日期解析失败: " + value);
                                }
                            }
                            break;
                        case "対象の特定情報":
                            goods.setSpecificInformation(cleanFieldValue(value));
                            break;
                        case "参照情報":
                            goods.setReferenceInformation(cleanFieldValue(value));
                            break;
                        case "備考":
                            goods.setRecycleCause(cleanFieldValue(value));
                            break;
                    }
                }
            } catch (Exception e) {
                error("提取详情数据失败", e);
            }

            // 10. 设置爬取相关信息
            goods.setSpiderTime(new Date());
            goods.setDetailsUrl(detailsUrl);
            goods.setStatus(0L); // 0: 爬取成功

            // 11. 设置ID用于更新
            goods.setId(id);

            // 12. 更新数据库记录
            int result = japanUnqualifiedGoodsMapper.updateJapanUnqualifiedGoods(goods);

            if (result > 0) {
                //更新翻译
                translateGoodsFields(goods,"2");
                long duration = System.currentTimeMillis() - startTime;
                log("单条数据爬取成功！耗时: " + duration + "ms");
                log("========== 单条数据爬取结束 ==========");
                return true;
            } else {
                log("单条数据爬取失败：数据库更新失败");

                // 更新状态为失败
                goods.setStatus(1L);
                japanUnqualifiedGoodsMapper.updateJapanUnqualifiedGoods(goods);

                return false;
            }

        } catch (Exception e) {
            error("单条数据爬取异常", e);

            try {
                // 更新爬取状态为失败
                JapanUnqualifiedGoods failedGoods = new JapanUnqualifiedGoods();
                failedGoods.setId(id);
                failedGoods.setStatus(1L); // 1: 爬取失败
                failedGoods.setSpiderTime(new Date());
                japanUnqualifiedGoodsMapper.updateJapanUnqualifiedGoods(failedGoods);
            } catch (Exception ex) {
                error("更新失败状态时出错", ex);
            }

            return false;
        } finally {
            // 清理资源
            if (page != null) {
                try {
                    page.close();
                } catch (Exception e) {
                    // 静默处理
                }
            }
            if (context != null) {
                try {
                    context.close();
                } catch (Exception e) {
                    // 静默处理
                }
            }

            long endTime = System.currentTimeMillis();
            log("单条数据爬取完成，总耗时: " + (endTime - startTime) + "ms");
        }
    }

    /**
     * 批量保存待处理商品到数据库
     */
    private void batchSavePendingGoods() {
        if (pendingGoodsList.isEmpty()) {
            log("没有待处理的成功记录需要保存");
            return;
        }

        log("\n========== 批量保存成功记录 ==========");
        log("需要保存的记录数: " + pendingGoodsList.size());

        int saveSuccessCount = 0;
        int saveFailureCount = 0;

        for (int i = 0; i < pendingGoodsList.size(); i++) {
            JapanUnqualifiedGoods goods = pendingGoodsList.get(i);

            try {
                // 保存到数据库
                int result = japanUnqualifiedGoodsMapper.insertJapanUnqualifiedGoods(goods);

                if (result > 0) {
                    saveSuccessCount++;
                    log("第 " + (i + 1) + " 条记录保存成功，ID: " + goods.getId());

                    // 翻译
                    translateGoodsFields(goods,"1");
                } else {
                    saveFailureCount++;
                    log("第 " + (i + 1) + " 条记录保存失败");
                }

            } catch (Exception e) {
                saveFailureCount++;
                error("保存第 " + (i + 1) + " 条记录时发生异常", e);
            }
        }

        log("成功记录批量保存完成，成功: " + saveSuccessCount + " 条，失败: " + saveFailureCount + " 条");
        pendingGoodsList.clear();
    }

    /**
     * 批量保存失败记录到数据库
     */
    private void batchSaveFailedGoods() {
        if (failedGoodsList.isEmpty()) {
            log("没有失败记录需要保存");
            return;
        }

        log("\n========== 批量保存失败记录 ==========");
        log("需要保存的失败记录数: " + failedGoodsList.size());

        int saveSuccessCount = 0;
        int saveFailureCount = 0;

        for (int i = 0; i < failedGoodsList.size(); i++) {
            JapanUnqualifiedGoods goods = failedGoodsList.get(i);
            JapanUnqualifiedGoodsZh goodsZh = new JapanUnqualifiedGoodsZh();
            try {
                // 保存到数据库
                int result = japanUnqualifiedGoodsMapper.insertJapanUnqualifiedGoods(goods);
                if (result > 0) {
                    //同时保存到翻译表中，为失败
                    BeanUtils.copyProperties(goods, goodsZh);
                    goodsZh.setBusinessId(goods.getId());
                    goodsZh.setStatus(1L);
                    japanUnqualifiedGoodsZhMapper.insertJapanUnqualifiedGoodsZh(goodsZh);
                    saveSuccessCount++;
                    log("失败记录第 " + (i + 1) + " 条保存成功，ID: " + goods.getId());
                } else {
                    saveFailureCount++;
                    log("失败记录第 " + (i + 1) + " 条保存失败");
                }

            } catch (Exception e) {
                saveFailureCount++;
                error("保存失败记录第 " + (i + 1) + " 条时发生异常", e);
            }
        }

        log("失败记录批量保存完成，成功: " + saveSuccessCount + " 条，失败: " + saveFailureCount + " 条");
    }

    /**
     * 重试失败记录
     */
    private void retryFailedGoods() {
        // 从数据库查询所有状态为失败（status=1）的记录
        List<JapanUnqualifiedGoods> allFailedGoods = japanUnqualifiedGoodsMapper.selectFailedGoods();

        if (allFailedGoods == null || allFailedGoods.isEmpty()) {
            log("没有需要重试的失败记录");
            return;
        }

        log("\n========== 开始重试失败记录 ==========");
        log("需要重试的记录数: " + allFailedGoods.size());

        int retrySuccessCount = 0;
        int retryFailureCount = 0;

        for (int i = 0; i < allFailedGoods.size(); i++) {
            JapanUnqualifiedGoods goods = allFailedGoods.get(i);

            try {
                log("重试第 " + (i + 1) + " 条记录 (ID: " + goods.getId() + ")");
                if (goods.getDetailsUrl() != null && goods.getId() != null) {
                    boolean retryResult = crawlSingleItem(goods.getDetailsUrl(), goods.getId());

                    if (retryResult) {
                        retrySuccessCount++;
                        log("第 " + (i + 1) + " 条记录重试成功");
                    } else {
                        retryFailureCount++;
                        log("第 " + (i + 1) + " 条记录重试失败");
                    }
                } else {
                    log("第 " + (i + 1) + " 条记录缺少必要信息，跳过重试");
                    retryFailureCount++;
                }

                // 随机延时
                randomDelay(2000, 4000);

            } catch (Exception e) {
                retryFailureCount++;
                error("重试第 " + (i + 1) + " 条记录时发生异常", e);
            }
        }

        log("重试完成，成功: " + retrySuccessCount + " 条，失败: " + retryFailureCount + " 条");
    }

// ===================== 内部类 =====================

    /**
     * 页面状态类
     */
    private static class PageState {
        int currentActivePage = 1;
        String displayRange = "";
        int rowCount = 0;
        String firstRowDate = "";

        @Override
        public String toString() {
            return "PageState{" +
                    "currentActivePage=" + currentActivePage +
                    ", displayRange='" + displayRange + '\'' +
                    ", rowCount=" + rowCount +
                    ", firstRowDate='" + firstRowDate + '\'' +
                    '}';
        }
    }


    /**
     * 处理单页的召回项目
     */
    private PageProcessResult processPageRecallItems(Elements recallItems, CrawlConditions conditions, int currentRecallCount) {
        boolean hasNewData = false;
        boolean shouldStopCrawling = false;
        String stopReason = "";

        for (int i = 0; i < recallItems.size(); i++) {
            Element item = recallItems.get(i);
            int seq = currentRecallCount + i + 1;

            try {
                // 检查应对开始日期
                String actionDateStr = extractActionDateFromListItem(item);
                if (actionDateStr != null && !"未提供".equals(actionDateStr)) {
                    try {
                        Date actionDate = parseJapanDate(actionDateStr);

                        // 检查是否应该停止爬取
                        if (shouldStopCrawlingBasedOnDate(actionDate, conditions)) {
                            shouldStopCrawling = true;
                            stopReason = conditions.hasExistingData ?
                                    "遇到小于或等于最新召回日期的数据" : "遇到非目标年份的数据";
                            log("停止爬取条件满足: " + stopReason + ", 应对开始日期: " + actionDateStr);
                            break;
                        }

                        // 检查是否是新数据
                        if (conditions.hasExistingData && !actionDate.after(conditions.minRecallDate)) {
                            // 增量模式下，跳过旧数据
                            continue;
                        }
                    } catch (ParseException e) {
                        warn("第 " + seq + " 条记录日期解析失败: " + actionDateStr);
                        // 日期解析失败，仍然尝试爬取
                    }
                }

                // 爬取详情
                boolean result = crawlRecallDetail(item, seq);
                if (result) {
                    hasNewData = true;
                    successCount++;
                } else {
                    failureCount++;
                }

                // 随机延时，避免请求过于频繁
                randomDelay(1000, 2000);

            } catch (Exception e) {
                failureCount++;
                error("处理第 " + seq + " 条记录时发生异常", e);
            }
        }

        return new PageProcessResult(hasNewData, shouldStopCrawling, stopReason);
    }

    /**
     * 爬取单个召回详情 - 改进版
     */
    /**
     * 爬取单个召回详情 - 修改版：不直接保存数据库
     */
    private boolean crawlRecallDetail(Element listItem, int seq) {
        String initialProductName = extractProductNameFromListItem(listItem);
        String detailRelativeUrl = extractDetailUrlFromListItem(listItem);
        String finalDetailUrl = buildDetailUrl(detailRelativeUrl);
        String actionDateStr = extractActionDateFromListItem(listItem);

        if (finalDetailUrl == null) {
            error("第 " + seq + " 条记录：无法构建详情页URL", null);
            // 创建失败记录并添加到失败列表
            JapanUnqualifiedGoods failedGoods = createFailedGoods(
                    initialProductName, finalDetailUrl, actionDateStr, seq, "无法构建详情页URL");
            failedGoodsList.add(failedGoods);
            return false;
        }

        log("第 " + seq + " 条记录开始爬取，商品: " + initialProductName + ", URL: " + finalDetailUrl);

        int retryCount = 0;
        while (retryCount < MAX_RETRIES) {
            BrowserContext context = null;
            Page page = null;

            try {
                context = browser.newContext();
                page = context.newPage();

                configurePage(page);
                log("第 " + seq + " 条记录：正在爬取详情页 (重试 " + (retryCount + 1) + "/" + MAX_RETRIES + ")");

                // 导航到详情页
                page.navigate(finalDetailUrl, new Page.NavigateOptions()
                        .setTimeout(DETAIL_PAGE_TIMEOUT * 2)
                        .setWaitUntil(WaitUntilState.LOAD));

                // 等待页面完全加载
                page.waitForLoadState(LoadState.NETWORKIDLE);
                safeSleep(3000);

                // 获取页面内容
                String detailHtml = page.content();

                // 验证页面是否包含必要的内容
                if (!detailHtml.contains("detail_box_left") && !detailHtml.contains("detail_cap")) {
                    log("第 " + seq + " 条记录：页面结构异常，可能未正确加载");
                    throw new Exception("页面结构异常");
                }

                Document detailDoc = Jsoup.parse(detailHtml);

                // 获取准确的商品名称
                String actualProductName = extractProductNameFromDetailPage(detailDoc);
                String productName = (!"未提供".equals(actualProductName) && !"提取失败".equals(actualProductName))
                        ? actualProductName : initialProductName;

                // 提取数据到实体对象
                JapanUnqualifiedGoods goods = extractDataToEntity(detailDoc, productName, finalDetailUrl, actionDateStr);
                if (goods != null) {
                    // 设置状态和爬取时间
                    goods.setStatus(0L); // 0: 爬取成功
                    goods.setSpiderTime(new Date());

                    // 添加到待处理列表
                    pendingGoodsList.add(goods);
                    log("第 " + seq + " 条记录：提取成功，已添加到待处理列表");
                    return true;
                } else {
                    // 提取失败，创建失败记录
                    JapanUnqualifiedGoods failedGoods = createFailedGoods(
                            productName, finalDetailUrl, actionDateStr, seq, "提取数据失败");
                    failedGoodsList.add(failedGoods);
                    return false;
                }

            } catch (Exception e) {
                retryCount++;
                if (retryCount >= MAX_RETRIES) {
                    error("第 " + seq + " 条记录：重试 " + MAX_RETRIES + " 次后仍失败", e);
                    // 创建失败记录
                    JapanUnqualifiedGoods failedGoods = createFailedGoods(
                            initialProductName, finalDetailUrl, actionDateStr, seq, "重试多次失败: " + e.getMessage());
                    failedGoodsList.add(failedGoods);
                } else {
                    warn("第 " + seq + " 条记录：第 " + retryCount + " 次重试失败: " + e.getMessage());
                    randomDelay(3000 * retryCount, 5000 * retryCount);
                }
            } finally {
                closeResources(page, context);
            }
        }

        return false;
    }

    /**
     * 创建失败的商品对象
     */
    private JapanUnqualifiedGoods createFailedGoods(String productName, String detailUrl, String actionDateStr, int seq, String reason) {
        JapanUnqualifiedGoods failedGoods = new JapanUnqualifiedGoods();

        // 设置商品名称
        String cleanedProductName = cleanProductName(productName != null ? productName : "未知商品");
        failedGoods.setGoodsName(cleanedProductName);

        // 设置爬取相关信息
        failedGoods.setSpiderTime(new Date());
        failedGoods.setDetailsUrl(detailUrl);
        failedGoods.setStatus(1L); // 1: 爬取失败

        // 使用备注字段存储失败原因
        failedGoods.setRecycleCause("爬取失败: " + reason + " (序号: " + seq + ")");

        // 尝试解析日期
        if (actionDateStr != null && !"未提供".equals(actionDateStr)) {
            try {
                Date actionDate = parseJapanDate(actionDateStr);
                failedGoods.setRecallTime(actionDate);
            } catch (ParseException e) {
                // 日期解析失败，忽略
            }
        }

        return failedGoods;
    }

    /**
     * 提取数据到实体对象
     */
    private JapanUnqualifiedGoods extractDataToEntity(Document detailDoc, String productName, String detailUrl, String actionDateStr) {
        try {
            JapanUnqualifiedGoods goods = new JapanUnqualifiedGoods();

            // 1. 商品名称
            goods.setGoodsName(cleanProductName(productName));

            // 2. 提取所有详情数据
            extractAllDetails(detailDoc, goods);

            // 3. 解析应对开始日期（如果详情页中没有，使用列表页的日期）
            if (goods.getRecallTime() == null && actionDateStr != null && !"未提供".equals(actionDateStr)) {
                try {
                    Date actionDate = parseJapanDate(actionDateStr);
                    goods.setRecallTime(actionDate);
                } catch (ParseException e) {
                    warn("日期解析失败: " + actionDateStr);
                }
            }

            // 4. 设置详情页URL
            goods.setDetailsUrl(detailUrl);

            // 5. 验证必要数据
            boolean hasEssentialData = goods.getGoodsName() != null &&
                    (goods.getPhone() != null ||
                            goods.getCopingMethod() != null ||
                            goods.getRecallTime() != null);

            if (!hasEssentialData) {
                warn("记录缺少必要数据");
                return null;
            }

            return goods;

        } catch (Exception e) {
            error("提取数据失败", e);
            return null;
        }
    }

/*
    */
/**
     * 提取数据并保存到数据库
     *//*

    private boolean extractAndSaveToDatabase(Document detailDoc, String productName, String detailUrl) {
        try {
            JapanUnqualifiedGoods goods = new JapanUnqualifiedGoods();

            // 设置商品名称
            String cleanedProductName = cleanProductName(cleanFieldValue(productName));
            goods.setGoodsName(cleanedProductName);

            // 提取联络方式
            goods.setPhone(cleanFieldValue(extractFieldByLabel(detailDoc, "連絡先")));

            // 提取应对方法
            goods.setCopingMethod(cleanFieldValue(extractFieldByLabel(detailDoc, "対応方法")));


            // 提取应对开始日期
            String actionStartDateStr = extractFieldByLabel(detailDoc, "対応開始日");
            if (isValidFieldValue(actionStartDateStr)) {
                try {
                    Date actionStartDate = japanDateFormat.parse(actionStartDateStr);
                    goods.setRecallTime(actionStartDate);
                } catch (ParseException e) {
                    warn("应对开始日期解析失败: " + actionStartDateStr);
                }
            }

            // 提取应对方法
            goods.setSpecificInformation(cleanFieldValue(extractFieldByLabel(detailDoc, "対象の特定情報")));

            goods.setReferenceInformation(cleanFieldValue(extractFieldByLabel(detailDoc, "参照情報")));

            goods.setRecycleCause(cleanFieldValue(extractFieldByLabel(detailDoc, "備考")));
            // 设置爬取相关信息
            goods.setSpiderTime(new Date());
            goods.setDetailsUrl(detailUrl);
            goods.setStatus(0L); // 0: 爬取成功

            // 保存到数据库
            int result = japanUnqualifiedGoodsMapper.insertJapanUnqualifiedGoods(goods);
            return result > 0;

        } catch (Exception e) {
            error("保存到数据库失败", e);
            return false;
        }
    }
*/

    /**
     * 保存失败记录到数据库
     */
    private void saveFailedRecord(int seq, String productName, String detailUrl) {
        try {
            JapanUnqualifiedGoods goods = new JapanUnqualifiedGoods();

            String cleanedProductName = cleanProductName(productName != null ? productName : "未知商品");
            goods.setGoodsName(cleanedProductName);

            goods.setPhone(null);
            goods.setCopingMethod(null);
            goods.setRecallTime(null);

            goods.setSpiderTime(new Date());
            goods.setDetailsUrl(detailUrl);
            goods.setStatus(1L); // 1: 爬取失败

            int result = japanUnqualifiedGoodsMapper.insertJapanUnqualifiedGoods(goods);
            if (result > 0) {
                log("第 " + seq + " 条失败记录已保存到数据库");
            } else {
                error("第 " + seq + " 条失败记录保存到数据库失败", null);
            }

        } catch (Exception e) {
            error("保存失败记录到数据库失败", e);
        }
    }

    // ===================== 辅助方法 =====================

    /**
     * 获取当前年份的开始日期
     */
    private Date getCurrentYearStartDate() {
        Calendar cal = Calendar.getInstance();
        cal.set(Calendar.MONTH, Calendar.JANUARY);
        cal.set(Calendar.DAY_OF_MONTH, 1);
        cal.set(Calendar.HOUR_OF_DAY, 0);
        cal.set(Calendar.MINUTE, 0);
        cal.set(Calendar.SECOND, 0);
        cal.set(Calendar.MILLISECOND, 0);
        return cal.getTime();
    }

    /**
     * 初始化Playwright和浏览器
     */
    private void initPlaywright() {
        try {
            playwright = Playwright.create();
            List<String> browserArgs = Arrays.asList(
                    "--disable-gpu",
                    "--disable-extensions",
                    "--no-sandbox",
                    "--disable-dev-shm-usage",
                    "--lang=ja-JP"
            );

            BrowserType.LaunchOptions launchOptions = new BrowserType.LaunchOptions()
                    .setHeadless(true)
                    .setArgs(browserArgs)
                    .setTimeout(120000);

            browser = playwright.chromium().launch(launchOptions);
            log("Playwright和浏览器初始化完成");
        } catch (Exception e) {
            error("Playwright初始化失败", e);
            throw new RuntimeException("初始化浏览器失败", e);
        }
    }

    /**
     * 清理资源
     */
    private void cleanupResources() {
        if (browser != null) {
            try {
                browser.close();
                log("浏览器已关闭");
            } catch (Exception e) {
                warn("关闭浏览器时出错: " + e.getMessage());
            }
        }

        if (playwright != null) {
            try {
                playwright.close();
                log("Playwright已关闭");
            } catch (Exception e) {
                warn("关闭Playwright时出错: " + e.getMessage());
            }
        }
    }

    /**
     * 配置浏览器
     */
    private void configureBrowser(Page page) {
        Map<String, String> headers = new HashMap<>();
        headers.put("User-Agent", USER_AGENT);
        headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        headers.put("Accept-Language", "ja-JP,ja;q=0.9,en;q=0.8");
        page.setExtraHTTPHeaders(headers);
    }

    /**
     * 配置页面
     */
    private void configurePage(Page page) {
        configureBrowser(page);
        page.addInitScript("() => {\n" +
                "  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });\n" +
                "  delete navigator.__proto__.webdriver;\n" +
                "}");
    }

    /**
     * 等待页面加载
     */
    private void waitForPageLoad(Page page) {
        try {
            page.waitForLoadState(LoadState.DOMCONTENTLOADED);
            randomDelay(1000, 2000);
        } catch (Exception e) {
            warn("等待页面加载失败: " + e.getMessage());
        }
    }

    /**
     * 等待页面稳定
     */
    private void waitForPageStable(Page page) {
        try {
            page.waitForLoadState(LoadState.NETWORKIDLE);
            randomDelay(1000, 1500);
        } catch (Exception e) {
            warn("等待页面稳定失败: " + e.getMessage());
        }
    }

    /**
     * 获取页面文档
     */
    private Document getPageDocument(Page page) {
        try {
            String content = page.content();
            return Jsoup.parse(content);
        } catch (Exception e) {
            error("获取页面文档失败", e);
            return null;
        }
    }

    /**
     * 验证页面内容
     */
    private boolean verifyPageContent(Page page) {
        try {
            Document doc = getPageDocument(page);
            return doc != null &&
                    doc.selectFirst("div.search_result_main") != null &&
                    doc.selectFirst("ul.pagination") != null;
        } catch (Exception e) {
            return false;
        }
    }

    /**
     * 提取召回项目 - 改进版
     */
    private Elements extractRecallItems(Document listDoc) {
        try {
            // 方法1：使用更精确的选择器
            Elements items = listDoc.select("div.search_result_main table.list_table tbody tr");

            // 如果找不到，尝试其他选择器
            if (items.isEmpty()) {
                items = listDoc.select("table.list_table tbody tr");
            }
            if (items.isEmpty()) {
                items = listDoc.select("div.search_result_main tbody tr");
            }
            if (items.isEmpty()) {
                items = listDoc.select("table tbody tr");
            }
            if (items.isEmpty()) {
                items = listDoc.select("tbody tr");
            }

            log("找到 " + items.size() + " 个列表项");

            // 验证每个项目是否都有必要的元素
            for (int i = 0; i < items.size(); i++) {
                Element item = items.get(i);
                Element link = item.selectFirst("td.new_window a");
                Element date = item.selectFirst("span.result_list_start_date");

                log("第 " + (i+1) + " 项 - 链接: " + (link != null ? "有" : "无") +
                        ", 日期: " + (date != null ? date.text() : "无"));
            }

            return items;
        } catch (Exception e) {
            error("提取召回项目失败", e);
            return new Elements();
        }
    }

    /**
     * 随机延时
     */
    private void randomDelay(int minMillis, int maxMillis) {
        try {
            int delay = minMillis + new Random().nextInt(maxMillis - minMillis);
            TimeUnit.MILLISECONDS.sleep(delay);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }

    /**
     * 根据日期判断是否应该停止爬取
     */
    private boolean shouldStopCrawlingBasedOnDate(Date recallDate, CrawlConditions conditions) {
        try {
            if (conditions.hasExistingData) {
                // 增量模式：遇到小于或等于最新召回日期的数据时停止
                return !recallDate.after(conditions.minRecallDate);
            } else {
                // 全量模式：遇到非目标年份的数据时停止
                Calendar cal = Calendar.getInstance();
                cal.setTime(recallDate);
                return cal.get(Calendar.YEAR) != conditions.targetYear;
            }
        } catch (Exception e) {
            return false;
        }
    }

    /**
     * 打印统计信息
     */
    private void printStatistics() {
        long endTime = System.currentTimeMillis();
        long durationSeconds = (endTime - startTime) / 1000;
        long durationMinutes = durationSeconds / 60;
        long remainingSeconds = durationSeconds % 60;

        log("\n==================== 爬取完成 ====================");
        log("总计耗时: " + durationMinutes + "分" + remainingSeconds + "秒 (" + durationSeconds + "秒)");
        log("总计处理页数: " + totalPagesProcessed);
        log("总计爬取召回商品数: " + recallCount);
        log("成功爬取: " + successCount + " 条");
        log("失败爬取: " + failureCount + " 条");
        if (recallCount > 0) {
            log("成功率: " + String.format("%.2f%%", (successCount * 100.0 / recallCount)));
        } else {
            log("成功率: 0%");
        }

        // 添加性能统计
        if (totalPagesProcessed > 0) {
            log("平均每页处理时间: " + String.format("%.2f", (durationSeconds * 1.0 / totalPagesProcessed)) + "秒");
        }
    }

    /**
     * 安全关闭资源
     */
    private void closeResources(Page page, BrowserContext context) {
        if (page != null) {
            try {
                page.close();
            } catch (Exception e) {
                // 静默处理
            }
        }
        if (context != null) {
            try {
                context.close();
            } catch (Exception e) {
                // 静默处理
            }
        }
    }

    // ===================== 日志方法 =====================

    private void log(String message) {
        System.out.println("[" + dateFormat.format(new Date()) + "] " + message);
    }

    private void warn(String message) {
        System.err.println("[" + dateFormat.format(new Date()) + "] [WARN] " + message);
    }

    private void error(String message, Exception e) {
        System.err.println("[" + dateFormat.format(new Date()) + "] [ERROR] " + message);
        if (e != null) {
            e.printStackTrace();
        }
    }

    // ===================== 数据提取方法 =====================

    /**
     * 从列表项中提取商品名称
     */
    private String extractProductNameFromListItem(Element listItem) {
        try {
            Element link = listItem.selectFirst("td.new_window a");
            return link != null ? link.text().trim() : "未知商品";
        } catch (Exception e) {
            return "未知商品";
        }
    }

    /**
     * 从列表项中提取详情页链接
     */
    private String extractDetailUrlFromListItem(Element listItem) {
        try {
            Element link = listItem.selectFirst("td.new_window a");
            if (link != null) {
                String href = link.attr("href");
                if (href.startsWith("/")) {
                    return BASE_URL + href;
                }
                return href;
            }
        } catch (Exception e) {
            // 静默处理
        }
        return "";
    }

    /**
     * 从详情页提取商品名称
     */
    private String extractProductNameFromDetailPage(Document detailDoc) {
        try {
            String productName = extractFieldByLabel(detailDoc, "商品名");
            return (productName != null && !productName.trim().isEmpty()) ? productName.trim() : "未提供";
        } catch (Exception e) {
            return "未提供";
        }
    }

    /**
     * 从列表项中提取应对开始日期
     */
    private String extractActionDateFromListItem(Element listItem) {
        try {
            Element dateElement = listItem.select("td span.result_list_start_date").first();
            return dateElement != null ? dateElement.text().trim() : "未提供";
        } catch (Exception e) {
            return "未提供";
        }
    }

    /**
     * 根据字段标签提取信息
     */
    private String extractFieldByLabel(Document doc, String fieldLabel) {
        try {
            // 方法1：在 detail_box_left 中查找
            Elements lisLeft = doc.select("div.detail_box_left ul li");
            for (Element li : lisLeft) {
                Element cap = li.selectFirst("span.detail_cap");
                if (cap != null && fieldLabel.equals(cap.text().trim())) {
                    return extractTextFromLi(li);
                }
            }

            // 方法2：在 detail_box_bottom 中查找
            Elements lisBottom = doc.select("div.detail_box_bottom ul li");
            for (Element li : lisBottom) {
                Element cap = li.selectFirst("span.detail_cap");
                if (cap != null && fieldLabel.equals(cap.text().trim())) {
                    return extractTextFromLi(li);
                }
            }

            // 方法3：查找所有包含指定标签的元素
            Elements allCaps = doc.select("span.detail_cap");
            for (Element cap : allCaps) {
                if (fieldLabel.equals(cap.text().trim())) {
                    Element parentLi = cap.parent();
                    if (parentLi != null) {
                        return extractTextFromLi(parentLi);
                    }
                }
            }

            return null;
        } catch (Exception e) {
            error("提取字段 '" + fieldLabel + "' 失败: " + e.getMessage(), e);
            return null;
        }
    }

    /**
     * 从li元素中提取文本内容（处理Quill编辑器）
     */
    private String extractTextFromLi(Element li) {
        try {
            // 先尝试从Quill编辑器中提取
            Element quillEditor = li.selectFirst(".ql-editor");
            if (quillEditor != null) {
                return quillEditor.text().trim();
            }

            // 如果没有Quill编辑器，尝试从detail_text中提取
            Element detailText = li.selectFirst("span.detail_text");
            if (detailText != null) {
                return detailText.text().trim();
            }

            // 最后提取整个li的文本（排除标签部分）
            String fullText = li.text().trim();
            if (fullText.contains(" ")) {
                // 去掉标签部分
                return fullText.substring(fullText.indexOf(" ") + 1).trim();
            }

            return fullText;
        } catch (Exception e) {
            return "";
        }
    }

    /**
     * 从详情页提取所有需要的数据
     */
/*
    private boolean extractAndSaveToDatabase(Document detailDoc, String productName, String detailUrl, int seq) {
        try {
            JapanUnqualifiedGoods goods = new JapanUnqualifiedGoods();

            log("处理第 " + seq + " 条记录的详情页，URL: " + detailUrl);

            // 1. 商品名称
            goods.setGoodsName(cleanProductName(productName));

            // 2. 提取所有可能的数据字段
            extractAllDetails(detailDoc, goods);

            // 3. 爬取相关信息
            goods.setSpiderTime(new Date());
            goods.setDetailsUrl(detailUrl);
            goods.setStatus(0L); // 0: 爬取成功

            // 详细日志输出，便于调试
            log("第 " + seq + " 条记录提取结果:");
            log("  商品名称: " + goods.getGoodsName());
            log("  联络方式: " + (goods.getPhone() != null ?
                    goods.getPhone().substring(0, Math.min(50, goods.getPhone().length())) + "..." : "null"));
            log("  对应方法: " + (goods.getCopingMethod() != null ?
                    goods.getCopingMethod().substring(0, Math.min(100, goods.getCopingMethod().length())) + "..." : "null"));
            log("  应对开始日: " + (goods.getRecallTime() != null ? dateFormat.format(goods.getRecallTime()) : "null"));

            // 检查是否有必要的数据
            boolean hasEssentialData = goods.getPhone() != null || goods.getCopingMethod() != null ||
                    goods.getRecallTime() != null;

            if (!hasEssentialData) {
                warn("第 " + seq + " 条记录缺少必要数据，可能提取失败");

                // 尝试备用提取方法
                boolean extractedByAlternative = extractByAlternativeMethod(detailDoc, goods);
                if (!extractedByAlternative) {
                    // 保存失败记录
                    saveFailedRecord(seq, productName, detailUrl);
                    return false;
                }
            }

            // 保存到数据库
            int result = japanUnqualifiedGoodsMapper.insertJapanUnqualifiedGoods(goods);
            if (result > 0) {
                log("第 " + seq + " 条记录保存成功");
                //翻译
                translateGoodsFields(goods);
                return true;
            } else {
                error("第 " + seq + " 条记录保存到数据库失败", null);
                return false;
            }

        } catch (Exception e) {
            error("第 " + seq + " 条记录提取和保存数据失败", e);
            return false;
        }
    }
*/

    /**
     * 提取所有详情数据
     */
    private void extractAllDetails(Document detailDoc, JapanUnqualifiedGoods goods) {
        try {
            // 获取所有detail_cap元素
            Elements detailCaps = detailDoc.select("span.detail_cap");
            log("找到 " + detailCaps.size() + " 个detail_cap元素");

            for (int i = 0; i < detailCaps.size(); i++) {
                Element cap = detailCaps.get(i);
                String label = cap.text().trim();
                String value = extractTextFromElement(cap.parent());

                log("第 " + (i+1) + " 个字段 - 标签: " + label + ", 值: " +
                        (value != null ? value.substring(0, Math.min(50, value.length())) + "..." : "null"));

                // 根据标签设置对应的字段
                setFieldByLabel(goods, label, value);
            }

            // 备用方法：直接搜索特定标签
            if (goods.getPhone() == null) {
                goods.setPhone(extractFieldByAlternativeMethod(detailDoc, "連絡先"));
            }
            if (goods.getCopingMethod() == null) {
                goods.setCopingMethod(extractFieldByAlternativeMethod(detailDoc, "対応方法"));
            }

        } catch (Exception e) {
            error("提取所有详情数据失败", e);
        }
    }

    /**
     * 翻译商品字段
     */
    private void translateGoodsFields(JapanUnqualifiedGoods goods,String type) {
        JapanUnqualifiedGoodsZh en = new JapanUnqualifiedGoodsZh();
        BeanUtils.copyProperties(goods, en);
        en.setBusinessId(goods.getId());
        try {
            //商品名称
            if (goods.getGoodsName() != null && !goods.getGoodsName().trim().isEmpty()) {
                String translatedName = baiduTranslationService.translateWithCache(
                        goods.getGoodsName(), "jp", "zh");
                en.setGoodsName(translatedName); // 需要实体类有此字段
            }

            //联络方式
            if (goods.getPhone() != null && !goods.getPhone().trim().isEmpty()) {
                String translatedName = baiduTranslationService.translateWithCache(
                        goods.getPhone(), "jp", "zh");
                en.setPhone(translatedName); // 需要实体类有此字段
            }

            // 应对方法
            if (goods.getCopingMethod() != null && !goods.getCopingMethod().trim().isEmpty()) {
                String translatedDanger = baiduTranslationService.translateWithCache(
                        goods.getCopingMethod(), "jp", "zh");
                en.setCopingMethod(translatedDanger);
            }

            //特定情报
            if (goods.getSpecificInformation() != null && !goods.getSpecificInformation().trim().isEmpty()) {
                String translatedDanger = baiduTranslationService.translateWithCache(
                        goods.getSpecificInformation(), "jp", "zh");
                en.setSpecificInformation(translatedDanger);
            }

            //参照情报
            if (goods.getReferenceInformation() != null && !goods.getReferenceInformation().trim().isEmpty()) {
                String translatedDanger = baiduTranslationService.translateWithCache(
                        goods.getReferenceInformation(), "jp", "zh");
                en.setReferenceInformation(translatedDanger);
            }

            //回收原因
            if (goods.getRecycleCause() != null && !goods.getRecycleCause().trim().isEmpty()) {
                String translatedDanger = baiduTranslationService.translateWithCache(
                        goods.getRecycleCause(), "jp", "zh");
                en.setRecycleCause(translatedDanger);
            }

            // 设置翻译状态
            en.setTranslated("0");
        } catch (Exception e) {
            System.err.println("翻译商品字段失败，但继续保存原文: " + e.getMessage());
            en.setTranslated("1");
            // 即使翻译失败，也继续保存原文
        } finally {
            en.setTranslatedTime(new Date());
            if("2".equals(type)){
                japanUnqualifiedGoodsZhMapper.updateJapanUnqualifiedGoodsZh(en);
            }else{
                japanUnqualifiedGoodsZhMapper.insertJapanUnqualifiedGoodsZh(en);
            }
        }

    }

    /**
     * 从元素中提取文本（增强版）
     */
    private String extractTextFromElement(Element element) {
        if (element == null) return null;

        try {
            // 方法1：尝试从Quill编辑器中提取
            Element quillEditor = element.selectFirst(".ql-editor");
            if (quillEditor != null) {
                return quillEditor.text().trim();
            }

            // 方法2：尝试从detail_text中提取
            Element detailText = element.selectFirst("span.detail_text");
            if (detailText != null) {
                return detailText.text().trim();
            }

            // 方法3：提取所有文本，排除标签部分
            String fullText = element.text().trim();

            // 尝试找到标签分隔符
            if (fullText.contains(" ")) {
                String[] parts = fullText.split("\\s+", 2);
                if (parts.length > 1) {
                    return parts[1].trim();
                }
            }

            return fullText;

        } catch (Exception e) {
            return null;
        }
    }

    /**
     * 备用字段提取方法（当标准方法失败时使用）
     */
    private String extractFieldByAlternativeMethod(Document detailDoc, String fieldLabel) {
        try {
            log("使用备用方法提取字段: " + fieldLabel);

            // 方法1：在detail_box_left中直接搜索
            Element detailBoxLeft = detailDoc.selectFirst("div.detail_box_left");
            if (detailBoxLeft != null) {
                String boxLeftHtml = detailBoxLeft.html();
                return extractFieldFromHtml(boxLeftHtml, fieldLabel);
            }

            // 方法2：在整个页面中搜索
            String fullHtml = detailDoc.html();
            return extractFieldFromHtml(fullHtml, fieldLabel);

        } catch (Exception e) {
            error("备用方法提取字段 '" + fieldLabel + "' 失败", e);
            return null;
        }
    }

    /**
     * 从HTML中提取字段内容
     */
    private String extractFieldFromHtml(String html, String fieldLabel) {
        try {
            // 构建正则表达式模式
            // 尝试匹配格式：<span class="detail_cap">連絡先</span>...</span>
            String pattern1 = "<span\\s+class\\s*=\\s*[\"']detail_cap[\"'][^>]*>" +
                    fieldLabel +
                    "</span>\\s*<span\\s+class\\s*=\\s*[\"']detail_text[\"'][^>]*>(.*?)</span>";

            java.util.regex.Matcher matcher1 = java.util.regex.Pattern
                    .compile(pattern1, java.util.regex.Pattern.DOTALL)
                    .matcher(html);

            if (matcher1.find()) {
                String value = matcher1.group(1).trim();
                // 清理HTML标签
                value = value.replaceAll("<[^>]+>", "").trim();
                if (!value.isEmpty()) {
                    log("通过正则模式1找到字段 '" + fieldLabel + "': " + value);
                    return value;
                }
            }

            // 方法2：尝试匹配：<span class="detail_cap">連絡先</span>任意内容</li>
            String pattern2 = "<span\\s+class\\s*=\\s*[\"']detail_cap[\"'][^>]*>" +
                    fieldLabel +
                    "</span>\\s*(.*?)\\s*</li>";

            java.util.regex.Matcher matcher2 = java.util.regex.Pattern
                    .compile(pattern2, java.util.regex.Pattern.DOTALL)
                    .matcher(html);

            if (matcher2.find()) {
                String value = matcher2.group(1).trim();
                // 移除可能的嵌套span标签
                value = value.replaceAll("<span[^>]*>", "").replaceAll("</span>", "").trim();
                value = value.replaceAll("<[^>]+>", "").trim();
                if (!value.isEmpty()) {
                    log("通过正则模式2找到字段 '" + fieldLabel + "': " + value);
                    return value;
                }
            }

            // 方法3：尝试匹配标签后跟冒号的内容
            String pattern3 = fieldLabel + "[：:](.*?)(?=<br>|</span>|</div>|</li>)";
            java.util.regex.Matcher matcher3 = java.util.regex.Pattern
                    .compile(pattern3, java.util.regex.Pattern.DOTALL)
                    .matcher(html);

            if (matcher3.find()) {
                String value = matcher3.group(1).trim();
                value = value.replaceAll("<[^>]+>", "").trim();
                if (!value.isEmpty()) {
                    log("通过正则模式3找到字段 '" + fieldLabel + "': " + value);
                    return value;
                }
            }

            log("未找到字段 '" + fieldLabel + "'");
            return null;

        } catch (Exception e) {
            error("从HTML提取字段 '" + fieldLabel + "' 失败", e);
            return null;
        }
    }

    /**
     * 备用提取方法 - 使用JSOUP直接搜索
     */
    private String extractFieldByJsoupSearch(Document detailDoc, String fieldLabel) {
        try {
            log("使用JSOUP搜索方法提取字段: " + fieldLabel);

            // 方法1：在detail_box_left中搜索所有li元素
            Elements lis = detailDoc.select("div.detail_box_left ul li");
            for (Element li : lis) {
                String liText = li.text().trim();
                if (liText.startsWith(fieldLabel + " ") || liText.startsWith(fieldLabel + "：") ||
                        liText.startsWith(fieldLabel + ":")) {
                    String value = liText.substring(fieldLabel.length()).trim();
                    // 移除开头的冒号或空格
                    value = value.replaceFirst("^[：:\\s]+", "").trim();
                    if (!value.isEmpty()) {
                        log("通过JSOUP搜索找到字段 '" + fieldLabel + "': " + value);
                        return value;
                    }
                }
            }

            // 方法2：搜索所有包含字段标签的文本
            Elements allElements = detailDoc.select("*:containsOwn(" + fieldLabel + ")");
            for (Element element : allElements) {
                String elementText = element.ownText().trim();
                if (elementText.contains(fieldLabel)) {
                    String value = elementText.substring(elementText.indexOf(fieldLabel) + fieldLabel.length()).trim();
                    value = value.replaceFirst("^[：:\\s]+", "").trim();
                    if (!value.isEmpty()) {
                        log("通过元素搜索找到字段 '" + fieldLabel + "': " + value);
                        return value;
                    }
                }
            }

            return null;

        } catch (Exception e) {
            error("JSOUP搜索提取字段 '" + fieldLabel + "' 失败", e);
            return null;
        }
    }

    /**
     * 备用提取方法
     */
    private boolean extractByAlternativeMethod(Document detailDoc, JapanUnqualifiedGoods goods) {
        try {
            // 方法1：直接搜索包含关键词的文本
            String html = detailDoc.html();

            // 查找连絡先
            if (goods.getPhone() == null) {
                String phonePattern = "連絡先[：:](.*?)(?=<br>|</span>|</li>)";
                java.util.regex.Matcher phoneMatcher = java.util.regex.Pattern
                        .compile(phonePattern, java.util.regex.Pattern.DOTALL)
                        .matcher(html);
                if (phoneMatcher.find()) {
                    goods.setPhone(cleanFieldValue(phoneMatcher.group(1).trim()));
                }
            }

            // 查找対応方法
            if (goods.getCopingMethod() == null) {
                String methodPattern = "対応方法[：:](.*?)(?=<br>|</span>|</li>)";
                java.util.regex.Matcher methodMatcher = java.util.regex.Pattern
                        .compile(methodPattern, java.util.regex.Pattern.DOTALL)
                        .matcher(html);
                if (methodMatcher.find()) {
                    goods.setCopingMethod(cleanFieldValue(methodMatcher.group(1).trim()));
                }
            }

            return goods.getPhone() != null || goods.getCopingMethod() != null;

        } catch (Exception e) {
            return false;
        }
    }

    /**
     * 设置字段根据标签
     */
    private void setFieldByLabel(JapanUnqualifiedGoods goods, String label, String value) {
        if (value == null || value.trim().isEmpty()) return;

        try {
            switch (label) {
                case "商品名":
                    goods.setGoodsName(cleanProductName(value));
                    break;
                case "連絡先":
                    goods.setPhone(cleanFieldValue(value));
                    break;
                case "対応方法":
                    goods.setCopingMethod(cleanFieldValue(value));
                    break;
                case "対応開始日":
                    try {
                        Date actionStartDate = japanDateFormat.parse(value);
                        goods.setRecallTime(actionStartDate);
                    } catch (ParseException e) {
                        // 尝试其他格式
                        try {
                            Date actionStartDate = japanDateFormat2.parse(value);
                            goods.setRecallTime(actionStartDate);
                        } catch (ParseException e2) {
                            warn("日期解析失败: " + value);
                        }
                    }
                    break;
                case "対象の特定情報":
                    goods.setSpecificInformation(cleanFieldValue(value));
                    break;
                case "参照情報":
                    goods.setReferenceInformation(cleanFieldValue(value));
                    break;
                case "備考":
                    goods.setRecycleCause(cleanFieldValue(value));
                    break;
            }
        } catch (Exception e) {
            error("设置字段 '" + label + "' 失败", e);
        }
    }


    /**
     * URL 构建方法
     */
    private String buildDetailUrl(String detailRelativeUrl) {
        if (detailRelativeUrl == null || detailRelativeUrl.isEmpty()) {
            return null;
        }
        if (detailRelativeUrl.startsWith("http")) {
            return detailRelativeUrl;
        } else if (detailRelativeUrl.startsWith("/")) {
            return BASE_URL + detailRelativeUrl;
        } else {
            return BASE_URL + "/" + detailRelativeUrl;
        }
    }

    /**
     * 解析日本日期格式
     */
    private Date parseJapanDate(String dateStr) throws ParseException {
        try {
            return japanDateFormat.parse(dateStr);
        } catch (ParseException e1) {
            try {
                return japanDateFormat2.parse(dateStr);
            } catch (ParseException e2) {
                throw new ParseException("无法解析日期格式: " + dateStr, 0);
            }
        }
    }

    /**
     * 清理商品名称
     */
    private String cleanProductName(String productName) {
        if (productName == null) {
            return null;
        }
        return productName.replace(" - 交換／回収", "")
                .replace(" - 返金／回収", "")
                .replace(" - 交換／返金", "")
                .replace(" - 修理", "")
                .trim();
    }

    /**
     * 清理字段值
     */
    private String cleanFieldValue(String fieldValue) {
        if (fieldValue == null || fieldValue.trim().isEmpty() ||
                "未提供".equals(fieldValue.trim()) || "提取失败".equals(fieldValue.trim())) {
            return null;
        }
        return fieldValue.trim();
    }

    /**
     * 检查字段值是否有效
     */
    private boolean isValidFieldValue(String fieldValue) {
        return fieldValue != null && !fieldValue.trim().isEmpty() &&
                !"未提供".equals(fieldValue.trim()) && !"提取失败".equals(fieldValue.trim());
    }

    /**
     * 安全的休眠方法，正确处理InterruptedException
     */
    private void safeSleep(long millis) {
        try {
            Thread.sleep(millis);
        } catch (InterruptedException e) {
            // 恢复中断状态
            Thread.currentThread().interrupt();
            throw new RuntimeException("线程休眠被中断", e);
        }
    }

    // ===================== 内部类 =====================

    /**
     * 页面处理结果
     */
    private static class PageProcessResult {
        final boolean hasNewData;
        final boolean shouldStopCrawling;
        final String stopReason;

        PageProcessResult(boolean hasNewData, boolean shouldStopCrawling, String stopReason) {
            this.hasNewData = hasNewData;
            this.shouldStopCrawling = shouldStopCrawling;
            this.stopReason = stopReason;
        }
    }

    /**
     * 爬取条件
     */
    private static class CrawlConditions {
        final Date minRecallDate;
        final boolean hasExistingData;
        final int targetYear;

        CrawlConditions(Date minRecallDate, boolean hasExistingData, int targetYear) {
            this.minRecallDate = minRecallDate;
            this.hasExistingData = hasExistingData;
            this.targetYear = targetYear;
        }
    }
}