feat(task): 支持了培育中交流事件的处理

1. 引入了 CommuEventButtonUI 类，专门处理交流事件按钮
2. 加入了对培育开始时交流事件（选一张技能卡或 P 饮料）的处理
3. 调整授業处理逻辑，改为总是选择 +30 选项
4. device 引入强制截图参数

experiments/commu_event_buttons/commu_event_buttons.py

@@ -0,0 +1,134 @@
+import os
+from time import sleep
+import cv2
+from cv2.typing import MatLike
+import numpy as np
+from typing import NamedTuple
+from kotonebot.client.fast_screenshot import AdbFastScreenshots
+
+def path(file_name):
+    return os.path.join(os.path.dirname(__file__), file_name)
+
+def cv_imread(filePath):
+    cv_img=cv2.imdecode(np.fromfile(filePath,dtype=np.uint8),-1)
+    ## imdecode读取的是rgb，如果后续需要opencv处理的话，需要转换成bgr，转换后图片颜色会变化
+    ##cv_img=cv2.cvtColor(cv_img,cv2.COLOR_RGB2BGR)
+    return cv_img
+
+def cv_imshow(name, img, overlay_msg: str = ''):
+    scale = 0.5
+    if overlay_msg:
+        cv2.putText(img, overlay_msg, (10, img.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
+    cv2.imshow(name, cv2.resize(img, (int(img.shape[1] * scale), int(img.shape[0] * scale))))
+
+img = cv_imread(path(r".\0.png"))
+img = cv_imread(path(r".\1.png"))
+# img = cv_imread(r"E:\GithubRepos\KotonesAutoAssistant\screenshots\produce\action_study2.png")
+
+def button(img, include_colors = []):
+    # 转换到 HSV 颜色空间
+    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+
+    # 定义白色的 HSV 范围
+    # H: 色相接近0, S: 饱和度很低, V: 亮度较高
+    lower_white = np.array([0, 0, 200])  
+    upper_white = np.array([180, 30, 255])
+
+    # 创建白色掩码
+    white_mask = cv2.inRange(hsv, lower_white, upper_white)
+    
+    # 合并所有颜色的掩码
+    final_mask = white_mask
+    for color_range in include_colors:
+        # 创建该颜色的掩码并合并
+        color_mask = cv2.inRange(hsv, color_range[0], color_range[1])
+        final_mask = cv2.bitwise_or(final_mask, color_mask)
+        # final_mask = color_mask
+
+    # 应用掩码到原图
+    masked = cv2.bitwise_and(img, img, mask=final_mask)
+
+    # 显示结果
+    cv_imshow('Mask', final_mask)
+
+    # 寻找轮廓
+    contours, hierarchy = cv2.findContours(final_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    # 在原图上绘制轮廓的外接矩形并裁剪
+    result_img = img.copy()
+    cropped_imgs = []  # 存储裁剪后的图像
+    for contour in contours:
+        # 获取轮廓的外接矩形
+        x, y, w, h = cv2.boundingRect(contour)
+        # 计算宽高比和面积
+        aspect_ratio = w / h
+        area = cv2.contourArea(contour)
+        # 只处理宽高比>=7且面积>=500的矩形
+        if aspect_ratio >= 7 and area >= 500:
+            cv2.rectangle(result_img, (x, y), (x+w, y+h), (0, 255, 0), 2)
+            # 裁剪图像
+            cropped = img[y:y+h, x:x+w]
+            cropped_imgs.append(cropped)
+
+    from kotonebot.backend.ocr import jp
+
+    for btn in cropped_imgs:
+        print(jp.ocr(btn))
+
+    # 显示结果
+    cv_imshow('Bounding Boxes', result_img)
+
+def prompt_text(img):
+    # 转换到 HSV 颜色空间
+    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+
+    # 定义白色的 HSV 范围
+    # H: 色相接近0, S: 饱和度很低, V: 亮度较高
+    lower_white = np.array([0, 0, 200])  
+    upper_white = np.array([180, 30, 255])
+
+    # 创建掩码
+    white_mask = cv2.inRange(hsv, lower_white, upper_white)
+
+    # 应用掩码到原图
+    white_only = cv2.bitwise_and(img, img, mask=white_mask)
+
+    # 显示结果
+    cv_imshow('White Mask', white_mask)
+
+    # 寻找轮廓
+    contours, hierarchy = cv2.findContours(white_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    # 在原图上绘制轮廓的外接矩形并裁剪
+    result_img = img.copy()
+    cropped_imgs = []  # 存储裁剪后的图像
+    for contour in contours:
+        # 获取轮廓的外接矩形
+        x, y, w, h = cv2.boundingRect(contour)
+        # 计算宽高比和面积
+        aspect_ratio = w / h
+        area = cv2.contourArea(contour)
+        # 只处理宽高比>=7且面积>=500的矩形
+        if aspect_ratio >= 3 and area >= 100:
+            cv2.rectangle(result_img, (x, y), (x+w, y+h), (0, 255, 0), 2)
+            # 裁剪图像
+            cropped = img[y:y+h, x:x+w]
+            cropped_imgs.append(cropped)
+    
+    cv_imshow('Bounding Boxes', result_img)
+
+def web2cv(hsv):
+    return (int(hsv[0]/360*180), int(hsv[1]/100*255), int(hsv[2]/100*255))
+
+from kotonebot.backend.context import init_context, manual_context, device
+init_context()
+manual_context().begin()
+PINK_TARGET = (335, 78, 95)
+PINK_LOW = (300, 70, 90)
+PINK_HIGH = (350, 80, 100)
+button(device.screenshot(), [
+    (web2cv(PINK_LOW), web2cv(PINK_HIGH))
+])
+# prompt_text(device.screenshot())
+
+cv2.waitKey(0)

kotonebot/backend/context/context.py

@@ -643,11 +643,13 @@ class ContextDevice(Device):
     def __init__(self, device: Device):
         self._device = device
 
-    def screenshot(self):
+    def screenshot(self, *, force: bool = False):
         """
         截图。返回截图数据，同时更新当前上下文的截图数据。
         """
         current = ContextStackVars.ensure_current()
+        if force:
+            current._inherit_screenshot = None
         if current._inherit_screenshot is not None:
             img = current._inherit_screenshot
             current._inherit_screenshot = None

kotonebot/tasks/actions/common.py

@@ -1,6 +1,9 @@
 from typing import Literal
 from logging import getLogger
 
+from kotonebot.tasks.actions.loading import loading
+
+from .. import R
 from kotonebot import (
     ocr,
     device,
@@ -10,10 +13,10 @@ from kotonebot import (
     sleep,
     Interval,
 )
+from ..game_ui import CommuEventButtonUI
+from .pdorinku import acquire_pdorinku
 from kotonebot.backend.dispatch import SimpleDispatcher
 from kotonebot.tasks.actions.commu import check_and_skip_commu
-from .. import R
-from .pdorinku import acquire_pdorinku
 
 logger = getLogger(__name__)
 
@@ -32,24 +35,19 @@ def acquire_skill_card():
     logger.info(f"Found {len(cards)} skill cards")
     logger.debug("Click first skill card")
     device.click(cards[0].rect)
-    sleep(1)
-    # # 确定
-    # logger.debug("Click 受け取る")
-    # device.click(ocr.expect(contains("受け取る")).rect)
-    # # 跳过动画
-    # device.click(image.expect_wait_any([
-    #     R.InPurodyuusu.PSkillCardIconBlue,
-    #     R.InPurodyuusu.PSkillCardIconColorful
-    # ], timeout=60))
-    device.screenshot()
-    (SimpleDispatcher('acquire_skill_card')
-        .click(contains("受け取る"), finish=True,  log="Skill card #1 acquired")
-        # .click_any([
-        #     R.InPurodyuusu.PSkillCardIconBlue,
-        #     R.InPurodyuusu.PSkillCardIconColorful
-        # ], finish=True, log="Skill card #1 acquired")
-    ).run()
-    # logger.info("Skill card #1 acquired")
+    sleep(0.2)
+    logger.debug("Click acquire button")
+    device.click(image.expect_wait(R.InPurodyuusu.AcquireBtnDisabled))
+    # acquisitions(['PSkillCardSelect']) 优先做这个
+    # device.screenshot()
+    # (SimpleDispatcher('acquire_skill_card')
+    #     .click(contains("受け取る"), finish=True,  log="Skill card #1 acquired")
+    #     # .click_any([
+    #     #     R.InPurodyuusu.PSkillCardIconBlue,
+    #     #     R.InPurodyuusu.PSkillCardIconColorful
+    #     # ], finish=True, log="Skill card #1 acquired")
+    # ).run()
+    # # logger.info("Skill card #1 acquired")
 
 @action('选择P物品', screenshot_mode='auto')
 def select_p_item():
@@ -82,6 +80,7 @@ AcquisitionType = Literal[
     "Clear", # 目标达成
     "NetworkError", # 网络中断弹窗
     "SkipCommu", # 跳过交流
+    "Loading", # 加载画面
 ]
 
 @action('处理培育事件', screenshot_mode='manual')
@@ -93,6 +92,11 @@ def acquisitions() -> AcquisitionType | None:
     bottom_pos = (int(screen_size[0] * 0.5), int(screen_size[1] * 0.7)) # 底部中间
     logger.info("Acquisition stuffs...")
 
+    # 加载画面
+    logger.debug("Check loading screen...")
+    if loading():
+        return "Loading"
+
     # P饮料领取
     logger.debug("Check PDrink acquire...")
     if image.find(R.InPurodyuusu.PDrinkIcon):
@@ -217,6 +221,26 @@ def until_acquisition_clear():
     while acquisitions():
         interval.wait()
 
+@action('处理交流事件', screenshot_mode='manual-inherit')
+def commut_event():
+    img = device.screenshot()
+    ui = CommuEventButtonUI()
+    buttons = ui.all(description=False, title=True)
+    if buttons:
+        for button in buttons:
+            # 冲刺课程，跳过处理
+            if '重点' in button.title:
+                break
+        logger.info(f"Found commu event: {button.title}")
+        logger.info("Select first choice")
+        if buttons[0].selected:
+            device.click(buttons[0])
+        else:
+            device.double_click(buttons[0])
+        return True
+    return False
+    
+
 if __name__ == '__main__':
     from logging import getLogger
     import logging

kotonebot/tasks/actions/in_purodyuusu.py

@@ -12,7 +12,7 @@ from .. import R
 from . import loading
 from ..common import conf
 from .scenes import at_home
-from .common import until_acquisition_clear
+from .common import until_acquisition_clear, acquisitions, commut_event
 from kotonebot.errors import UnrecoverableError
 from kotonebot.backend.util import AdaptiveWait, Countdown, crop, cropped
 from kotonebot.backend.dispatch import DispatcherContext, SimpleDispatcher
@@ -351,7 +351,7 @@ def obtain_cards(img: MatLike | None = None):
 
 
 @action('等待进入行动场景')
-def until_action_scene():
+def until_action_scene(week_first: bool = False):
     """等待进入行动场景"""
     # 检测是否到行动页面
     while not image.find_multi([
@@ -359,7 +359,11 @@ def until_action_scene():
         R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
     ]):
         logger.info("Action scene not detected. Retry...")
-        until_acquisition_clear()
+        if acquisitions():
+            continue
+        if commut_event():
+            continue
+        sleep(0.2)
     else:
         logger.info("Now at action scene.")
         return 
@@ -684,10 +688,8 @@ def hajime_regular(week: int = -1, start_from: int = 1):
         week_final_lesson, # 12: 追い込みレッスン
         week_final_exam, # 13: 最終試験
     ]
-    if week not in [6, 13] and start_from not in [6, 13]:
-        until_action_scene()
-    else:
-        until_exam_scene()
+    if week == 0 or start_from == 0:
+        until_action_scene(True)
     if week != -1:
         logger.info("Week %d started.", week)
         weeks[week - 1]()
@@ -778,7 +780,10 @@ def detect_regular_produce_scene(ctx: DispatcherContext) -> ProduceStage:
         ctx.finish()
         return 'exam-start'
     else:
-        until_acquisition_clear()
+        if acquisitions():
+            return 'unknown'
+        if commut_event():
+            return 'unknown'
         return 'unknown'
 
 @action('开始 Regular 培育')

kotonebot/tasks/actions/non_lesson_actions.py

@@ -5,11 +5,12 @@
 """
 from logging import getLogger
 
-from kotonebot import device, image, ocr, debug, action, sleep
-from kotonebot.tasks import R
-from ..actions.loading import wait_loading_end, wait_loading_start
+from .. import R
+from ..game_ui import CommuEventButtonUI, EventButton
 from .common import acquisitions, AcquisitionType
-from .commu import check_and_skip_commu
+from kotonebot import device, image, ocr, debug, action, sleep
+from kotonebot.errors import UnrecoverableError
+from ..actions.loading import wait_loading_end, wait_loading_start
 
 logger = getLogger(__name__)
 
@@ -40,16 +41,26 @@ def enter_study():
     # [screenshots/produce/action_study1.png]
     logger.debug("Double clicking on 授業.")
     device.double_click(image.expect_wait(R.InPurodyuusu.ButtonIconStudy))
-    sleep(1.3)
     # 等待进入页面。中间可能会出现未读交流
     # [screenshots/produce/action_study2.png]
     while not image.find(R.InPurodyuusu.IconTitleStudy):
         logger.debug("Waiting for 授業 screen.")
-        check_and_skip_commu()
         acquisitions()
-    # 固定点击 Vi. 选项
-    logger.debug("Clicking on Vi. option.")
-    device.double_click(image.expect_wait(R.InPurodyuusu.ButtonIconStudyVisual))
+    # 获取三个选项的内容
+    ui = CommuEventButtonUI()
+    buttons = ui.all()
+    if not buttons:
+        raise UnrecoverableError("Failed to find any buttons.")
+    # 选中 +30 的选项
+    target_btn = next((btn for btn in buttons if '+30' in btn.description), None)
+    if target_btn is None:
+        logger.error("Failed to find +30 option. Pick the first button instead.")
+        target_btn = buttons[0]
+    logger.debug('Clicking "%s".', target_btn.description)
+    if target_btn.selected:
+        device.click(target_btn)
+    else:
+        device.double_click(target_btn)
     while acquisitions() is None:
         logger.info("Waiting for acquisitions finished.")
     logger.info("授業 completed.")
@@ -110,3 +121,27 @@ def rest():
     device.click(image.expect_wait(R.InPurodyuusu.Rest))
     # 确定
     device.click(image.expect_wait(R.InPurodyuusu.RestConfirmBtn))
+
+if __name__ == '__main__':
+    from kotonebot.backend.context import manual_context, init_context
+    init_context()
+    manual_context().begin()
+    # 获取三个选项的内容
+    ui = CommuEventButtonUI()
+    buttons = ui.all()
+    if not buttons:
+        raise UnrecoverableError("Failed to find any buttons.")
+    # 选中 +30 的选项
+    target_btn = next((btn for btn in buttons if btn.description == '+30'), None)
+    if target_btn is None:
+        logger.error("Failed to find +30 option. Pick the first button instead.")
+        target_btn = buttons[0]
+    # 固定点击 Vi. 选项
+    logger.debug('Clicking "%s".', target_btn.description)
+    if target_btn.selected:
+        device.click(target_btn)
+    else:
+        device.double_click(target_btn)
+    while acquisitions() is None:
+        logger.info("Waiting for acquisitions finished.")
+    logger.info("授業 completed.")

kotonebot/tasks/game_ui.py

@@ -1,9 +1,16 @@
-from typing import Literal
-from cv2.typing import MatLike
+from dataclasses import dataclass
+from typing import Literal, NamedTuple
 
-from kotonebot import action, device, color, image
+import numpy as np
+
+from . import R
+from kotonebot import action, device, color, image, ocr, sleep
+from kotonebot.backend.color import HsvColor
 from kotonebot.backend.util import Rect
-from kotonebot.backend.core import Image
+from kotonebot.backend.core import HintBox, Image
+
+import cv2
+from cv2.typing import MatLike
 
 @action('按钮是否禁用', screenshot_mode='manual-inherit')
 def button_state(*, target: Image | None = None, rect: Rect | None = None) -> bool | None:
@@ -29,3 +36,157 @@ def button_state(*, target: Image | None = None, rect: Rect | None = None) -> bo
         return True
     else:
         raise ValueError(f'Unknown button state: {img}')
+
+def web2cv(hsv: HsvColor):
+    return (int(hsv[0]/360*180), int(hsv[1]/100*255), int(hsv[2]/100*255))
+
+WHITE_LOW = (0, 0, 200)
+WHITE_HIGH = (180, 30, 255)
+
+PINK_TARGET = (335, 78, 95)
+PINK_LOW = (300, 70, 90)
+PINK_HIGH = (350, 80, 100)
+
+BLUE_TARGET = (210, 88, 93)
+BLUE_LOW = (200, 80, 90)
+BLUE_HIGH = (220, 90, 100)
+
+YELLOW_TARGET = (39, 81, 97)
+YELLOW_LOW = (30, 70, 90)
+YELLOW_HIGH = (45, 90, 100)
+
+DEFAULT_COLORS = [
+    (web2cv(PINK_LOW), web2cv(PINK_HIGH)),
+    (web2cv(YELLOW_LOW), web2cv(YELLOW_HIGH)),
+    (web2cv(BLUE_LOW), web2cv(BLUE_HIGH)),
+]
+
+def filter_rectangles(
+    img: MatLike,
+    color_ranges: tuple[HsvColor, HsvColor],
+    aspect_ratio_threshold: float,
+    area_threshold: int,
+    rect: Rect | None = None
+) -> list[Rect]:
+    """
+    过滤出指定颜色，并执行轮廓查找，返回符合要求的轮廓的 bound box。
+    返回结果按照 y 坐标排序。
+    """
+    img_hsv =cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+
+    white_mask = cv2.inRange(img_hsv, np.array(color_ranges[0]), np.array(color_ranges[1]))
+    contours, _ = cv2.findContours(white_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    result_rects = []
+    for contour in contours:
+        x, y, w, h = cv2.boundingRect(contour)
+        # 如果不在指定范围内，跳过
+        if rect is not None:
+            rect_x1, rect_y1, rect_w, rect_h = rect
+            rect_x2 = rect_x1 + rect_w
+            rect_y2 = rect_y1 + rect_h
+            if not (
+                x >= rect_x1 and
+                y >= rect_y1 and
+                x + w <= rect_x2 and
+                y + h <= rect_y2
+            ):
+                continue
+        aspect_ratio = w / h
+        area = cv2.contourArea(contour)
+        if aspect_ratio >= aspect_ratio_threshold and area >= area_threshold:
+            result_rects.append((x, y, w, h))
+    result_rects.sort(key=lambda x: x[1])
+    return result_rects
+
+@dataclass
+class EventButton:
+    rect: Rect
+    selected: bool
+    description: str
+    title: str
+
+# 参考图片：
+# [screenshots/produce/action_study3.png]
+class CommuEventButtonUI:
+    def __init__(
+        self,
+        selected_colors: list[tuple[HsvColor, HsvColor]] = DEFAULT_COLORS,
+        rect: HintBox = R.InPurodyuusu.BoxCommuEventButtonsArea
+    ):
+        """
+        :param selected_colors: 按钮选中后的主题色。
+        :param rect: 识别范围
+        """
+        self.color_ranges = selected_colors
+        self.rect = rect
+
+    @action('交流事件按钮.识别选中', screenshot_mode='manual-inherit')
+    def selected(self, description: bool = True, title: bool = False) -> EventButton | None:
+        img = device.screenshot()
+        for i, color_range in enumerate(self.color_ranges):
+            rects = filter_rectangles(img, color_range, 7, 500, rect=self.rect)
+            if len(rects) > 0:
+                desc_text = self.description() if description else ''
+                title_text = ocr.ocr(rect=rects[0]).squash().text if title else ''
+                return EventButton(rects[0], True, desc_text, title_text)
+        return None
+
+    @action('交流事件按钮.识别按钮', screenshot_mode='manual-inherit')
+    def all(self, description: bool = True, title: bool = False) -> list[EventButton]:
+        """
+        识别所有按钮的位置以及选中后的描述文本
+
+        前置条件：当前显示了交流事件按钮\n
+        结束状态：-
+
+        :param description: 是否识别描述文本。
+        :param title: 是否识别标题。
+        """
+        img = device.screenshot()
+        rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 7, 500, rect=self.rect)
+        selected = self.selected()
+        result: list[EventButton] = []
+        for rect in rects:
+            desc_text = ''
+            title_text = ''
+            if title:
+                title_text = ocr.ocr(rect=rect).squash().text
+            if description:
+                device.click(rect)
+                sleep(0.15)
+                device.screenshot()
+                desc_text = self.description()
+            result.append(EventButton(rect, False, desc_text, title_text))
+        # 修改最后一次点击的按钮为 selected 状态
+        if len(result) > 0:
+            result[-1].selected = True
+        if selected is not None:
+            result.append(selected)
+            selected.selected = False
+        result.sort(key=lambda x: x.rect[1])
+        return result
+
+    @action('交流事件按钮.识别描述', screenshot_mode='manual-inherit')
+    def description(self) -> str:
+        """
+        识别当前选中按钮的描述文本
+
+        前置条件：有选中按钮\n
+        结束状态：-
+        """
+        img = device.screenshot()
+        rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 3, 1000, rect=self.rect)
+        rects.sort(key=lambda x: x[1])
+        ocr_result = ocr.raw().ocr(img, rect=rects[0])
+        return ocr_result.squash().text
+    
+
+if __name__ == '__main__':
+    from pprint import pprint as print
+    from kotonebot.backend.context import init_context, manual_context, device
+    init_context()
+    manual_context().begin()
+    btn = CommuEventButtonUI()
+    # print(btn.selected())
+    # print(btn.all())
+    print(btn.all())

kotonebot/tasks/produce.py

@@ -204,13 +204,6 @@ def produce_task(count: Optional[int] = None, idols: Optional[list[PIdol]] = Non
         end_time = time.time()
         logger.info(f"Produce time used: {format_time(end_time - start_time)}")
 
-@action('测试')
-def a():
-    ocr.expect_wait(contains('メモリー'), rect=R.Produce.BoxStepIndicator)
-    device.click(image.expect_wait(R.Produce.ButtonAutoSet))
-    device.click(image.expect_wait(R.Common.ButtonConfirm, colored=True))
-    device.click(image.expect_wait(R.Common.ButtonNextNoIcon))
-
 if __name__ == '__main__':
     import logging
     logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] [%(funcName)s] [%(lineno)d] %(message)s')

res/sprites/jp/in_purodyuusu/screenshot_study.png.json

@@ -0,0 +1 @@
+{"definitions":{"f29b6663-1f88-477f-bb7a-8e0e77ad9138":{"name":"InPurodyuusu.BoxCommuEventButtonsArea","displayName":"交流事件按钮区域","type":"hint-box","annotationId":"f29b6663-1f88-477f-bb7a-8e0e77ad9138","useHintRect":false}},"annotations":[{"id":"f29b6663-1f88-477f-bb7a-8e0e77ad9138","type":"rect","data":{"x1":14,"y1":412,"x2":703,"y2":1089}}]}