feat(task): 培育中对推荐行动的识别从 OCR 改为模板匹配

kotonebot/backend/context.py

@@ -12,7 +12,7 @@ from cv2.typing import MatLike
 
 
 import kotonebot.backend.image as raw_image
-from kotonebot.backend.image import CropResult, TemplateMatchResult, find_crop, expect, find, find_any
+from kotonebot.backend.image import CropResult, TemplateMatchResult, MultipleTemplateMatchResult, find_crop, expect, find, find_any
 from kotonebot.backend.util import Rect
 from kotonebot.client import DeviceABC
 from kotonebot.backend.ocr import Ocr, OcrResult, jp, en, StringMatchFunction
@@ -224,7 +224,7 @@ class ContextImage:
             templates: list[str | MatLike],
             masks: list[str | MatLike | None] | None = None,
             threshold: float = 0.9
-        ) -> TemplateMatchResult | None:
+        ) -> MultipleTemplateMatchResult | None:
         """
         寻找指定图像中的任意一个。
         """

kotonebot/backend/image.py

@@ -1,5 +1,5 @@
 import os
-from typing import NamedTuple, Protocol, TypeVar
+from typing import NamedTuple, Protocol, TypeVar, Sequence, runtime_checkable
 from logging import getLogger
 
 from .debug import result, debug, img
@@ -17,10 +17,12 @@ class TemplateNotFoundError(Exception):
         self.template = template
         super().__init__(f"Template not found: {template}")
 
-
+@runtime_checkable
 class ResultProtocol(Protocol):
-    score: float
-    position: Point
+    @property
+    def rect(self) -> Rect:
+        """结果区域。左上角坐标和宽高。"""
+        ...
 
 
 class TemplateMatchResult(NamedTuple):
@@ -40,6 +42,25 @@ class TemplateMatchResult(NamedTuple):
         """结果右下角坐标。"""
         return (self.position[0] + self.size[0], self.position[1] + self.size[1])
 
+class MultipleTemplateMatchResult(NamedTuple):
+    score: float
+    position: Point
+    """结果位置。左上角坐标。"""
+    size: Size
+    """命中模板的大小。宽高。"""
+    index: int
+    """命中模板在列表中的索引。"""
+
+    @property
+    def rect(self) -> Rect:
+        """结果区域。左上角坐标和宽高。"""
+        return (self.position[0], self.position[1], self.size[0], self.size[1])
+    
+    @property
+    def right_bottom(self) -> Point:
+        """结果右下角坐标。"""
+        return (self.position[0] + self.size[0], self.position[1] + self.size[1])
+
 class CropResult(NamedTuple):
     score: float
     position: Point
@@ -69,10 +90,10 @@ def _remove_duplicate_matches(
         result.append(match)
     return result
 
-def _draw_result(image: MatLike, matches: list[TemplateMatchResult] | TemplateMatchResult | None) -> MatLike:
+def _draw_result(image: MatLike, matches: Sequence[ResultProtocol] | ResultProtocol | None) -> MatLike:
     if matches is None:
         return image
-    if isinstance(matches, TemplateMatchResult):
+    if isinstance(matches, ResultProtocol):
         matches = [matches]
     result_image = image.copy()
     for match in matches:
@@ -94,8 +115,6 @@ def template_match(
     .. note::
         `mask` 和 `transparent` 参数不能同时使用。
 
-    <img src="vscode-file://vscode-app/E:/GithubRepos/KotonesAutoAssistant/original.png" width="100">
-
     :param template: 模板图像，可以是图像路径或 cv2.Mat。
     :param image: 图像，可以是图像路径或 cv2.Mat。
     :param mask: 掩码图像，可以是图像路径或 cv2.Mat。
@@ -186,17 +205,23 @@ def find_any(
     masks: list[MatLike | str | None] | None = None,
     transparent: bool = False,
     threshold: float = 0.8,
-) -> TemplateMatchResult | None:
+) -> MultipleTemplateMatchResult | None:
     """指定多个模板，返回第一个匹配到的结果"""
     ret = None
     if masks is None:
         _masks = [None] * len(templates)
     else:
         _masks = masks
-    for template, mask in zip(templates, _masks):
-        ret = find(image, template, mask, transparent, threshold, debug_output=False)
+    for index, (template, mask) in enumerate(zip(templates, _masks)):
+        find_result = find(image, template, mask, transparent, threshold, debug_output=False)
         # 调试输出
-        if ret is not None:
+        if find_result is not None:
+            ret = MultipleTemplateMatchResult(
+                score=find_result.score,
+                position=find_result.position,
+                size=find_result.size,
+                index=index
+            )
             break
     if debug.enabled:
         msg = (

kotonebot/backend/ocr.py

@@ -4,7 +4,7 @@ import unicodedata
 from os import PathLike
 from typing import TYPE_CHECKING, Callable, NamedTuple, overload
 
-from .util import Rect
+from .util import Rect, grayscaled
 from .debug import result as debug_result, debug
 
 import cv2
@@ -121,7 +121,7 @@ class Ocr:
 
         :return: 所有识别结果
         """
-        img_content = img
+        img_content = grayscaled(img)
         result, elapse = self.__engine(img_content)
         if result is None:
             return []

kotonebot/tasks/actions/in_purodyuusu.py

@@ -18,7 +18,8 @@ from .common import acquisitions, AcquisitionType, acquire_skill_card
 logger = logging.getLogger(__name__)
 
 ActionType = None | Literal['lesson', 'rest']
-def enter_recommended_action(final_week: bool = False) -> ActionType:
+@deprecated('OCR 方法效果不佳')
+def enter_recommended_action_ocr(final_week: bool = False) -> ActionType:
     """
     在行动选择页面，执行推荐行动
 
@@ -63,6 +64,57 @@ def enter_recommended_action(final_week: bool = False) -> ActionType:
         device.double_click(image.expect_wait(template))
         return 'lesson'
 
+def enter_recommended_action(final_week: bool = False) -> ActionType:
+    """
+    在行动选择页面，执行推荐行动
+
+    :param final_week: 是否是考试前复习周
+    :return: 是否成功执行推荐行动
+    """
+    # 获取课程
+    logger.debug("Getting recommended lesson...")
+    with device.hook(cropper_y(0.00, 0.30)):
+        result = image.find_any([
+            R.InPurodyuusu.TextSenseiTipDance,
+            R.InPurodyuusu.TextSenseiTipVocal,
+            R.InPurodyuusu.TextSenseiTipVisual,
+            # R.InPurodyuusu.TextSenseiTipRest, # TODO: 体力提示截图
+        ])
+    logger.debug("ocr.wait_for: %s", result)
+    if result is None:
+        logger.debug("No recommended lesson found")
+        return None
+    if not final_week:
+        if result.index == 0:
+            lesson_text = "Vo"
+        elif result.index == 1:
+            lesson_text = "Da"
+        elif result.index == 2:
+            lesson_text = "Vi"
+        elif result.index == 3:
+            rest()
+            return 'rest'
+        else:
+            return None
+        logger.info("Rec. lesson: %s", lesson_text)
+        # 点击课程
+        logger.debug("Try clicking lesson...")
+        lesson_ret = ocr.expect(contains(lesson_text))
+        device.double_click(lesson_ret.rect)
+        return 'lesson'
+    else:
+        if result.index == 0:
+            template = R.InPurodyuusu.ButtonFinalPracticeVocal
+        elif result.index == 1:
+            template = R.InPurodyuusu.ButtonFinalPracticeDance
+        elif result.index == 2:
+            template = R.InPurodyuusu.ButtonFinalPracticeVisual
+        else:
+            return None
+        logger.debug("Try clicking lesson...")
+        device.double_click(image.expect_wait(template))
+        return 'lesson'
+
 def before_start_action():
     """检测支援卡剧情、领取资源等"""
     raise NotImplementedError()
@@ -395,13 +447,14 @@ def exam():
 
 def produce_end():
     """执行考试结束"""
-    # 考试结束对话
+    # 考试结束对话 [screenshots\produce_end\step2.jpg]
     image.expect_wait(R.InPurodyuusu.TextAsariProduceEnd, timeout=30)
     bottom = (int(device.screen_size[0] / 2), int(device.screen_size[1] * 0.9))
     device.click(*bottom)
+    # 对话第二句 [screenshots\produce_end\step3.jpg]
     sleep(3)
     device.click_center()
-    sleep(3)
+    sleep(6)
     device.click(*bottom)
     sleep(3)
     device.click(*bottom)
@@ -412,7 +465,7 @@ def produce_end():
 
     # 结算
     # 最終プロデュース評価
-    image.expect_wait(R.InPurodyuusu.TextFinalProduceRating, timeout=60 * 2.5)
+    image.expect_wait(R.InPurodyuusu.TextFinalProduceRating, timeout=60 * 4)
     device.click_center()
     sleep(3)
     # 次へ
@@ -687,7 +740,7 @@ if __name__ == '__main__':
     # acquisitions()
     # acquire_pdorinku(0)
     # image.wait_for(R.InPurodyuusu.InPractice.PDorinkuIcon)
-    hajime_regular(start_from=9)
+    hajime_regular(start_from=12)
     # until_practice_scene()
     # device.click(image.expect_wait_any([
     #     R.InPurodyuusu.PSkillCardIconBlue,