feat(*): 优化首页按钮检测 & 修复商店购买 bug

1. 优化工具栏首页按钮检测
2. 修复商店购买中由于截图未更新导致的无法自动确认购买提示

kotonebot/backend/core.py

@@ -1,6 +1,6 @@
 import logging
-
-from typing import Callable, ParamSpec, TypeVar, overload, TYPE_CHECKING
+from functools import cache
+from typing import Callable, overload, TYPE_CHECKING
 
 import cv2
 from cv2.typing import MatLike
@@ -20,7 +20,6 @@ class Ocr:
         self.language = language
 
 
-# TODO: 支持透明背景
 class Image:
     def __init__(
         self,
@@ -34,6 +33,10 @@ class Image:
         self.__data: MatLike | None = data
         self.__data_with_alpha: MatLike | None = None
 
+    @cache
+    def binary(self) -> 'Image':
+        return Image(data=cv2.cvtColor(self.data, cv2.COLOR_BGR2GRAY))
+
     @property
     def data(self) -> MatLike:
         if self.__data is None:

kotonebot/backend/image.py

@@ -176,12 +176,6 @@ def template_match(
         # 从透明图像中提取 alpha 通道作为 mask
         mask = cv2.threshold(template[:, :, 3], 0, 255, cv2.THRESH_BINARY)[1]
         template = template[:, :, :3]
-    if (transparent or mask is not None) and threshold < 0.999:
-        logger.warning(
-            f'For template matching with transparent image or mask, '
-            'threshold is recommended to be set to 0.999 or higher. '
-            f'Got {threshold}'
-        )
     # 匹配模板
     if mask is not None:
         # https://stackoverflow.com/questions/35642497/python-opencv-cv2-matchtemplate-with-transparency

kotonebot/tasks/actions/scenes.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Callable
 
+
 from .. import R
 from .loading import loading
+from ..game_ui import toolbar_home
 from kotonebot import device, image, action, cropped, until, sleep
 from kotonebot.errors import UnrecoverableError
 
@@ -39,15 +41,10 @@ def goto_home():
     """
     logger.info("Going home.")
     with cropped(device, y1=0.7):
-        if image.find(
-            R.Common.ButtonToolbarHome,
-            transparent=True,
-            threshold=0.9999,
-            colored=True
-        ):
-            device.click()
+        if home := toolbar_home():
+            device.click(home)
             while loading():
-                sleep(0.5)
+                pass
         elif image.find(R.Common.ButtonHome):
             device.click()
         else:

kotonebot/tasks/game_ui.py

@@ -181,14 +181,25 @@ class CommuEventButtonUI:
         rects.sort(key=lambda x: x[1])
         ocr_result = ocr.raw().ocr(img, rect=rects[0])
         return ocr_result.squash().text
-    
+
+def filter_white(img: MatLike):
+    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+    lower_white = np.array([0, 0, 180])
+    upper_white = np.array([180, 50, 255])
+    return cv2.inRange(hsv, lower_white, upper_white)
+
+# TODO: image 对象加入自定义 hook，处理 post-process 和 pre-process
+@action('工具栏按钮.寻找首页', screenshot_mode='manual-inherit')
+def toolbar_home():
+    img = device.screenshot()
+    img = filter_white(img)
+    result = image.raw().find(img, R.Common.ButtonToolbarHomeBinary.binary())
+    return result
+
 
 if __name__ == '__main__':
     from pprint import pprint as print
     from kotonebot.backend.context import init_context, manual_context, device
     init_context()
     manual_context().begin()
-    btn = CommuEventButtonUI()
-    # print(btn.selected())
-    # print(btn.all())
-    print(btn.all())
+    print(toolbar_home())

kotonebot/tasks/purchase.py

@@ -89,10 +89,12 @@ def dispatch_purchase_dialog(ctx: DispatcherContext):
     结束状态：对话框关闭后原来的界面
     """
     # 前置条件：[screenshots\shop\dialog.png]
+    device.screenshot()
     if image.find(R.Daily.ButtonShopCountAdd, colored=True):
         logger.debug('Adjusting quantity(+1)...')
         device.click()
     elif image.find(R.Common.ButtonConfirm):
+        sleep(0.1)
         logger.debug('Confirming purchase...')
         device.click()
         ctx.finish()
@@ -150,7 +152,6 @@ def purchase():
     if not at_daily_shop():
         goto_shop()
     # 进入每日商店 [screenshots\shop\shop.png]
-    # [ap1.png]
     device.click(image.expect(R.Daily.ButtonDailyShop)) # TODO: memoable
     sleep(1)
 

res/sprites/jp/common/screenshot_binary_shop.png.json

@@ -0,0 +1 @@
+{"definitions":{"2f68e457-4491-4ebc-b436-e3e63db92e14":{"name":"Common.ButtonToolbarHomeBinary","displayName":"底部工具栏 首页按钮 二值化图片","type":"template","annotationId":"2f68e457-4491-4ebc-b436-e3e63db92e14","useHintRect":false},"bf6dcf21-2ffc-441f-82f3-59319f2313b1":{"name":"Common.ButtonToolbarMenuBinary","displayName":"底部工具栏 菜单按钮 二值化图片","type":"template","annotationId":"bf6dcf21-2ffc-441f-82f3-59319f2313b1","useHintRect":false}},"annotations":[{"id":"2f68e457-4491-4ebc-b436-e3e63db92e14","type":"rect","data":{"x1":59,"y1":1188,"x2":112,"y2":1240}},{"id":"bf6dcf21-2ffc-441f-82f3-59319f2313b1","type":"rect","data":{"x1":631,"y1":1194,"x2":674,"y2":1232}}]}