feat(task): 培育任务支持中断继续 & AP、钻石信息获取

1. 培育任务支持了从原先培育进度继续
2. 新增获取当前 AP 和钻石信息
3. Protocol 类支持直接点击 HintBox 对象

kotonebot/backend/ocr.py

@@ -37,6 +37,9 @@ class OcrResult(NamedTuple):
     rect: Rect
     confidence: float
 
+    def __repr__(self) -> str:
+        return f'OcrResult(text="{self.text}", rect={self.rect}, confidence={self.confidence})'
+
     def regex(self, pattern: re.Pattern | str) -> list[str]:
         """
         提取识别结果中符合正则表达式的文本。
@@ -401,35 +404,4 @@ en = Ocr(_engine_en)
 
 
 if __name__ == '__main__':
-    import time
-    from pprint import pprint as print
-    import cv2
-    print('small')
-    img_path = r"C:\Users\user\Downloads\Screenshot_2025.01.28_21.00.40.172.png"
-    img = cv2.imread(img_path)
-    time_start = time.time()
-    result1 = jp.ocr(img)
-    time_end = time.time()
-    print(time_end - time_start)
-    # print(result1)
-
-    for i in np.linspace(300, 1000, 20):
-        i = int(i)
-        print('small-pad: ' + f'{str(i)}x{str(i)}')
-        img = pad_to(img, (int(i), int(i)))
-        time_start = time.time()
-        result1 = jp.ocr(img)
-        time_end = time.time()
-        print(time_end - time_start)
-
-        # print(result1)
-
-
-    print('big')
-    img_path = r"C:\Users\user\Pictures\BlueStacks\Screenshot_2025.01.28_21.00.40.172.png"
-    img = cv2.imread(img_path)
-    time_start = time.time()
-    result1 = jp.ocr(img)
-    time_end = time.time()
-    print(time_end - time_start)
-    # print(result1)
+    pass

kotonebot/client/device/adb.py

@@ -8,6 +8,7 @@ from cv2.typing import MatLike
 from adbutils import AdbClient, adb
 from adbutils._device import AdbDevice as Device
 
+from kotonebot.backend.core import HintBox
 from kotonebot.backend.util import Rect, is_rect
 from ..protocol import DeviceABC, ClickableObjectProtocol
 
@@ -27,6 +28,8 @@ class AdbDevice(DeviceABC):
     def click(self, arg1=None, arg2=None) -> None:
         if arg1 is None:
             self.__click_last()
+        elif isinstance(arg1, HintBox):
+            self.__click_hint_box(arg1)
         elif is_rect(arg1):
             self.__click_rect(arg1)
         elif isinstance(arg1, int) and isinstance(arg2, int):
@@ -60,8 +63,12 @@ class AdbDevice(DeviceABC):
     def __click_clickable(self, clickable: ClickableObjectProtocol) -> None:
         self.click(clickable.rect)
 
+    def __click_hint_box(self, hint_box: HintBox) -> None:
+        self.click(hint_box.rect)
+
     @override
     def swipe(self, x1: int, y1: int, x2: int, y2: int, duration: float|None = None) -> None:
+
         if duration is not None:
             logger.warning("Swipe duration is not supported with AdbDevice. Ignoring duration.")
         self.device.shell(f"input touchscreen swipe {x1} {y1} {x2} {y2}")

kotonebot/client/protocol.py

@@ -5,6 +5,7 @@ from typing_extensions import deprecated
 
 from cv2.typing import MatLike
 
+from kotonebot.backend.core import HintBox
 from kotonebot.backend.util import Rect, is_rect
 
 @runtime_checkable
@@ -109,6 +110,13 @@ class DeviceABC(ABC):
         """
         ...
 
+    @overload
+    def click(self, hint_box: HintBox) -> None:
+        """
+        点击屏幕上的某个矩形区域
+        """
+        ...
+
     @overload
     def click(self, rect: Rect) -> None:
         """
@@ -118,6 +126,7 @@ class DeviceABC(ABC):
 
     @overload
     def click(self, clickable: ClickableObjectProtocol) -> None:
+
         """
         点击屏幕上的某个可点击对象
         """

kotonebot/tasks/actions/in_purodyuusu.py

@@ -15,7 +15,10 @@ from ..common import conf
 from kotonebot.backend.dispatch import DispatcherContext
 from kotonebot.backend.util import AdaptiveWait, UnrecoverableError, crop, cropped
 from kotonebot import ocr, device, contains, image, regex, action, debug, config, sleep
-from .non_lesson_actions import enter_allowance, allowance_available, study_available, enter_study
+from .non_lesson_actions import (
+    enter_allowance, allowance_available, study_available, enter_study,
+    is_rest_available, rest
+)
 
 logger = logging.getLogger(__name__)
 
@@ -339,16 +342,6 @@ def remaing_turns_and_points():
     logger.debug("turns_ocr: %s", turns_ocr)
 
 
-@action('执行休息')
-def rest():
-    """执行休息"""
-    logger.info("Rest for this week.")
-    # 点击休息
-    device.click(image.expect_wait(R.InPurodyuusu.Rest))
-    # 确定
-    device.click(image.expect_wait(R.InPurodyuusu.RestConfirmBtn))
-
-
 @action('等待进入行动场景')
 def until_action_scene():
     """等待进入行动场景"""
@@ -570,6 +563,8 @@ def hajime_regular(week: int = -1, start_from: int = 1):
             enter_allowance()
         elif study_available():
             enter_study()
+        elif is_rest_available():
+            rest()
         else:
             raise ValueError("No action available.")
         until_action_scene()
@@ -646,9 +641,9 @@ ProduceStage = Literal[
     'exam-end', # 考试结束
     'unknown', # 未知场景
 ]
-@action('检测培育阶段并开始培育', dispatcher=True)
 
-def detect_regular_produce_stage(ctx: DispatcherContext) -> ProduceStage:
+@action('检测当前培育场景', dispatcher=True)
+def detect_regular_produce_scene(ctx: DispatcherContext) -> ProduceStage:
     """
     判断当前是培育的什么阶段，并开始 Regular 培育。
 
@@ -706,6 +701,15 @@ def hajime_regular_from_stage(stage: ProduceStage):
         logger.info("Exam ongoing. Start exam.")
         exam()
         produce_end()
+    else:
+        raise UnrecoverableError(f'Cannot resume produce REGULAR from stage "{stage}".')
+
+@action('继续 Regular 培育')
+def resume_regular_produce():
+    """
+    继续 Regular 培育。
+    """
+    hajime_regular_from_stage(detect_regular_produce_scene())
 
 if __name__ == '__main__':
     from logging import getLogger
@@ -714,8 +718,10 @@ if __name__ == '__main__':
     getLogger('kotonebot').setLevel(logging.DEBUG)
     getLogger(__name__).setLevel(logging.DEBUG)
 
-    # stage = (detect_regular_produce_stage())
+    # stage = (detect_regular_produce_scene())
     # hajime_regular_from_stage(stage)
 
     # click_recommended_card(card_count=skill_card_count())
     # exam()
+
+    hajime_regular(start_from=7)

kotonebot/tasks/actions/non_lesson_actions.py

@@ -97,7 +97,19 @@ def enter_allowance():
     # 可能会出现的新动画
     # 技能卡：[screenshots\allowance\step_4.png]
 
+@action('判断是否可以休息')
+def is_rest_available():
+    """
+    判断是否可以休息。
+    """
+    return image.find(R.InPurodyuusu.Rest) is not None
 
-def study():
-    """授業"""
-    pass
+
+@action('执行休息')
+def rest():
+    """执行休息"""
+    logger.info("Rest for this week.")
+    # 点击休息
+    device.click(image.expect_wait(R.InPurodyuusu.Rest))
+    # 确定
+    device.click(image.expect_wait(R.InPurodyuusu.RestConfirmBtn))

kotonebot/tasks/actions/stats.py

@@ -0,0 +1,51 @@
+import logging
+from typing import NamedTuple
+from datetime import timedelta
+from .. import R
+from kotonebot import action, ocr, device, regex
+
+logger = logging.getLogger(__name__)
+
+class AP(NamedTuple):
+    current: int
+    total: int
+    next_refresh: timedelta
+
+    def __repr__(self):
+        return f'AP({self.current}/{self.total} {self.next_refresh})'
+
+@action('获取当前 AP')
+def ap() -> AP | None:
+    texts = ocr.ocr(rect=R.Daily.BoxHomeAP)
+    logger.info(f'BoxHomeAP ocr result: {texts}')
+    # 当前 AP 和总 AP
+    ap = texts.where(regex(r'\d+/\d+')).first()
+    if not ap:
+        logger.warning('AP not found.')
+        return None
+    current, total = ap.numbers()
+    # 下一次刷新时间
+    next_refresh = texts.where(regex(r'\d+:\d+')).first()
+    if not next_refresh:
+        logger.warning('Next refresh time not found.')
+        return None
+    next_refresh = timedelta(minutes=next_refresh.numbers()[0], seconds=next_refresh.numbers()[1])
+    return AP(current=current, total=total, next_refresh=next_refresh)
+
+@action('获取当前宝石')
+def jewel() -> int | None:
+    jewel = ocr.find(regex(r'[\d,]+'), rect=R.Daily.BoxHomeJewel)
+    logger.info(f'BoxHomeJewel find result: {jewel}')
+    if not jewel:
+        logger.warning('Jewel not found.')
+        return None
+    return int(jewel.text.replace(',', '').replace('+', ''))
+
+
+if __name__ == '__main__':
+    import logging
+    logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] [%(funcName)s] [%(lineno)d] %(message)s')
+    logging.getLogger('kotonebot').setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+    print(ap())
+    print(jewel())

kotonebot/tasks/assignment.py

@@ -1,12 +1,13 @@
 """工作。お仕事"""
 import logging
 from typing import Literal
+from datetime import timedelta
 
 from . import R
 from .common import conf
 from .actions.loading import wait_loading_end
 from .actions.scenes import at_home, goto_home
-from kotonebot import task, device, image, action, ocr, contains, cropped, rect_expand, color, sleep
+from kotonebot import task, device, image, action, ocr, contains, cropped, rect_expand, color, sleep, regex
 
 logger = logging.getLogger(__name__)
 
@@ -104,6 +105,26 @@ def assign(type: Literal['mini', 'online']) -> bool:
     device.click(image.expect_wait(R.Common.ButtonStart, timeout=5))
     return True
 
+@action('获取剩余时间')
+def get_remaining_time() -> timedelta | None:
+    """
+    获取剩余时间
+
+    前置条件：首页 \n
+    结束状态：-
+    """
+    texts = ocr.ocr(rect=R.Daily.BoxHomeAssignment)
+    if not texts.where(contains('お仕事')):
+        logger.warning('お仕事 area not found.')
+        return None
+    time = texts.where(regex(r'\d+:\d+:\d+')).first()
+    if not time:
+        logger.warning('お仕事 remaining time not found.')
+        return None
+    logger.info(f'お仕事 remaining time: {time}')
+    return timedelta(hours=time.numbers()[0], minutes=time.numbers()[1], seconds=time.numbers()[2])
+
+
 @task('工作')
 def assignment():
     """领取工作奖励并重新分配工作"""
@@ -122,6 +143,7 @@ def assignment():
         notification_dot = color.find_rgb('#ff134a', rect=notification_rect)
         if not notification_dot and not completed:
             logger.info('No action needed.')
+            # TODO: 获取剩余时间，并根据时间更新调度
             return
 
     # 点击工作按钮
@@ -152,4 +174,5 @@ if __name__ == '__main__':
     import logging
     logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] [%(funcName)s] [%(lineno)d] %(message)s')
     logger.setLevel(logging.DEBUG)
-    assignment()
+    # assignment()
+    print(get_remaining_time())

kotonebot/tasks/produce.py

@@ -8,7 +8,7 @@ from . import R
 from .common import conf, PIdol
 from .actions.loading import wait_loading_end
 from .actions.in_purodyuusu import hajime_regular
-from kotonebot import device, image, ocr, task, action, sleep, equals
+from kotonebot import device, image, ocr, task, action, sleep, equals, contains
 from .actions.scenes import loading, at_home, goto_home
 
 logger = logging.getLogger(__name__)
@@ -96,6 +96,23 @@ def select_idol(target_titles: list[str] | PIdol):
     device.click(image.expect(R.Common.ButtonConfirmNoIcon))
     return found
 
+@action('继续当前培育')
+def resume_produce():
+    """
+    继续当前培育
+
+    前置条件：游戏首页，且当前有进行中培育\n
+    结束状态：游戏首页
+    """
+    # 点击 プロデュース中
+    # [res/sprites/jp/daily/home_1.png]
+    logger.info('Click ongoing produce button.')
+    device.click(R.Produce.BoxProduceOngoing)
+    # 点击 再開する
+    # [res/sprites/jp/produce/produce_resume.png]
+    logger.info('Click resume button.')
+    device.click(image.expect_wait(R.Produce.ButtonResume))
+
 @action('执行培育')
 def do_produce(idol: PIdol | None = None):
     """
@@ -108,6 +125,11 @@ def do_produce(idol: PIdol | None = None):
     """
     if not at_home():
         goto_home()
+    # 有进行中培育的情况
+    if ocr.find(contains('プロデュース中'), rect=R.Produce.BoxProduceOngoing):
+        logger.info('Ongoing produce found. Try to resume produce.')
+        resume_produce()
+        return
     # [screenshots/produce/home.png]
     device.click(image.expect_wait(R.Produce.ButtonProduce))
     sleep(0.3)
@@ -202,5 +224,5 @@ if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] [%(funcName)s] [%(lineno)d] %(message)s')
     logging.getLogger('kotonebot').setLevel(logging.DEBUG)
     logger.setLevel(logging.DEBUG)
-    produce_task()
+    do_produce()
     # select_idol(PIdol.藤田ことね_学園生活)

res/sprites/jp/daily/home_1.png.json

@@ -0,0 +1 @@
+{"definitions":{"85c24a02-dac3-4e6c-978e-b11963e0e92d":{"name":"Produce.BoxProduceOngoing","displayName":"首页开始培育按钮（当前有培育）","type":"hint-box","annotationId":"85c24a02-dac3-4e6c-978e-b11963e0e92d","useHintRect":false},"4fe748c8-a535-4824-aefc-244e3ad34bd4":{"name":"Daily.BoxHomeAssignment","displayName":"首页工作按钮","type":"hint-box","annotationId":"4fe748c8-a535-4824-aefc-244e3ad34bd4","useHintRect":false},"469f7f21-067c-476c-9bfc-c3ec83b935ea":{"name":"Daily.BoxHomeAP","displayName":"首页体力","type":"hint-box","annotationId":"469f7f21-067c-476c-9bfc-c3ec83b935ea","useHintRect":false},"565df1a0-d494-41a4-a3bf-603857bd8dec":{"name":"Daily.BoxHomeJewel","displayName":"首页珠宝数量","type":"hint-box","annotationId":"565df1a0-d494-41a4-a3bf-603857bd8dec","useHintRect":false},"76c92bd0-496b-403e-b545-92eb1f2f941f":{"name":"Daily.BoxHomeActivelyFunds","displayName":"首页活动费按钮","type":"hint-box","annotationId":"76c92bd0-496b-403e-b545-92eb1f2f941f","useHintRect":false}},"annotations":[{"id":"85c24a02-dac3-4e6c-978e-b11963e0e92d","type":"rect","data":{"x1":179,"y1":937,"x2":551,"y2":1091}},{"id":"4fe748c8-a535-4824-aefc-244e3ad34bd4","type":"rect","data":{"x1":16,"y1":642,"x2":127,"y2":752}},{"id":"469f7f21-067c-476c-9bfc-c3ec83b935ea","type":"rect","data":{"x1":291,"y1":4,"x2":500,"y2":82}},{"id":"565df1a0-d494-41a4-a3bf-603857bd8dec","type":"rect","data":{"x1":500,"y1":7,"x2":703,"y2":82}},{"id":"76c92bd0-496b-403e-b545-92eb1f2f941f","type":"rect","data":{"x1":11,"y1":517,"x2":137,"y2":637}}]}

res/sprites/jp/produce/produce_resume.png.json

@@ -0,0 +1 @@
+{"definitions":{"ccbcb114-7f73-43d1-904a-3a7ae660c531":{"name":"Produce.ButtonResume","displayName":"再開する","type":"template","annotationId":"ccbcb114-7f73-43d1-904a-3a7ae660c531","useHintRect":false},"daf3d823-b7f1-4584-acf3-90b9d880332c":{"name":"Produce.BoxResumeDialogProduceType","displayName":"培育再开始对话框 培育类型","type":"hint-box","annotationId":"daf3d823-b7f1-4584-acf3-90b9d880332c","useHintRect":false},"616273d6-49f1-42e5-a6be-df5408d69a4b":{"name":"Produce.BoxResumeDialogIdolTitle","displayName":"培育再开始对话框 偶像名称","type":"hint-box","annotationId":"616273d6-49f1-42e5-a6be-df5408d69a4b","useHintRect":false}},"annotations":[{"id":"ccbcb114-7f73-43d1-904a-3a7ae660c531","type":"rect","data":{"x1":390,"y1":1141,"x2":580,"y2":1181}},{"id":"daf3d823-b7f1-4584-acf3-90b9d880332c","type":"rect","data":{"x1":186,"y1":491,"x2":541,"y2":550}},{"id":"616273d6-49f1-42e5-a6be-df5408d69a4b","type":"rect","data":{"x1":194,"y1":794,"x2":523,"y2":840}}]}