xcantloadx
/
kotones-auto-assistant
mirror of https://github.com/XcantloadX/kotones-auto-assistant
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(task): 将 kotonebot.tasks.game_ui 模块拆分为多个文件

This commit is contained in:
XcantloadX 2025-03-26 19:29:24 +08:00
parent 0c98e19bfb
commit 96be9740ba
11 changed files with 278 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
kotonebot/tasks/actions/common.py
View File

@ -1,23 +1,19 @@
from typing import Literal
from logging import getLogger
from kotonebot.backend.preprocessor import HsvColorFilter
from kotonebot.tasks.actions.loading import loading
from .. import R
from kotonebot import (
ocr,
device,
contains,
image,
regex,
action,
sleep,
Interval,
)
from ..game_ui import CommuEventButtonUI, WhiteFilter
from kotonebot.tasks.game_ui import WhiteFilter, CommuEventButtonUI
from .pdorinku import acquire_pdorinku
from kotonebot.backend.dispatch import SimpleDispatcher
from kotonebot.tasks.actions.commu import handle_unread_commu
from kotonebot.util import measure_time

6
kotonebot/tasks/actions/commu.py
View File

@ -5,9 +5,9 @@ from cv2.typing import MatLike
from .. import R
from ..game_ui import WhiteFilter
from kotonebot.util import Countdown, Interval
from kotonebot import device, image, color, user, rect_expand, until, action, sleep, use_screenshot
from kotonebot.util import Interval
from kotonebot.tasks.game_ui import WhiteFilter
from kotonebot import device, image, user, action, use_screenshot
logger = logging.getLogger(__name__)

6
kotonebot/tasks/actions/in_purodyuusu.py
View File

@ -1,4 +1,3 @@
import math
import time
import logging
from typing_extensions import deprecated, assert_never
@ -13,13 +12,13 @@ from .. import R
from . import loading
from .scenes import at_home
from ..util.trace import trace
from ..game_ui import WhiteFilter
from kotonebot.tasks.game_ui import WhiteFilter
from .commu import handle_unread_commu
from ..common import ProduceAction, RecommendCardDetectionMode, conf
from kotonebot.errors import UnrecoverableError
from kotonebot.backend.context.context import use_screenshot
from .common import until_acquisition_clear, acquisitions, commut_event
from kotonebot.util import AdaptiveWait, Countdown, Interval, crop, cropped
from kotonebot.util import Countdown, Interval, crop, cropped
from kotonebot.backend.dispatch import DispatcherContext, SimpleDispatcher
from kotonebot import ocr, device, contains, image, regex, action, sleep, color, Rect, wait
from .non_lesson_actions import (
@ -1168,7 +1167,6 @@ if __name__ == '__main__':
file_handler.setFormatter(logging.Formatter('[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s'))
logging.getLogger().addHandler(file_handler)
from kotonebot.util import Profiler
from kotonebot.backend.context import init_context, manual_context
from ..common import BaseConfig
from kotonebot.backend.debug import debug

2
kotonebot/tasks/actions/non_lesson_actions.py
View File

@ -9,7 +9,7 @@ from kotonebot.tasks.common import conf
from .. import R
from .common import acquisitions
from ..game_ui import CommuEventButtonUI
from kotonebot.tasks.game_ui.commu_event_buttons import CommuEventButtonUI
from kotonebot.util import Interval
from kotonebot.errors import UnrecoverableError
from kotonebot import device, image, action, sleep

10
kotonebot/tasks/actions/scenes.py
View File

@ -1,14 +1,9 @@
import logging
from typing import Callable
from .. import R
from .loading import loading
from kotonebot.util import Interval
from ..game_ui import toolbar_home
from kotonebot import device, image, action, cropped, until, sleep
from kotonebot.errors import UnrecoverableError
from kotonebot.tasks.game_ui import toolbar_home
from kotonebot import device, image, action, until, sleep
logger = logging.getLogger(__name__)
@ -78,6 +73,5 @@ def goto_shop():
until(at_daily_shop, critical=True)
if __name__ == "__main__":
import time
goto_home()

2
kotonebot/tasks/contest.py
View File

@ -4,7 +4,7 @@ from gettext import gettext as _
from . import R
from .common import conf
from .game_ui import WhiteFilter
from kotonebot.tasks.game_ui import WhiteFilter
from .actions.scenes import at_home, goto_home
from .actions.loading import wait_loading_end
from kotonebot import device, image, ocr, color, action, task, user, rect_expand, sleep, contains

246
kotonebot/tasks/game_ui.py
View File

@ -1,246 +0,0 @@
from dataclasses import dataclass
from typing import Literal, NamedTuple, overload
import cv2
import numpy as np
from cv2.typing import MatLike
from kotonebot.backend.image import TemplateMatchResult
from . import R
from kotonebot import action, device, color, image, ocr, sleep
from kotonebot.backend.color import HsvColor
from kotonebot.util import Rect
from kotonebot.backend.core import HintBox, Image
from kotonebot.backend.preprocessor import HsvColorFilter
@action('按钮是否禁用', screenshot_mode='manual-inherit')
def button_state(*, target: Image | None = None, rect: Rect | None = None) -> bool | None:
"""
判断按钮是否处于禁用状态。
:param rect: 按钮的矩形区域。必须包括文字或图标部分。
:param target: 按钮目标模板。
"""
img = device.screenshot()
if rect is not None:
_rect = rect
elif target is not None:
result = image.find(target)
if result is None:
return None
_rect = result.rect
else:
raise ValueError('Either rect or target must be provided.')
if color.find('#babcbd', rect=_rect):
return False
elif color.find('#ffffff', rect=_rect):
return True
else:
raise ValueError(f'Unknown button state: {img}')
def web2cv(hsv: HsvColor):
return (int(hsv[0]/360*180), int(hsv[1]/100*255), int(hsv[2]/100*255))
WHITE_LOW = (0, 0, 200)
WHITE_HIGH = (180, 30, 255)
PINK_TARGET = (335, 78, 95)
PINK_LOW = (300, 70, 90)
PINK_HIGH = (350, 80, 100)
BLUE_TARGET = (210, 88, 93)
BLUE_LOW = (200, 80, 90)
BLUE_HIGH = (220, 90, 100)
YELLOW_TARGET = (39, 81, 97)
YELLOW_LOW = (30, 70, 90)
YELLOW_HIGH = (45, 90, 100)
DEFAULT_COLORS = [
(web2cv(PINK_LOW), web2cv(PINK_HIGH)),
(web2cv(YELLOW_LOW), web2cv(YELLOW_HIGH)),
(web2cv(BLUE_LOW), web2cv(BLUE_HIGH)),
]
def filter_rectangles(
img: MatLike,
color_ranges: tuple[HsvColor, HsvColor],
aspect_ratio_threshold: float,
area_threshold: int,
rect: Rect | None = None
) -> list[Rect]:
"""
过滤出指定颜色，并执行轮廓查找，返回符合要求的轮廓的 bound box。
返回结果按照 y 坐标排序。
"""
img_hsv =cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
white_mask = cv2.inRange(img_hsv, np.array(color_ranges[0]), np.array(color_ranges[1]))
contours, _ = cv2.findContours(white_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
result_rects = []
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
# 如果不在指定范围内，跳过
if rect is not None:
rect_x1, rect_y1, rect_w, rect_h = rect
rect_x2 = rect_x1 + rect_w
rect_y2 = rect_y1 + rect_h
if not (
x >= rect_x1 and
y >= rect_y1 and
x + w <= rect_x2 and
y + h <= rect_y2
):
continue
aspect_ratio = w / h
area = cv2.contourArea(contour)
if aspect_ratio >= aspect_ratio_threshold and area >= area_threshold:
result_rects.append((x, y, w, h))
result_rects.sort(key=lambda x: x[1])
return result_rects
@dataclass
class EventButton:
rect: Rect
selected: bool
description: str
title: str
# 参考图片：
# [screenshots/produce/action_study3.png]
# TODO: CommuEventButtonUI 需要能够识别不可用的按钮
class CommuEventButtonUI:
"""
此类用于识别培育中交流中出现的事件/效果里的按钮。
例如外出（おでかけ）、冲刺周课程选择这两个页面的选择按钮。
"""
def __init__(
self,
selected_colors: list[tuple[HsvColor, HsvColor]] = DEFAULT_COLORS,
rect: HintBox = R.InPurodyuusu.BoxCommuEventButtonsArea
):
"""
:param selected_colors: 按钮选中后的主题色。
:param rect: 识别范围
"""
self.color_ranges = selected_colors
self.rect = rect
@action('交流事件按钮.识别选中', screenshot_mode='manual-inherit')
def selected(self, description: bool = True, title: bool = False) -> EventButton | None:
img = device.screenshot()
for i, color_range in enumerate(self.color_ranges):
rects = filter_rectangles(img, color_range, 7, 500, rect=self.rect)
if len(rects) > 0:
desc_text = self.description() if description else ''
title_text = ocr.ocr(rect=rects[0]).squash().text if title else ''
return EventButton(rects[0], True, desc_text, title_text)
return None
@action('交流事件按钮.识别按钮', screenshot_mode='manual-inherit')
def all(self, description: bool = True, title: bool = False) -> list[EventButton]:
"""
识别所有按钮的位置以及选中后的描述文本
前置条件：当前显示了交流事件按钮\n
结束状态：-
:param description: 是否识别描述文本。
:param title: 是否识别标题。
"""
img = device.screenshot()
rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 7, 500, rect=self.rect)
if not rects:
return []
selected = self.selected()
result: list[EventButton] = []
for rect in rects:
desc_text = ''
title_text = ''
if title:
title_text = ocr.ocr(rect=rect).squash().text
if description:
device.click(rect)
sleep(0.15)
device.screenshot()
desc_text = self.description()
result.append(EventButton(rect, False, desc_text, title_text))
# 修改最后一次点击的按钮为 selected 状态
if len(result) > 0:
result[-1].selected = True
if selected is not None:
result.append(selected)
selected.selected = False
result.sort(key=lambda x: x.rect[1])
return result
@action('交流事件按钮.识别描述', screenshot_mode='manual-inherit')
def description(self) -> str:
"""
识别当前选中按钮的描述文本
前置条件：有选中按钮\n
结束状态：-
"""
img = device.screenshot()
rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 3, 1000, rect=self.rect)
rects.sort(key=lambda x: x[1])
# TODO: 这里 rects 可能为空，需要加入判断重试
ocr_result = ocr.raw().ocr(img, rect=rects[0])
return ocr_result.squash().text
class WhiteFilter(HsvColorFilter):
"""
匹配时，只匹配图像和模板中的白色部分。
此类用于识别空心/透明背景的白色图标或文字。
"""
def __init__(self):
super().__init__(WHITE_LOW, WHITE_HIGH)
@overload
def toolbar_home(critical: Literal[False] = False) -> TemplateMatchResult | None:
"""寻找工具栏上的首页按钮。"""
...
@overload
def toolbar_home(critical: Literal[True]) -> TemplateMatchResult:
"""寻找工具栏上的首页按钮。若未找到，则抛出异常。"""
...
@action('工具栏按钮.寻找首页', screenshot_mode='manual-inherit')
def toolbar_home(critical: bool = False):
device.screenshot()
if critical:
return image.expect_wait(R.Common.ButtonToolbarHome, preprocessors=[WhiteFilter()])
else:
return image.find(R.Common.ButtonToolbarHome, preprocessors=[WhiteFilter()])
@overload
def toolbar_menu(critical: Literal[False] = False) -> TemplateMatchResult | None:
"""寻找工具栏上的菜单按钮。"""
...
@overload
def toolbar_menu(critical: Literal[True]) -> TemplateMatchResult:
"""寻找工具栏上的菜单按钮。若未找到，则抛出异常。"""
...
@action('工具栏按钮.寻找菜单', screenshot_mode='manual-inherit')
def toolbar_menu(critical: bool = False):
device.screenshot()
if critical:
return image.expect_wait(R.Common.ButtonToolbarMenu, preprocessors=[WhiteFilter()])
else:
return image.find(R.Common.ButtonToolbarMenu, preprocessors=[WhiteFilter()])
if __name__ == '__main__':
from pprint import pprint as print
from kotonebot.backend.context import init_context, manual_context, device
init_context()
manual_context().begin()
print(toolbar_home())

3
kotonebot/tasks/game_ui/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .toolbar import toolbar_home, toolbar_menu
from .commu_event_buttons import CommuEventButtonUI
from .common import WhiteFilter

101
kotonebot/tasks/game_ui/common.py Normal file
View File

@ -0,0 +1,101 @@
from dataclasses import dataclass
from typing import Literal, overload
import cv2
import numpy as np
from cv2.typing import MatLike
from kotonebot.backend.image import TemplateMatchResult
from kotonebot.tasks import R
from kotonebot import action, color, image
from kotonebot.backend.color import HsvColor
from kotonebot.util import Rect
from kotonebot.backend.core import Image
from kotonebot.backend.preprocessor import HsvColorFilter
def filter_rectangles(
img: MatLike,
color_ranges: tuple[HsvColor, HsvColor],
aspect_ratio_threshold: float,
area_threshold: int,
rect: Rect | None = None
) -> list[Rect]:
"""
过滤出指定颜色，并执行轮廓查找，返回符合要求的轮廓的 bound box。
返回结果按照 y 坐标排序。
"""
img_hsv =cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
white_mask = cv2.inRange(img_hsv, np.array(color_ranges[0]), np.array(color_ranges[1]))
contours, _ = cv2.findContours(white_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
result_rects = []
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
# 如果不在指定范围内，跳过
if rect is not None:
rect_x1, rect_y1, rect_w, rect_h = rect
rect_x2 = rect_x1 + rect_w
rect_y2 = rect_y1 + rect_h
if not (
x >= rect_x1 and
y >= rect_y1 and
x + w <= rect_x2 and
y + h <= rect_y2
):
continue
aspect_ratio = w / h
area = cv2.contourArea(contour)
if aspect_ratio >= aspect_ratio_threshold and area >= area_threshold:
result_rects.append((x, y, w, h))
result_rects.sort(key=lambda x: x[1])
return result_rects
@action('按钮是否禁用', screenshot_mode='manual-inherit')
def button_state(*, target: Image | None = None, rect: Rect | None = None) -> bool | None:
"""
判断按钮是否处于禁用状态。
:param rect: 按钮的矩形区域。必须包括文字或图标部分。
:param target: 按钮目标模板。
"""
img = device.screenshot()
if rect is not None:
_rect = rect
elif target is not None:
result = image.find(target)
if result is None:
return None
_rect = result.rect
else:
raise ValueError('Either rect or target must be provided.')
if color.find('#babcbd', rect=_rect):
return False
elif color.find('#ffffff', rect=_rect):
return True
else:
raise ValueError(f'Unknown button state: {img}')
WHITE_LOW = (0, 0, 200)
WHITE_HIGH = (180, 30, 255)
class WhiteFilter(HsvColorFilter):
"""
匹配时，只匹配图像和模板中的白色部分。
此类用于识别空心/透明背景的白色图标或文字。
"""
def __init__(self):
super().__init__(WHITE_LOW, WHITE_HIGH)
if __name__ == '__main__':
from pprint import pprint as print
from kotonebot.backend.context import init_context, manual_context, device
init_context()
manual_context().begin()

121
kotonebot/tasks/game_ui/commu_event_buttons.py Normal file
View File

@ -0,0 +1,121 @@
from dataclasses import dataclass
from kotonebot.tasks import R
from kotonebot.backend.core import HintBox
from kotonebot.backend.color import HsvColor
from kotonebot import action, device, ocr, sleep, Rect
from .common import filter_rectangles, WHITE_LOW, WHITE_HIGH
@dataclass
class EventButton:
rect: Rect
selected: bool
description: str
title: str
def web2cv(hsv: HsvColor):
return (int(hsv[0]/360*180), int(hsv[1]/100*255), int(hsv[2]/100*255))
PINK_TARGET = (335, 78, 95)
PINK_LOW = (300, 70, 90)
PINK_HIGH = (350, 80, 100)
BLUE_TARGET = (210, 88, 93)
BLUE_LOW = (200, 80, 90)
BLUE_HIGH = (220, 90, 100)
YELLOW_TARGET = (39, 81, 97)
YELLOW_LOW = (30, 70, 90)
YELLOW_HIGH = (45, 90, 100)
DEFAULT_COLORS = [
(web2cv(PINK_LOW), web2cv(PINK_HIGH)),
(web2cv(YELLOW_LOW), web2cv(YELLOW_HIGH)),
(web2cv(BLUE_LOW), web2cv(BLUE_HIGH)),
]
# 参考图片：
# [screenshots/produce/action_study3.png]
# TODO: CommuEventButtonUI 需要能够识别不可用的按钮
class CommuEventButtonUI:
"""
此类用于识别培育中交流中出现的事件/效果里的按钮。
例如外出（おでかけ）、冲刺周课程选择这两个页面的选择按钮。
"""
def __init__(
self,
selected_colors: list[tuple[HsvColor, HsvColor]] = DEFAULT_COLORS,
rect: HintBox = R.InPurodyuusu.BoxCommuEventButtonsArea
):
"""
:param selected_colors: 按钮选中后的主题色。
:param rect: 识别范围
"""
self.color_ranges = selected_colors
self.rect = rect
@action('交流事件按钮.识别选中', screenshot_mode='manual-inherit')
def selected(self, description: bool = True, title: bool = False) -> EventButton | None:
img = device.screenshot()
for i, color_range in enumerate(self.color_ranges):
rects = filter_rectangles(img, color_range, 7, 500, rect=self.rect)
if len(rects) > 0:
desc_text = self.description() if description else ''
title_text = ocr.ocr(rect=rects[0]).squash().text if title else ''
return EventButton(rects[0], True, desc_text, title_text)
return None
@action('交流事件按钮.识别按钮', screenshot_mode='manual-inherit')
def all(self, description: bool = True, title: bool = False) -> list[EventButton]:
"""
识别所有按钮的位置以及选中后的描述文本
前置条件：当前显示了交流事件按钮\n
结束状态：-
:param description: 是否识别描述文本。
:param title: 是否识别标题。
"""
img = device.screenshot()
rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 7, 500, rect=self.rect)
if not rects:
return []
selected = self.selected()
result: list[EventButton] = []
for rect in rects:
desc_text = ''
title_text = ''
if title:
title_text = ocr.ocr(rect=rect).squash().text
if description:
device.click(rect)
sleep(0.15)
device.screenshot()
desc_text = self.description()
result.append(EventButton(rect, False, desc_text, title_text))
# 修改最后一次点击的按钮为 selected 状态
if len(result) > 0:
result[-1].selected = True
if selected is not None:
result.append(selected)
selected.selected = False
result.sort(key=lambda x: x.rect[1])
return result
@action('交流事件按钮.识别描述', screenshot_mode='manual-inherit')
def description(self) -> str:
"""
识别当前选中按钮的描述文本
前置条件：有选中按钮\n
结束状态：-
"""
img = device.screenshot()
rects = filter_rectangles(img, (WHITE_LOW, WHITE_HIGH), 3, 1000, rect=self.rect)
rects.sort(key=lambda x: x[1])
# TODO: 这里 rects 可能为空，需要加入判断重试
ocr_result = ocr.raw().ocr(img, rect=rects[0])
return ocr_result.squash().text

43
kotonebot/tasks/game_ui/toolbar.py Normal file
View File

@ -0,0 +1,43 @@
from typing import Literal, overload
from kotonebot.backend.image import TemplateMatchResult
from kotonebot.tasks import R
from .common import WhiteFilter
from kotonebot import action, device, image
@overload
def toolbar_home(critical: Literal[False] = False) -> TemplateMatchResult | None:
"""寻找工具栏上的首页按钮。"""
...
@overload
def toolbar_home(critical: Literal[True]) -> TemplateMatchResult:
"""寻找工具栏上的首页按钮。若未找到，则抛出异常。"""
...
@action('工具栏按钮.寻找首页', screenshot_mode='manual-inherit')
def toolbar_home(critical: bool = False):
device.screenshot()
if critical:
return image.expect_wait(R.Common.ButtonToolbarHome, preprocessors=[WhiteFilter()])
else:
return image.find(R.Common.ButtonToolbarHome, preprocessors=[WhiteFilter()])
@overload
def toolbar_menu(critical: Literal[False] = False) -> TemplateMatchResult | None:
"""寻找工具栏上的菜单按钮。"""
...
@overload
def toolbar_menu(critical: Literal[True]) -> TemplateMatchResult:
"""寻找工具栏上的菜单按钮。若未找到，则抛出异常。"""
...
@action('工具栏按钮.寻找菜单', screenshot_mode='manual-inherit')
def toolbar_menu(critical: bool = False):
device.screenshot()
if critical:
return image.expect_wait(R.Common.ButtonToolbarMenu, preprocessors=[WhiteFilter()])
else:
return image.find(R.Common.ButtonToolbarMenu, preprocessors=[WhiteFilter()])