feat(*): 支持培育自动检测当前周数 & OCR API 部分调整

1. 现在支持在培育的行动页面开始培育,而且不需要手动指定周数
2. OcrResult 类添加两个方法 regex number,便于从其中提取需要的数据
3. 将 OCR 识别结果返回类型改为 OcrResultList 类
4. 调整 OCR 单测
This commit is contained in:
XcantloadX 2025-02-08 21:42:45 +08:00
parent 62dab137da
commit d9f2be5f93
7 changed files with 188 additions and 132 deletions

View File

@ -39,7 +39,7 @@ from kotonebot.backend.image import (
)
import kotonebot.backend.color as raw_color
from kotonebot.backend.color import find_rgb
from kotonebot.backend.ocr import Ocr, OcrResult, jp, en, StringMatchFunction
from kotonebot.backend.ocr import Ocr, OcrResult, OcrResultList, jp, en, StringMatchFunction
from kotonebot.config.manager import load_config, save_config
from kotonebot.config.base_config import UserConfig
from kotonebot.backend.core import Image, HintBox
@ -244,22 +244,13 @@ class ContextOcr:
case _:
raise ValueError(f"Invalid language: {lang}")
@overload
def ocr(self) -> list[OcrResult]:
"""OCR 当前设备画面。"""
...
@overload
@deprecated('使用 `ocr.raw().ocr()` 代替')
def ocr(self, img: 'MatLike') -> list[OcrResult]:
"""OCR 指定图像。"""
...
def ocr(self, img: 'MatLike | None' = None, rect: Rect | None = None) -> list[OcrResult]:
def ocr(
self,
rect: Rect | None = None,
) -> OcrResultList:
"""OCR 当前设备画面或指定图像。"""
if img is None:
return self.__engine.ocr(ContextStackVars.ensure_current().screenshot, rect=rect)
return self.__engine.ocr(img)
return self.__engine.ocr(ContextStackVars.ensure_current().screenshot, rect=rect)
def find(
self,

View File

@ -30,12 +30,40 @@ _engine_en = RapidOCR(
)
StringMatchFunction = Callable[[str], bool]
REGEX_NUMBERS = re.compile(r'\d+')
class OcrResult(NamedTuple):
text: str
rect: Rect
confidence: float
def regex(self, pattern: re.Pattern | str) -> list[str]:
"""
提取识别结果中符合正则表达式的文本
"""
if isinstance(pattern, str):
pattern = re.compile(pattern)
return pattern.findall(self.text)
def numbers(self) -> list[int]:
"""
提取识别结果中的数字
"""
return [int(x) for x in REGEX_NUMBERS.findall(self.text)]
class OcrResultList(list[OcrResult]):
def first(self) -> OcrResult | None:
"""
返回第一个识别结果
"""
return self[0] if self else None
def where(self, pattern: StringMatchFunction) -> 'OcrResultList':
"""
返回符合条件的识别结果
"""
return OcrResultList([x for x in self if pattern(x.text)])
class TextNotFoundError(Exception):
def __init__(self, pattern: str | re.Pattern | StringMatchFunction, image: 'MatLike'):
self.pattern = pattern
@ -234,7 +262,7 @@ class Ocr:
*,
rect: Rect | None = None,
pad: bool = True,
) -> list[OcrResult]:
) -> OcrResultList:
"""
OCR 一个 cv2 的图像注意识别结果中的**全角字符会被转换为半角字符**
@ -260,8 +288,7 @@ class Ocr:
img_content = grayscaled(img)
result, elapse = self.__engine(img_content)
if result is None:
return []
return OcrResultList()
ret = [OcrResult(
text=unicodedata.normalize('NFKC', r[1]).replace('ą', 'a'), # HACK: 识别结果中包含奇怪的符号,暂时替换掉
# r[0] = [左上, 右上, 右下, 左下]
@ -270,6 +297,7 @@ class Ocr:
rect=tuple(int(x) for x in bounding_box(r[0])), # type: ignore
confidence=r[2] # type: ignore
) for r in result] # type: ignore
ret = OcrResultList(ret)
if debug.enabled:
result_image = _draw_result(img, ret)
debug_result(
@ -292,7 +320,7 @@ class Ocr:
pad: bool = True,
) -> OcrResult | None:
"""
寻找指定文本
识别图像中的文本寻找满足指定要求的文本
:param hint: 如果指定则首先只识别 HintBox 范围内的文本若未命中再全局寻找
:param rect: 如果指定则只识别指定矩形区域此参数优先级低于 `hint`
@ -317,11 +345,11 @@ class Ocr:
pad: bool = True,
) -> list[OcrResult | None]:
"""
寻找所有文本
识别图像中的文本并寻找多个满足指定要求的文本
:return:
所有找到的文本结果顺序与输入顺序相同
若某个文本未找到位置为 None
若某个文本未找到位置为 None
"""
# HintBox 处理
if hint is not None:
@ -351,7 +379,7 @@ class Ocr:
pad: bool = True,
) -> OcrResult:
"""
寻找指定文本如果未找到则抛出异常
识别图像中的文本并寻找满足指定要求的文本如果未找到则抛出异常
:param hint: 如果指定则首先只识别 HintBox 范围内的文本若未命中再全局寻找
:param rect: 如果指定则只识别指定矩形区域此参数优先级高于 `hint`

View File

@ -12,7 +12,8 @@ from . import loading
from .scenes import at_home
from .common import acquisitions
from ..common import conf
from kotonebot.backend.util import AdaptiveWait, crop, cropped
from kotonebot.backend.dispatch import DispatcherContext
from kotonebot.backend.util import AdaptiveWait, UnrecoverableError, crop, cropped
from kotonebot import ocr, device, contains, image, regex, action, debug, config, sleep
from .non_lesson_actions import enter_allowance, allowance_available, study_available, enter_study
@ -88,11 +89,11 @@ def enter_recommended_action(final_week: bool = False) -> ActionType:
return None
if not final_week:
if result.index == 0:
lesson_text = "Da"
lesson_text = contains("Da")
elif result.index == 1:
lesson_text = "Vo"
lesson_text = regex("Vo|V0")
elif result.index == 2:
lesson_text = "Vi"
lesson_text = contains("Vi")
elif result.index == 3:
rest()
return 'rest'
@ -101,7 +102,7 @@ def enter_recommended_action(final_week: bool = False) -> ActionType:
logger.info("Rec. lesson: %s", lesson_text)
# 点击课程
logger.debug("Try clicking lesson...")
lesson_ret = ocr.expect(contains(lesson_text))
lesson_ret = ocr.expect(lesson_text)
device.double_click(lesson_ret.rect)
return 'lesson'
else:
@ -355,7 +356,7 @@ def remaing_turns_and_points():
turns_rect_extended[1]:turns_rect_extended[1]+turns_rect_extended[3],
turns_rect_extended[0]:turns_rect_extended[0]+turns_rect_extended[2]
]
turns_ocr = ocr.ocr(turns_img)
turns_ocr = ocr.raw().ocr(turns_img)
logger.debug("turns_ocr: %s", turns_ocr)
@ -653,96 +654,72 @@ def hajime_regular(week: int = -1, start_from: int = 1):
logger.info("Week %d started.", i + start_from)
w()
def purodyuusu(
# TODO: 参数:成员、支援、记忆、 两个道具
):
# 流程:
# 1. Sensei 对话
# 2. Idol 对话
# 3. 领取P饮料
# 4. 触发支援卡事件。触发后必定需要领取物品
pass
ProduceStage = Literal[
'action', # 行动场景
'practice', # 练习场景
'exam', # 考试场景
'unknown', # 未知场景
]
@action('检测培育阶段并开始培育', dispatcher=True)
def detect_regular_produce_stage(ctx: DispatcherContext) -> ProduceStage:
"""
判断当前是培育的什么阶段并开始 Regular 培育
__actions__ = [enter_recommended_action]
前置条件培育中的任意场景\n
结束状态游戏主页面\n
"""
logger.info("Detecting current produce stage...")
# 行动场景
if (
image.find_multi([
R.InPurodyuusu.TextPDiary, # 普通周
R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
])
and (week_ret := ocr.find(contains(''), rect=R.InPurodyuusu.BoxWeeksUntilExam))
):
week = week_ret.numbers()
if week:
logger.info("Detection result: At action scene. Current week: %d", week[0])
# hajime_regular(week=week[0])
ctx.finish()
return 'action'
else:
return 'unknown'
else:
return 'unknown'
@action('开始 Regular 培育')
def hajime_regular_from_stage(stage: ProduceStage):
"""
开始 Regular 培育
"""
if stage == 'action':
texts = ocr.ocr(rect=R.InPurodyuusu.BoxWeeksUntilExam)
week_text = texts.where(contains('')).first()
if not week_text:
raise UnrecoverableError("Failed to detect week.")
# 提取周数
remaining_week = week_text.numbers()
if not remaining_week:
raise UnrecoverableError("Failed to detect week.")
# 判断阶段
if texts.where(contains('中間')):
week = 6 - remaining_week[0]
hajime_regular(start_from=week)
elif texts.where(contains('最終')):
week = 13 - remaining_week[0]
hajime_regular(start_from=week)
else:
raise UnrecoverableError("Failed to detect produce stage.")
if __name__ == '__main__':
from logging import getLogger
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s')
getLogger('kotonebot').setLevel(logging.DEBUG)
getLogger(__name__).setLevel(logging.DEBUG)
# exam()
# until_action_scene()
stage = (detect_regular_produce_stage())
hajime_regular_from_stage(stage)
# exam()
# produce_end()
# import cProfile
# p = cProfile.Prof
# ile()
# p.enable()
# acquisitions()
# p.disable()
# p.print_stats()
# p.dump_stats('profile.prof')
# until_action_scene()
# # 第一个箱子 [screenshots\allowance\step_2.png]
# logger.info("Clicking on the first lootbox.")
# device.click(image.expect_wait_any([
# R.InPurodyuusu.LootboxSliverLock
# ]))
# while acquisitions() is None:
# logger.info("Waiting for acquisitions finished.")
# sleep(2)
# # 第二个箱子
# logger.info("Clicking on the second lootbox.")
# device.click(image.expect_wait_any([
# R.InPurodyuusu.LootboxSliverLock
# ]))
# while acquisitions() is None:
# logger.info("Waiting for acquisitions finished.")
# sleep(2)
# logger.info("活動支給 completed.")
# while not image.wait_for_any([
# R.InPurodyuusu.TextPDiary, # 普通周
# R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
# ], timeout=2):
# logger.info("Action scene not detected. Retry...")
# acquisitions()
# sleep(3)
# image.wait_for_any([
# R.InPurodyuusu.TextPDiary, # 普通周
# R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
# ], timeout=2)
# while True:
# sleep(10)
# exam()
# produce_end()
# enter_recommended_action()
# remaing_turns_and_points()
# practice()
# until_action_scene()
# acquisitions()
# acquire_pdorinku(0)
# image.wait_for(R.InPurodyuusu.InPractice.PDorinkuIcon)
hajime_regular(start_from=4)
# until_practice_scene()
# device.click(image.expect_wait_any([
# R.InPurodyuusu.PSkillCardIconBlue,
# R.InPurodyuusu.PSkill
# CardIconColorful
# ]).rect)
# exam()
# device.double_click(image.expect_wait(R.InPurodyuusu.Action.VocalWhiteBg).rect)
# print(skill_card_count())
# click_recommended_card(card_count=skill_card_count())
# click_recommended_card(card_count=2)
# acquire_skill_card()
# rest()
# enter_recommended_lesson(final_week=True)

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

View File

@ -0,0 +1 @@
{"definitions":{"be6836bd-ee42-432b-9166-469c74f32f0b":{"name":"InPurodyuusu.BoxWeeksUntilExam","displayName":"培育中下次考试剩余周数","type":"hint-box","annotationId":"be6836bd-ee42-432b-9166-469c74f32f0b","useHintRect":false}},"annotations":[{"id":"be6836bd-ee42-432b-9166-469c74f32f0b","type":"rect","data":{"x1":11,"y1":8,"x2":237,"y2":196}}]}

View File

@ -1,8 +1,15 @@
import re
import unittest
from kotonebot.backend.ocr import bounding_box
from kotonebot.backend.ocr import jp
from kotonebot.backend.ocr import OcrResult, OcrResultList, bounding_box
import cv2
class TestOcr(unittest.TestCase):
def setUp(self):
self.img = cv2.imread('tests/images/acquire_pdorinku.png')
def test_bounding_box(self):
# 测试基本情况
# 矩形
@ -42,3 +49,74 @@ class TestOcr(unittest.TestCase):
# 测试重复点
points = [(5, 5), (5, 5), (5, 5)]
assert bounding_box(points) == (5, 5, 0, 0)
def test_ocr_ocr(self):
result = jp.ocr(self.img)
self.assertGreater(len(result), 0)
def test_ocr_find(self):
self.assertTrue(jp.find(self.img, '中間まで'))
self.assertTrue(jp.find(self.img, '受け取るPドリンクを選んでください。'))
self.assertTrue(jp.find(self.img, '受け取る'))
class TestOcrResult(unittest.TestCase):
def test_regex(self):
result = OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95)
self.assertEqual(result.regex(r'\d+'), ['123', '4567', '890'])
self.assertEqual(result.regex(re.compile(r'\d+')), ['123', '4567', '890'])
def test_numbers(self):
result = OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95)
self.assertEqual(result.numbers(), [123, 4567, 890])
result2 = OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95)
self.assertEqual(result2.numbers(), [])
result3 = OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95)
self.assertEqual(result3.numbers(), [1234567890])
class TestOcrResultList(unittest.TestCase):
def test_list_compatibility(self):
result = OcrResultList([
OcrResult(text='abc', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='def', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='ghi', rect=(0, 0, 100, 100), confidence=0.95),
])
self.assertEqual(result[0].text, 'abc')
self.assertEqual(result[-1].text, 'ghi')
self.assertEqual(len(result[1:]), 2)
self.assertEqual(result[1:][0].text, 'def')
self.assertEqual([r.text for r in result], ['abc', 'def', 'ghi'])
self.assertEqual(len(result), 3)
self.assertTrue(result[0] in result)
# 空列表
result = OcrResultList()
self.assertEqual(result, [])
self.assertEqual(bool(result), False)
self.assertEqual(len(result), 0)
with self.assertRaises(IndexError):
result[0]
with self.assertRaises(IndexError):
result[-1]
self.assertEqual(result[1:], [])
self.assertEqual([r.text for r in result], [])
def test_where(self):
result = OcrResultList([
OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95),
])
self.assertEqual(result.where(lambda x: x.startswith('123')), [result[0], result[2]])
def test_first(self):
result = OcrResultList([
OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95),
OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95),
])
self.assertEqual(result.first(), result[0])
result2 = OcrResultList()
self.assertIsNone(result2.first())

View File

@ -1,19 +0,0 @@
import unittest
from kotonebot.backend.ocr import jp
import cv2
class TestOcr(unittest.TestCase):
def setUp(self):
self.img = cv2.imread('test_images/acquire_pdorinku.png')
def test_ocr_ocr(self):
result = jp.ocr(self.img)
self.assertGreater(len(result), 0)
def test_ocr_find(self):
self.assertTrue(jp.find(self.img, '中間まで'))
self.assertTrue(jp.find(self.img, '受け取るPドリンクを選んでください。'))
self.assertTrue(jp.find(self.img, '受け取る'))