feat(*): 支持培育自动检测当前周数 & OCR API 部分调整
1. 现在支持在培育的行动页面开始培育,而且不需要手动指定周数 2. OcrResult 类添加两个方法 regex number,便于从其中提取需要的数据 3. 将 OCR 识别结果返回类型改为 OcrResultList 类 4. 调整 OCR 单测
This commit is contained in:
parent
62dab137da
commit
d9f2be5f93
|
@ -39,7 +39,7 @@ from kotonebot.backend.image import (
|
|||
)
|
||||
import kotonebot.backend.color as raw_color
|
||||
from kotonebot.backend.color import find_rgb
|
||||
from kotonebot.backend.ocr import Ocr, OcrResult, jp, en, StringMatchFunction
|
||||
from kotonebot.backend.ocr import Ocr, OcrResult, OcrResultList, jp, en, StringMatchFunction
|
||||
from kotonebot.config.manager import load_config, save_config
|
||||
from kotonebot.config.base_config import UserConfig
|
||||
from kotonebot.backend.core import Image, HintBox
|
||||
|
@ -244,22 +244,13 @@ class ContextOcr:
|
|||
case _:
|
||||
raise ValueError(f"Invalid language: {lang}")
|
||||
|
||||
@overload
|
||||
def ocr(self) -> list[OcrResult]:
|
||||
"""OCR 当前设备画面。"""
|
||||
...
|
||||
|
||||
@overload
|
||||
@deprecated('使用 `ocr.raw().ocr()` 代替')
|
||||
def ocr(self, img: 'MatLike') -> list[OcrResult]:
|
||||
"""OCR 指定图像。"""
|
||||
...
|
||||
|
||||
def ocr(self, img: 'MatLike | None' = None, rect: Rect | None = None) -> list[OcrResult]:
|
||||
def ocr(
|
||||
self,
|
||||
rect: Rect | None = None,
|
||||
) -> OcrResultList:
|
||||
"""OCR 当前设备画面或指定图像。"""
|
||||
if img is None:
|
||||
return self.__engine.ocr(ContextStackVars.ensure_current().screenshot, rect=rect)
|
||||
return self.__engine.ocr(img)
|
||||
return self.__engine.ocr(ContextStackVars.ensure_current().screenshot, rect=rect)
|
||||
|
||||
|
||||
def find(
|
||||
self,
|
||||
|
|
|
@ -30,12 +30,40 @@ _engine_en = RapidOCR(
|
|||
)
|
||||
|
||||
StringMatchFunction = Callable[[str], bool]
|
||||
REGEX_NUMBERS = re.compile(r'\d+')
|
||||
|
||||
class OcrResult(NamedTuple):
|
||||
text: str
|
||||
rect: Rect
|
||||
confidence: float
|
||||
|
||||
def regex(self, pattern: re.Pattern | str) -> list[str]:
|
||||
"""
|
||||
提取识别结果中符合正则表达式的文本。
|
||||
"""
|
||||
if isinstance(pattern, str):
|
||||
pattern = re.compile(pattern)
|
||||
return pattern.findall(self.text)
|
||||
|
||||
def numbers(self) -> list[int]:
|
||||
"""
|
||||
提取识别结果中的数字。
|
||||
"""
|
||||
return [int(x) for x in REGEX_NUMBERS.findall(self.text)]
|
||||
|
||||
class OcrResultList(list[OcrResult]):
|
||||
def first(self) -> OcrResult | None:
|
||||
"""
|
||||
返回第一个识别结果。
|
||||
"""
|
||||
return self[0] if self else None
|
||||
|
||||
def where(self, pattern: StringMatchFunction) -> 'OcrResultList':
|
||||
"""
|
||||
返回符合条件的识别结果。
|
||||
"""
|
||||
return OcrResultList([x for x in self if pattern(x.text)])
|
||||
|
||||
class TextNotFoundError(Exception):
|
||||
def __init__(self, pattern: str | re.Pattern | StringMatchFunction, image: 'MatLike'):
|
||||
self.pattern = pattern
|
||||
|
@ -234,7 +262,7 @@ class Ocr:
|
|||
*,
|
||||
rect: Rect | None = None,
|
||||
pad: bool = True,
|
||||
) -> list[OcrResult]:
|
||||
) -> OcrResultList:
|
||||
"""
|
||||
OCR 一个 cv2 的图像。注意识别结果中的**全角字符会被转换为半角字符**。
|
||||
|
||||
|
@ -260,8 +288,7 @@ class Ocr:
|
|||
img_content = grayscaled(img)
|
||||
result, elapse = self.__engine(img_content)
|
||||
if result is None:
|
||||
|
||||
return []
|
||||
return OcrResultList()
|
||||
ret = [OcrResult(
|
||||
text=unicodedata.normalize('NFKC', r[1]).replace('ą', 'a'), # HACK: 识别结果中包含奇怪的符号,暂时替换掉
|
||||
# r[0] = [左上, 右上, 右下, 左下]
|
||||
|
@ -270,6 +297,7 @@ class Ocr:
|
|||
rect=tuple(int(x) for x in bounding_box(r[0])), # type: ignore
|
||||
confidence=r[2] # type: ignore
|
||||
) for r in result] # type: ignore
|
||||
ret = OcrResultList(ret)
|
||||
if debug.enabled:
|
||||
result_image = _draw_result(img, ret)
|
||||
debug_result(
|
||||
|
@ -292,7 +320,7 @@ class Ocr:
|
|||
pad: bool = True,
|
||||
) -> OcrResult | None:
|
||||
"""
|
||||
寻找指定文本。
|
||||
识别图像中的文本,并寻找满足指定要求的文本。
|
||||
|
||||
:param hint: 如果指定,则首先只识别 HintBox 范围内的文本,若未命中,再全局寻找。
|
||||
:param rect: 如果指定,则只识别指定矩形区域。此参数优先级低于 `hint`。
|
||||
|
@ -317,11 +345,11 @@ class Ocr:
|
|||
pad: bool = True,
|
||||
) -> list[OcrResult | None]:
|
||||
"""
|
||||
寻找所有文本。
|
||||
识别图像中的文本,并寻找多个满足指定要求的文本。
|
||||
|
||||
:return:
|
||||
所有找到的文本,结果顺序与输入顺序相同。
|
||||
若某个文本未找到,则改位置为 None。
|
||||
若某个文本未找到,则该位置为 None。
|
||||
"""
|
||||
# HintBox 处理
|
||||
if hint is not None:
|
||||
|
@ -351,7 +379,7 @@ class Ocr:
|
|||
pad: bool = True,
|
||||
) -> OcrResult:
|
||||
"""
|
||||
寻找指定文本,如果未找到则抛出异常。
|
||||
识别图像中的文本,并寻找满足指定要求的文本。如果未找到则抛出异常。
|
||||
|
||||
:param hint: 如果指定,则首先只识别 HintBox 范围内的文本,若未命中,再全局寻找。
|
||||
:param rect: 如果指定,则只识别指定矩形区域。此参数优先级高于 `hint`。
|
||||
|
|
|
@ -12,7 +12,8 @@ from . import loading
|
|||
from .scenes import at_home
|
||||
from .common import acquisitions
|
||||
from ..common import conf
|
||||
from kotonebot.backend.util import AdaptiveWait, crop, cropped
|
||||
from kotonebot.backend.dispatch import DispatcherContext
|
||||
from kotonebot.backend.util import AdaptiveWait, UnrecoverableError, crop, cropped
|
||||
from kotonebot import ocr, device, contains, image, regex, action, debug, config, sleep
|
||||
from .non_lesson_actions import enter_allowance, allowance_available, study_available, enter_study
|
||||
|
||||
|
@ -88,11 +89,11 @@ def enter_recommended_action(final_week: bool = False) -> ActionType:
|
|||
return None
|
||||
if not final_week:
|
||||
if result.index == 0:
|
||||
lesson_text = "Da"
|
||||
lesson_text = contains("Da")
|
||||
elif result.index == 1:
|
||||
lesson_text = "Vo"
|
||||
lesson_text = regex("Vo|V0")
|
||||
elif result.index == 2:
|
||||
lesson_text = "Vi"
|
||||
lesson_text = contains("Vi")
|
||||
elif result.index == 3:
|
||||
rest()
|
||||
return 'rest'
|
||||
|
@ -101,7 +102,7 @@ def enter_recommended_action(final_week: bool = False) -> ActionType:
|
|||
logger.info("Rec. lesson: %s", lesson_text)
|
||||
# 点击课程
|
||||
logger.debug("Try clicking lesson...")
|
||||
lesson_ret = ocr.expect(contains(lesson_text))
|
||||
lesson_ret = ocr.expect(lesson_text)
|
||||
device.double_click(lesson_ret.rect)
|
||||
return 'lesson'
|
||||
else:
|
||||
|
@ -355,7 +356,7 @@ def remaing_turns_and_points():
|
|||
turns_rect_extended[1]:turns_rect_extended[1]+turns_rect_extended[3],
|
||||
turns_rect_extended[0]:turns_rect_extended[0]+turns_rect_extended[2]
|
||||
]
|
||||
turns_ocr = ocr.ocr(turns_img)
|
||||
turns_ocr = ocr.raw().ocr(turns_img)
|
||||
logger.debug("turns_ocr: %s", turns_ocr)
|
||||
|
||||
|
||||
|
@ -653,96 +654,72 @@ def hajime_regular(week: int = -1, start_from: int = 1):
|
|||
logger.info("Week %d started.", i + start_from)
|
||||
w()
|
||||
|
||||
def purodyuusu(
|
||||
# TODO: 参数:成员、支援、记忆、 两个道具
|
||||
):
|
||||
# 流程:
|
||||
# 1. Sensei 对话
|
||||
# 2. Idol 对话
|
||||
# 3. 领取P饮料(?)
|
||||
# 4. 触发支援卡事件。触发后必定需要领取物品
|
||||
pass
|
||||
ProduceStage = Literal[
|
||||
'action', # 行动场景
|
||||
'practice', # 练习场景
|
||||
'exam', # 考试场景
|
||||
'unknown', # 未知场景
|
||||
]
|
||||
@action('检测培育阶段并开始培育', dispatcher=True)
|
||||
def detect_regular_produce_stage(ctx: DispatcherContext) -> ProduceStage:
|
||||
"""
|
||||
|
||||
判断当前是培育的什么阶段,并开始 Regular 培育。
|
||||
|
||||
__actions__ = [enter_recommended_action]
|
||||
前置条件:培育中的任意场景\n
|
||||
结束状态:游戏主页面\n
|
||||
"""
|
||||
logger.info("Detecting current produce stage...")
|
||||
# 行动场景
|
||||
if (
|
||||
image.find_multi([
|
||||
R.InPurodyuusu.TextPDiary, # 普通周
|
||||
R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
|
||||
])
|
||||
and (week_ret := ocr.find(contains('週'), rect=R.InPurodyuusu.BoxWeeksUntilExam))
|
||||
):
|
||||
week = week_ret.numbers()
|
||||
if week:
|
||||
logger.info("Detection result: At action scene. Current week: %d", week[0])
|
||||
# hajime_regular(week=week[0])
|
||||
ctx.finish()
|
||||
return 'action'
|
||||
else:
|
||||
return 'unknown'
|
||||
else:
|
||||
return 'unknown'
|
||||
|
||||
@action('开始 Regular 培育')
|
||||
def hajime_regular_from_stage(stage: ProduceStage):
|
||||
"""
|
||||
开始 Regular 培育。
|
||||
"""
|
||||
if stage == 'action':
|
||||
texts = ocr.ocr(rect=R.InPurodyuusu.BoxWeeksUntilExam)
|
||||
week_text = texts.where(contains('週')).first()
|
||||
if not week_text:
|
||||
raise UnrecoverableError("Failed to detect week.")
|
||||
# 提取周数
|
||||
remaining_week = week_text.numbers()
|
||||
if not remaining_week:
|
||||
raise UnrecoverableError("Failed to detect week.")
|
||||
# 判断阶段
|
||||
if texts.where(contains('中間')):
|
||||
week = 6 - remaining_week[0]
|
||||
hajime_regular(start_from=week)
|
||||
elif texts.where(contains('最終')):
|
||||
week = 13 - remaining_week[0]
|
||||
hajime_regular(start_from=week)
|
||||
else:
|
||||
raise UnrecoverableError("Failed to detect produce stage.")
|
||||
|
||||
if __name__ == '__main__':
|
||||
from logging import getLogger
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s')
|
||||
getLogger('kotonebot').setLevel(logging.DEBUG)
|
||||
getLogger(__name__).setLevel(logging.DEBUG)
|
||||
|
||||
# exam()
|
||||
# until_action_scene()
|
||||
stage = (detect_regular_produce_stage())
|
||||
hajime_regular_from_stage(stage)
|
||||
|
||||
# exam()
|
||||
# produce_end()
|
||||
|
||||
# import cProfile
|
||||
# p = cProfile.Prof
|
||||
# ile()
|
||||
# p.enable()
|
||||
# acquisitions()
|
||||
# p.disable()
|
||||
# p.print_stats()
|
||||
# p.dump_stats('profile.prof')
|
||||
|
||||
# until_action_scene()
|
||||
|
||||
# # 第一个箱子 [screenshots\allowance\step_2.png]
|
||||
# logger.info("Clicking on the first lootbox.")
|
||||
# device.click(image.expect_wait_any([
|
||||
# R.InPurodyuusu.LootboxSliverLock
|
||||
# ]))
|
||||
# while acquisitions() is None:
|
||||
# logger.info("Waiting for acquisitions finished.")
|
||||
# sleep(2)
|
||||
# # 第二个箱子
|
||||
# logger.info("Clicking on the second lootbox.")
|
||||
# device.click(image.expect_wait_any([
|
||||
# R.InPurodyuusu.LootboxSliverLock
|
||||
# ]))
|
||||
# while acquisitions() is None:
|
||||
# logger.info("Waiting for acquisitions finished.")
|
||||
# sleep(2)
|
||||
# logger.info("活動支給 completed.")
|
||||
|
||||
# while not image.wait_for_any([
|
||||
# R.InPurodyuusu.TextPDiary, # 普通周
|
||||
# R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
|
||||
# ], timeout=2):
|
||||
# logger.info("Action scene not detected. Retry...")
|
||||
# acquisitions()
|
||||
# sleep(3)
|
||||
|
||||
# image.wait_for_any([
|
||||
# R.InPurodyuusu.TextPDiary, # 普通周
|
||||
# R.InPurodyuusu.ButtonFinalPracticeDance # 离考试剩余一周
|
||||
# ], timeout=2)
|
||||
# while True:
|
||||
# sleep(10)
|
||||
|
||||
# exam()
|
||||
# produce_end()
|
||||
# enter_recommended_action()
|
||||
# remaing_turns_and_points()
|
||||
# practice()
|
||||
# until_action_scene()
|
||||
# acquisitions()
|
||||
# acquire_pdorinku(0)
|
||||
# image.wait_for(R.InPurodyuusu.InPractice.PDorinkuIcon)
|
||||
hajime_regular(start_from=4)
|
||||
# until_practice_scene()
|
||||
# device.click(image.expect_wait_any([
|
||||
# R.InPurodyuusu.PSkillCardIconBlue,
|
||||
# R.InPurodyuusu.PSkill
|
||||
# CardIconColorful
|
||||
# ]).rect)
|
||||
# exam()
|
||||
# device.double_click(image.expect_wait(R.InPurodyuusu.Action.VocalWhiteBg).rect)
|
||||
# print(skill_card_count())
|
||||
# click_recommended_card(card_count=skill_card_count())
|
||||
# click_recommended_card(card_count=2)
|
||||
# acquire_skill_card()
|
||||
# rest()
|
||||
# enter_recommended_lesson(final_week=True)
|
||||
|
|
Binary file not shown.
After Width: | Height: | Size: 1.1 MiB |
|
@ -0,0 +1 @@
|
|||
{"definitions":{"be6836bd-ee42-432b-9166-469c74f32f0b":{"name":"InPurodyuusu.BoxWeeksUntilExam","displayName":"培育中下次考试剩余周数","type":"hint-box","annotationId":"be6836bd-ee42-432b-9166-469c74f32f0b","useHintRect":false}},"annotations":[{"id":"be6836bd-ee42-432b-9166-469c74f32f0b","type":"rect","data":{"x1":11,"y1":8,"x2":237,"y2":196}}]}
|
|
@ -1,8 +1,15 @@
|
|||
import re
|
||||
import unittest
|
||||
|
||||
from kotonebot.backend.ocr import bounding_box
|
||||
from kotonebot.backend.ocr import jp
|
||||
from kotonebot.backend.ocr import OcrResult, OcrResultList, bounding_box
|
||||
import cv2
|
||||
|
||||
|
||||
class TestOcr(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.img = cv2.imread('tests/images/acquire_pdorinku.png')
|
||||
|
||||
def test_bounding_box(self):
|
||||
# 测试基本情况
|
||||
# 矩形
|
||||
|
@ -42,3 +49,74 @@ class TestOcr(unittest.TestCase):
|
|||
# 测试重复点
|
||||
points = [(5, 5), (5, 5), (5, 5)]
|
||||
assert bounding_box(points) == (5, 5, 0, 0)
|
||||
|
||||
def test_ocr_ocr(self):
|
||||
result = jp.ocr(self.img)
|
||||
self.assertGreater(len(result), 0)
|
||||
|
||||
def test_ocr_find(self):
|
||||
self.assertTrue(jp.find(self.img, '中間まで'))
|
||||
self.assertTrue(jp.find(self.img, '受け取るPドリンクを選んでください。'))
|
||||
self.assertTrue(jp.find(self.img, '受け取る'))
|
||||
|
||||
|
||||
class TestOcrResult(unittest.TestCase):
|
||||
def test_regex(self):
|
||||
result = OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95)
|
||||
self.assertEqual(result.regex(r'\d+'), ['123', '4567', '890'])
|
||||
self.assertEqual(result.regex(re.compile(r'\d+')), ['123', '4567', '890'])
|
||||
|
||||
def test_numbers(self):
|
||||
result = OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95)
|
||||
self.assertEqual(result.numbers(), [123, 4567, 890])
|
||||
result2 = OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95)
|
||||
self.assertEqual(result2.numbers(), [])
|
||||
result3 = OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95)
|
||||
self.assertEqual(result3.numbers(), [1234567890])
|
||||
|
||||
|
||||
class TestOcrResultList(unittest.TestCase):
|
||||
def test_list_compatibility(self):
|
||||
result = OcrResultList([
|
||||
OcrResult(text='abc', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='def', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='ghi', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
])
|
||||
|
||||
self.assertEqual(result[0].text, 'abc')
|
||||
self.assertEqual(result[-1].text, 'ghi')
|
||||
self.assertEqual(len(result[1:]), 2)
|
||||
self.assertEqual(result[1:][0].text, 'def')
|
||||
self.assertEqual([r.text for r in result], ['abc', 'def', 'ghi'])
|
||||
self.assertEqual(len(result), 3)
|
||||
self.assertTrue(result[0] in result)
|
||||
|
||||
# 空列表
|
||||
result = OcrResultList()
|
||||
self.assertEqual(result, [])
|
||||
self.assertEqual(bool(result), False)
|
||||
self.assertEqual(len(result), 0)
|
||||
with self.assertRaises(IndexError):
|
||||
result[0]
|
||||
with self.assertRaises(IndexError):
|
||||
result[-1]
|
||||
self.assertEqual(result[1:], [])
|
||||
self.assertEqual([r.text for r in result], [])
|
||||
|
||||
def test_where(self):
|
||||
result = OcrResultList([
|
||||
OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
])
|
||||
self.assertEqual(result.where(lambda x: x.startswith('123')), [result[0], result[2]])
|
||||
|
||||
def test_first(self):
|
||||
result = OcrResultList([
|
||||
OcrResult(text='123dd4567rr890', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='aaa', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
OcrResult(text='1234567890', rect=(0, 0, 100, 100), confidence=0.95),
|
||||
])
|
||||
self.assertEqual(result.first(), result[0])
|
||||
result2 = OcrResultList()
|
||||
self.assertIsNone(result2.first())
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
import unittest
|
||||
|
||||
from kotonebot.backend.ocr import jp
|
||||
|
||||
import cv2
|
||||
|
||||
|
||||
class TestOcr(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.img = cv2.imread('test_images/acquire_pdorinku.png')
|
||||
|
||||
def test_ocr_ocr(self):
|
||||
result = jp.ocr(self.img)
|
||||
self.assertGreater(len(result), 0)
|
||||
|
||||
def test_ocr_find(self):
|
||||
self.assertTrue(jp.find(self.img, '中間まで'))
|
||||
self.assertTrue(jp.find(self.img, '受け取るPドリンクを選んでください。'))
|
||||
self.assertTrue(jp.find(self.img, '受け取る'))
|
Loading…
Reference in New Issue