mirror of https://github.com/alantang1977/pg
Update emoji_manager.py
This commit is contained in:
alantang
GitHub
parent
253eea616e
commit
d045c96df2
|
|
@ -4,41 +4,37 @@ import json
|
|||
import csv
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
from collections import Counter, deque
|
||||
|
||||
# Android-safe emoji sets by categories
|
||||
# Android-safe emoji pool（可自定义扩充）
|
||||
EMOJI_POOL = [
|
||||
# 动物与自然
|
||||
"🐶", "🐱", "🐭", "🐹", "🐰", "🦊", "🐻", "🐼", "🐨", "🐯",
|
||||
"🦁", "🐮", "🐷", "🐸", "🐵", "🦄", "🐔", "🐧", "🐦", "🐤",
|
||||
"🐝", "🐞", "🦋", "🐢", "🐍", "🐠", "🐬", "🐳", "🦑", "🦀",
|
||||
"🌸", "🌻", "🌹", "🌺", "🌼", "🌷", "🌱", "🌲", "🌳", "🌴",
|
||||
|
||||
# 食物与饮料
|
||||
"🍏", "🍎", "🍐", "🍊", "🍋", "🍌", "🍉", "🍇", "🍓", "🫐",
|
||||
"🍈", "🍒", "🍑", "🥭", "🍍", "🥥", "🥝", "🍅", "🍆", "🥑",
|
||||
"🥦", "🥬", "🥕", "🌽", "🌶️", "🧄", "🧅", "🥔", "🥚", "🍞",
|
||||
"🥐", "🥯", "🥨", "🧀", "🥓", "🍗", "🍖", "🌭", "🍔", "🍟",
|
||||
|
||||
# 活动
|
||||
"⚽", "🏀", "🏈", "⚾", "🎾", "🏐", "🏉", "🎱", "🏓", "🏸",
|
||||
"🥅", "🏒", "🏑", "🏏", "⛳", "🏹", "🎣", "🤿", "🥊", "🥋",
|
||||
"🎯", "🎳", "🪁", "🛹", "🥌", "🛷", "⛷️", "🏂", "🏄♂️", "🏊♀️",
|
||||
"🎮", "🎲", "🧩", "🀄", "♟️", "🃏", "🧗", "🏆", "🥇", "🥈", "🥉",
|
||||
|
||||
# 物体
|
||||
"⌚", "📱", "💻", "🖨️", "🕹️", "🎮", "📷", "📸", "📹", "🎥",
|
||||
"📺", "📻", "🎙️", "🎚️", "🎛️", "☎️", "📞", "📟", "📠", "🔋",
|
||||
"🔌", "💡", "🔦", "🕯️", "🛢️", "💰", "💳", "🗝️", "🔑", "🚪",
|
||||
"🔒", "🔓", "🔑", "🔐", "🛡️", "🧲", "⚖️", "🔗", "🧰", "🔧",
|
||||
"🔨", "🪓", "⛏️", "🛠️", "🗡️", "⚔️", "🔫", "🏹", "🛏️", "🛋️",
|
||||
|
||||
# 旅行与地点
|
||||
"🚗", "🚕", "🚙", "🚌", "🚎", "🏎️", "🚓", "🚑", "🚒", "🚐",
|
||||
"🛻", "🚚", "🚛", "🚜", "🏍️", "🛵", "🚲", "🛴", "🚨", "🚔",
|
||||
"🚦", "🚧", "🛣️", "🛤️", "✈️", "🛩️", "🚁", "🚀", "🛸", "⛵",
|
||||
"🚢", "🛳️", "⛴️", "🚤", "🛥️", "🗽", "🗼", "🏰", "🏯", "🌋",
|
||||
"🗻", "🏕️", "🏞️", "🏜️", "🏝️", "🏖️", "🏟️", "🏛️", "🏗️",
|
||||
|
||||
# 人物
|
||||
"😀", "😃", "😄", "😁", "😆", "😅", "😂", "🤣", "😊", "😇",
|
||||
"🙂", "🙃", "😉", "😌", "😍", "🥰", "😘", "😗", "😙", "😚",
|
||||
|
|
@ -53,7 +49,6 @@ EMOJI_POOL = [
|
|||
"🖕", "👇", "☝️", "🫳", "🫴", "💪", "🦾", "🦿", "🦵", "🦶",
|
||||
"👀", "👁️", "👅", "👄", "🧑", "👦", "👧", "👨", "👩", "🧓",
|
||||
"👴", "👵", "🧔", "👱♂️", "👱♀️", "👨🦰", "👩🦰", "👨🦱", "👩🦱",
|
||||
|
||||
# 符号
|
||||
"❤️", "🧡", "💛", "💚", "💙", "💜", "🤎", "🖤", "🤍", "💔",
|
||||
"❣️", "💕", "💞", "💓", "💗", "💖", "💘", "💝", "💟", "☮️",
|
||||
|
|
@ -68,11 +63,8 @@ EMOJI_POOL = [
|
|||
"🔫", "🏹", "🛏️", "🛋️", "🛒", "🧺", "🧻", "🧼", "🪥", "🧽", "🪣", "🧴", "🧷", "🧹", "🪒", "🧯", "🛎️"
|
||||
]
|
||||
|
||||
# Regex pattern for emoji (cover most cases, including ZWJ sequences)
|
||||
EMOJI_REGEX = re.compile(
|
||||
r"("
|
||||
r"(?:[\U0001F3FB-\U0001F3FF])|" # skin tone
|
||||
r"(?:[\U0001F9B0-\U0001F9B3])|" # hair
|
||||
r"(?:[\U0001F1E6-\U0001F1FF]{2})|" # flags
|
||||
r"(?:[\U0001F600-\U0001F64F])|"
|
||||
r"(?:[\U0001F300-\U0001F5FF])|"
|
||||
|
|
@ -87,7 +79,7 @@ EMOJI_REGEX = re.compile(
|
|||
r"(?:[\u2700-\u27BF])|"
|
||||
r"(?:[\u2300-\u23FF])|"
|
||||
r"(?:[\u2B05-\u2B07])|"
|
||||
r"(?:\u200D)" # zwj joiner
|
||||
r"(?:\u200D)"
|
||||
r")+",
|
||||
re.UNICODE
|
||||
)
|
||||
|
|
@ -98,106 +90,119 @@ def extract_emojis(text):
|
|||
# Find all emoji/sequence, keeping order, including duplicates
|
||||
return [m.group(0) for m in EMOJI_REGEX.finditer(text)]
|
||||
|
||||
def find_duplicates(emoji_list):
|
||||
# Return a set of emojis that occur more than once
|
||||
from collections import Counter
|
||||
count = Counter(emoji_list)
|
||||
return {em for em, c in count.items() if c > 1}
|
||||
def build_duplicate_emoji_mapping(emoji_list, emoji_pool):
|
||||
"""
|
||||
保留每个emoji首次出现不变，仅对重复出现的emoji做替换为pool中未用过的emoji。
|
||||
若emoji_pool用尽则循环使用。
|
||||
返回字典：{原emoji: [None, 替换1, 替换2...]}，None表示首个不换。
|
||||
"""
|
||||
counts = Counter(emoji_list)
|
||||
duplicates = {em for em, c in counts.items() if c > 1}
|
||||
# 统计每个emoji出现了几次
|
||||
positions = {}
|
||||
for idx, em in enumerate(emoji_list):
|
||||
positions.setdefault(em, []).append(idx)
|
||||
# 新emoji分配池
|
||||
used = set(emoji_list)
|
||||
pool = deque([e for e in emoji_pool if e not in used])
|
||||
emoji_replace_map = {}
|
||||
for em in duplicates:
|
||||
occurrence = len(positions[em])
|
||||
repls = [None] # 第一次出现不替换
|
||||
for _ in range(occurrence - 1):
|
||||
if not pool:
|
||||
pool = deque([e for e in emoji_pool if e not in used])
|
||||
if pool:
|
||||
new_em = pool.popleft()
|
||||
repls.append(new_em)
|
||||
pool.append(new_em)
|
||||
else:
|
||||
# 纯保险，理论不会到这里
|
||||
repls.append(em)
|
||||
emoji_replace_map[em] = repls
|
||||
return emoji_replace_map, positions
|
||||
|
||||
def build_emoji_replace_map(emoji_list, emoji_pool):
|
||||
# Only map duplicate emojis; unique ones stay unchanged
|
||||
from collections import Counter, deque
|
||||
|
||||
counter = Counter(emoji_list)
|
||||
dups = [em for em in emoji_list if counter[em] > 1]
|
||||
seen = set()
|
||||
dups_unique = []
|
||||
for em in dups:
|
||||
if em not in seen:
|
||||
dups_unique.append(em)
|
||||
seen.add(em)
|
||||
pool = deque([e for e in emoji_pool if e not in emoji_list]) # don't use emojis already present
|
||||
emoji_map = {}
|
||||
for em in dups_unique:
|
||||
if not pool:
|
||||
pool = deque([e for e in emoji_pool if e not in emoji_list])
|
||||
if pool:
|
||||
emoji_map[em] = pool.popleft()
|
||||
return emoji_map
|
||||
|
||||
def replace_duplicate_emojis(text, emoji_map):
|
||||
# Only replace duplicates (the 2nd, 3rd, ...) occurrence for each
|
||||
# First occurrence stays, others replaced
|
||||
def replace_duplicates_in_text(text, emoji_replace_map, positions):
|
||||
"""
|
||||
仅替换重复的emoji，第一次出现保留，后续出现才换。
|
||||
"""
|
||||
matches = list(EMOJI_REGEX.finditer(text))
|
||||
new_text = []
|
||||
# 记录每个emoji已出现次数
|
||||
occur_dict = {em:0 for em in emoji_replace_map}
|
||||
last_idx = 0
|
||||
seen = {}
|
||||
result = []
|
||||
for m in matches:
|
||||
em = m.group(0)
|
||||
new_text.append(text[last_idx:m.start()])
|
||||
seen.setdefault(em, 0)
|
||||
seen[em] += 1
|
||||
if em in emoji_map and seen[em] > 1:
|
||||
new_text.append(emoji_map[em])
|
||||
result.append(text[last_idx:m.start()])
|
||||
if em in emoji_replace_map:
|
||||
occur_dict[em] += 1
|
||||
# 首次出现用原emoji，后续用映射
|
||||
idx = occur_dict[em]
|
||||
repl = emoji_replace_map[em][idx-1] # idx-1：第0次None, 从1开始替换
|
||||
if repl is None:
|
||||
result.append(em)
|
||||
else:
|
||||
result.append(repl)
|
||||
else:
|
||||
new_text.append(em)
|
||||
result.append(em)
|
||||
last_idx = m.end()
|
||||
new_text.append(text[last_idx:])
|
||||
return ''.join(new_text)
|
||||
result.append(text[last_idx:])
|
||||
return ''.join(result)
|
||||
|
||||
def process_json_file(src, dst, emoji_map):
|
||||
def process_json_file(src, dst, emoji_replace_map, positions):
|
||||
with open(src, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
def recursive_replace(obj):
|
||||
def recursive(obj):
|
||||
if isinstance(obj, str):
|
||||
return replace_duplicate_emojis(obj, emoji_map)
|
||||
return replace_duplicates_in_text(obj, emoji_replace_map, positions)
|
||||
elif isinstance(obj, list):
|
||||
return [recursive_replace(item) for item in obj]
|
||||
return [recursive(o) for o in obj]
|
||||
elif isinstance(obj, dict):
|
||||
return {k: recursive_replace(v) for k, v in obj.items()}
|
||||
return {k: recursive(v) for k, v in obj.items()}
|
||||
return obj
|
||||
new_data = recursive_replace(data)
|
||||
data_new = recursive(data)
|
||||
with open(dst, "w", encoding="utf-8") as f:
|
||||
json.dump(new_data, f, ensure_ascii=False, indent=2)
|
||||
json.dump(data_new, f, ensure_ascii=False, indent=2)
|
||||
|
||||
def process_csv_file(src, dst, emoji_map):
|
||||
def process_csv_file(src, dst, emoji_replace_map, positions):
|
||||
with open(src, "r", encoding="utf-8", newline='') as f:
|
||||
reader = list(csv.reader(f))
|
||||
new_rows = []
|
||||
for row in reader:
|
||||
new_row = [replace_duplicate_emojis(cell, emoji_map) for cell in row]
|
||||
new_row = [replace_duplicates_in_text(cell, emoji_replace_map, positions) for cell in row]
|
||||
new_rows.append(new_row)
|
||||
with open(dst, "w", encoding="utf-8", newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(new_rows)
|
||||
|
||||
def process_txt_file(src, dst, emoji_map):
|
||||
def process_txt_file(src, dst, emoji_replace_map, positions):
|
||||
with open(src, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
new_text = replace_duplicate_emojis(text, emoji_map)
|
||||
new_text = replace_duplicates_in_text(text, emoji_replace_map, positions)
|
||||
with open(dst, "w", encoding="utf-8") as f:
|
||||
f.write(new_text)
|
||||
|
||||
def process_md_file(src, dst, emoji_map):
|
||||
process_txt_file(src, dst, emoji_map)
|
||||
def process_md_file(src, dst, emoji_replace_map, positions):
|
||||
process_txt_file(src, dst, emoji_replace_map, positions)
|
||||
|
||||
def process_xml_file(src, dst, emoji_map):
|
||||
def process_xml_file(src, dst, emoji_replace_map, positions):
|
||||
tree = ET.parse(src)
|
||||
root = tree.getroot()
|
||||
def recursive_xml(elem):
|
||||
if elem.text:
|
||||
elem.text = replace_duplicate_emojis(elem.text, emoji_map)
|
||||
elem.text = replace_duplicates_in_text(elem.text, emoji_replace_map, positions)
|
||||
if elem.tail:
|
||||
elem.tail = replace_duplicate_emojis(elem.tail, emoji_map)
|
||||
elem.tail = replace_duplicates_in_text(elem.tail, emoji_replace_map, positions)
|
||||
for child in elem:
|
||||
recursive_xml(child)
|
||||
recursive_xml(root)
|
||||
tree.write(dst, encoding="utf-8", xml_declaration=True)
|
||||
|
||||
class MyHTMLParser(HTMLParser):
|
||||
def __init__(self, emoji_map):
|
||||
def __init__(self, emoji_replace_map, positions):
|
||||
super().__init__()
|
||||
self.emoji_map = emoji_map
|
||||
self.emoji_replace_map = emoji_replace_map
|
||||
self.positions = positions
|
||||
self.result = []
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attr_str = ''.join([f' {k}="{v}"' for k, v in attrs])
|
||||
|
|
@ -205,7 +210,7 @@ class MyHTMLParser(HTMLParser):
|
|||
def handle_endtag(self, tag):
|
||||
self.result.append(f"</{tag}>")
|
||||
def handle_data(self, data):
|
||||
self.result.append(replace_duplicate_emojis(data, self.emoji_map))
|
||||
self.result.append(replace_duplicates_in_text(data, self.emoji_replace_map, self.positions))
|
||||
def handle_entityref(self, name):
|
||||
self.result.append(f"&{name};")
|
||||
def handle_charref(self, name):
|
||||
|
|
@ -213,10 +218,10 @@ class MyHTMLParser(HTMLParser):
|
|||
def get_html(self):
|
||||
return "".join(self.result)
|
||||
|
||||
def process_html_file(src, dst, emoji_map):
|
||||
def process_html_file(src, dst, emoji_replace_map, positions):
|
||||
with open(src, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
parser = MyHTMLParser(emoji_map)
|
||||
parser = MyHTMLParser(emoji_replace_map, positions)
|
||||
parser.feed(text)
|
||||
with open(dst, "w", encoding="utf-8") as f:
|
||||
f.write(parser.get_html())
|
||||
|
|
@ -239,20 +244,20 @@ def main():
|
|||
if not emoji_list:
|
||||
print(f" No emojis found in {filename}, skip.")
|
||||
continue
|
||||
emoji_map = build_emoji_replace_map(emoji_list, EMOJI_POOL)
|
||||
emoji_replace_map, positions = build_duplicate_emoji_mapping(emoji_list, EMOJI_POOL)
|
||||
ext = os.path.splitext(filename)[-1].lower()
|
||||
if ext == ".json":
|
||||
process_json_file(src_path, dst_path, emoji_map)
|
||||
process_json_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
elif ext == ".txt":
|
||||
process_txt_file(src_path, dst_path, emoji_map)
|
||||
process_txt_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
elif ext == ".md":
|
||||
process_md_file(src_path, dst_path, emoji_map)
|
||||
process_md_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
elif ext == ".csv":
|
||||
process_csv_file(src_path, dst_path, emoji_map)
|
||||
process_csv_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
elif ext == ".xml":
|
||||
process_xml_file(src_path, dst_path, emoji_map)
|
||||
process_xml_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
elif ext in (".html", ".htm"):
|
||||
process_html_file(src_path, dst_path, emoji_map)
|
||||
process_html_file(src_path, dst_path, emoji_replace_map, positions)
|
||||
else:
|
||||
print(f" Unsupported file type: {filename}")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue