Update emoji_manager.py

emojis/emoji_manager.py

@@ -4,41 +4,37 @@ import json
 import csv
 import xml.etree.ElementTree as ET
 from html.parser import HTMLParser
+from collections import Counter, deque
 
-# Android-safe emoji sets by categories
+# Android-safe emoji pool（可自定义扩充）
 EMOJI_POOL = [
     # 动物与自然
     "🐶", "🐱", "🐭", "🐹", "🐰", "🦊", "🐻", "🐼", "🐨", "🐯",
     "🦁", "🐮", "🐷", "🐸", "🐵", "🦄", "🐔", "🐧", "🐦", "🐤",
     "🐝", "🐞", "🦋", "🐢", "🐍", "🐠", "🐬", "🐳", "🦑", "🦀",
     "🌸", "🌻", "🌹", "🌺", "🌼", "🌷", "🌱", "🌲", "🌳", "🌴",
-
     # 食物与饮料
     "🍏", "🍎", "🍐", "🍊", "🍋", "🍌", "🍉", "🍇", "🍓", "🫐",
     "🍈", "🍒", "🍑", "🥭", "🍍", "🥥", "🥝", "🍅", "🍆", "🥑",
     "🥦", "🥬", "🥕", "🌽", "🌶️", "🧄", "🧅", "🥔", "🥚", "🍞",
     "🥐", "🥯", "🥨", "🧀", "🥓", "🍗", "🍖", "🌭", "🍔", "🍟",
-
     # 活动
     "⚽", "🏀", "🏈", "⚾", "🎾", "🏐", "🏉", "🎱", "🏓", "🏸",
     "🥅", "🏒", "🏑", "🏏", "⛳", "🏹", "🎣", "🤿", "🥊", "🥋",
     "🎯", "🎳", "🪁", "🛹", "🥌", "🛷", "⛷️", "🏂", "🏄‍♂️", "🏊‍♀️",
     "🎮", "🎲", "🧩", "🀄", "♟️", "🃏", "🧗", "🏆", "🥇", "🥈", "🥉",
-
     # 物体
     "⌚", "📱", "💻", "🖨️", "🕹️", "🎮", "📷", "📸", "📹", "🎥",
     "📺", "📻", "🎙️", "🎚️", "🎛️", "☎️", "📞", "📟", "📠", "🔋",
     "🔌", "💡", "🔦", "🕯️", "🛢️", "💰", "💳", "🗝️", "🔑", "🚪",
     "🔒", "🔓", "🔑", "🔐", "🛡️", "🧲", "⚖️", "🔗", "🧰", "🔧",
     "🔨", "🪓", "⛏️", "🛠️", "🗡️", "⚔️", "🔫", "🏹", "🛏️", "🛋️",
-
     # 旅行与地点
     "🚗", "🚕", "🚙", "🚌", "🚎", "🏎️", "🚓", "🚑", "🚒", "🚐",
     "🛻", "🚚", "🚛", "🚜", "🏍️", "🛵", "🚲", "🛴", "🚨", "🚔",
     "🚦", "🚧", "🛣️", "🛤️", "✈️", "🛩️", "🚁", "🚀", "🛸", "⛵",
     "🚢", "🛳️", "⛴️", "🚤", "🛥️", "🗽", "🗼", "🏰", "🏯", "🌋",
     "🗻", "🏕️", "🏞️", "🏜️", "🏝️", "🏖️", "🏟️", "🏛️", "🏗️",
-
     # 人物
     "😀", "😃", "😄", "😁", "😆", "😅", "😂", "🤣", "😊", "😇",
     "🙂", "🙃", "😉", "😌", "😍", "🥰", "😘", "😗", "😙", "😚",
@@ -53,7 +49,6 @@ EMOJI_POOL = [
     "🖕", "👇", "☝️", "🫳", "🫴", "💪", "🦾", "🦿", "🦵", "🦶",
     "👀", "👁️", "👅", "👄", "🧑", "👦", "👧", "👨", "👩", "🧓",
     "👴", "👵", "🧔", "👱‍♂️", "👱‍♀️", "👨‍🦰", "👩‍🦰", "👨‍🦱", "👩‍🦱",
-
     # 符号
     "❤️", "🧡", "💛", "💚", "💙", "💜", "🤎", "🖤", "🤍", "💔",
     "❣️", "💕", "💞", "💓", "💗", "💖", "💘", "💝", "💟", "☮️",
@@ -68,11 +63,8 @@ EMOJI_POOL = [
     "🔫", "🏹", "🛏️", "🛋️", "🛒", "🧺", "🧻", "🧼", "🪥", "🧽", "🪣", "🧴", "🧷", "🧹", "🪒", "🧯", "🛎️"
 ]
 
-# Regex pattern for emoji (cover most cases, including ZWJ sequences)
 EMOJI_REGEX = re.compile(
     r"("
-    r"(?:[\U0001F3FB-\U0001F3FF])|"  # skin tone
-    r"(?:[\U0001F9B0-\U0001F9B3])|"  # hair
     r"(?:[\U0001F1E6-\U0001F1FF]{2})|"  # flags
     r"(?:[\U0001F600-\U0001F64F])|"
     r"(?:[\U0001F300-\U0001F5FF])|"
@@ -87,7 +79,7 @@ EMOJI_REGEX = re.compile(
     r"(?:[\u2700-\u27BF])|"
     r"(?:[\u2300-\u23FF])|"
     r"(?:[\u2B05-\u2B07])|"
-    r"(?:\u200D)"  # zwj joiner
+    r"(?:\u200D)"
     r")+",
     re.UNICODE
 )
@@ -98,106 +90,119 @@ def extract_emojis(text):
     # Find all emoji/sequence, keeping order, including duplicates
     return [m.group(0) for m in EMOJI_REGEX.finditer(text)]
 
-def find_duplicates(emoji_list):
+def build_duplicate_emoji_mapping(emoji_list, emoji_pool):
-    # Return a set of emojis that occur more than once
+    """
-    from collections import Counter
+    保留每个emoji首次出现不变，仅对重复出现的emoji做替换为pool中未用过的emoji。
-    count = Counter(emoji_list)
+    若emoji_pool用尽则循环使用。
-    return {em for em, c in count.items() if c > 1}
+    返回字典：{原emoji: [None, 替换1, 替换2...]}，None表示首个不换。
+    """
+    counts = Counter(emoji_list)
+    duplicates = {em for em, c in counts.items() if c > 1}
+    # 统计每个emoji出现了几次
+    positions = {}
+    for idx, em in enumerate(emoji_list):
+        positions.setdefault(em, []).append(idx)
+    # 新emoji分配池
+    used = set(emoji_list)
+    pool = deque([e for e in emoji_pool if e not in used])
+    emoji_replace_map = {}
+    for em in duplicates:
+        occurrence = len(positions[em])
+        repls = [None]  # 第一次出现不替换
+        for _ in range(occurrence - 1):
+            if not pool:
+                pool = deque([e for e in emoji_pool if e not in used])
+            if pool:
+                new_em = pool.popleft()
+                repls.append(new_em)
+                pool.append(new_em)
+            else:
+                # 纯保险，理论不会到这里
+                repls.append(em)
+        emoji_replace_map[em] = repls
+    return emoji_replace_map, positions
 
-def build_emoji_replace_map(emoji_list, emoji_pool):
+def replace_duplicates_in_text(text, emoji_replace_map, positions):
-    # Only map duplicate emojis; unique ones stay unchanged
+    """
-    from collections import Counter, deque
+    仅替换重复的emoji，第一次出现保留，后续出现才换。
-
+    """
-    counter = Counter(emoji_list)
-    dups = [em for em in emoji_list if counter[em] > 1]
-    seen = set()
-    dups_unique = []
-    for em in dups:
-        if em not in seen:
-            dups_unique.append(em)
-            seen.add(em)
-    pool = deque([e for e in emoji_pool if e not in emoji_list])  # don't use emojis already present
-    emoji_map = {}
-    for em in dups_unique:
-        if not pool:
-            pool = deque([e for e in emoji_pool if e not in emoji_list])
-        if pool:
-            emoji_map[em] = pool.popleft()
-    return emoji_map
-
-def replace_duplicate_emojis(text, emoji_map):
-    # Only replace duplicates (the 2nd, 3rd, ...) occurrence for each
-    # First occurrence stays, others replaced
     matches = list(EMOJI_REGEX.finditer(text))
-    new_text = []
+    # 记录每个emoji已出现次数
+    occur_dict = {em:0 for em in emoji_replace_map}
     last_idx = 0
-    seen = {}
+    result = []
     for m in matches:
         em = m.group(0)
-        new_text.append(text[last_idx:m.start()])
+        result.append(text[last_idx:m.start()])
-        seen.setdefault(em, 0)
+        if em in emoji_replace_map:
-        seen[em] += 1
+            occur_dict[em] += 1
-        if em in emoji_map and seen[em] > 1:
+            # 首次出现用原emoji，后续用映射
-            new_text.append(emoji_map[em])
+            idx = occur_dict[em]
+            repl = emoji_replace_map[em][idx-1]  # idx-1：第0次None, 从1开始替换
+            if repl is None:
+                result.append(em)
+            else:
+                result.append(repl)
         else:
-            new_text.append(em)
+            result.append(em)
         last_idx = m.end()
-    new_text.append(text[last_idx:])
+    result.append(text[last_idx:])
-    return ''.join(new_text)
+    return ''.join(result)
 
-def process_json_file(src, dst, emoji_map):
+def process_json_file(src, dst, emoji_replace_map, positions):
     with open(src, "r", encoding="utf-8") as f:
         data = json.load(f)
-    def recursive_replace(obj):
+    def recursive(obj):
         if isinstance(obj, str):
-            return replace_duplicate_emojis(obj, emoji_map)
+            return replace_duplicates_in_text(obj, emoji_replace_map, positions)
         elif isinstance(obj, list):
-            return [recursive_replace(item) for item in obj]
+            return [recursive(o) for o in obj]
         elif isinstance(obj, dict):
-            return {k: recursive_replace(v) for k, v in obj.items()}
+            return {k: recursive(v) for k, v in obj.items()}
         return obj
-    new_data = recursive_replace(data)
+    data_new = recursive(data)
     with open(dst, "w", encoding="utf-8") as f:
-        json.dump(new_data, f, ensure_ascii=False, indent=2)
+        json.dump(data_new, f, ensure_ascii=False, indent=2)
 
-def process_csv_file(src, dst, emoji_map):
+def process_csv_file(src, dst, emoji_replace_map, positions):
     with open(src, "r", encoding="utf-8", newline='') as f:
         reader = list(csv.reader(f))
     new_rows = []
     for row in reader:
-        new_row = [replace_duplicate_emojis(cell, emoji_map) for cell in row]
+        new_row = [replace_duplicates_in_text(cell, emoji_replace_map, positions) for cell in row]
         new_rows.append(new_row)
     with open(dst, "w", encoding="utf-8", newline='') as f:
         writer = csv.writer(f)
         writer.writerows(new_rows)
 
-def process_txt_file(src, dst, emoji_map):
+def process_txt_file(src, dst, emoji_replace_map, positions):
     with open(src, "r", encoding="utf-8") as f:
         text = f.read()
-    new_text = replace_duplicate_emojis(text, emoji_map)
+    new_text = replace_duplicates_in_text(text, emoji_replace_map, positions)
     with open(dst, "w", encoding="utf-8") as f:
         f.write(new_text)
 
-def process_md_file(src, dst, emoji_map):
+def process_md_file(src, dst, emoji_replace_map, positions):
-    process_txt_file(src, dst, emoji_map)
+    process_txt_file(src, dst, emoji_replace_map, positions)
 
-def process_xml_file(src, dst, emoji_map):
+def process_xml_file(src, dst, emoji_replace_map, positions):
     tree = ET.parse(src)
     root = tree.getroot()
     def recursive_xml(elem):
         if elem.text:
-            elem.text = replace_duplicate_emojis(elem.text, emoji_map)
+            elem.text = replace_duplicates_in_text(elem.text, emoji_replace_map, positions)
         if elem.tail:
-            elem.tail = replace_duplicate_emojis(elem.tail, emoji_map)
+            elem.tail = replace_duplicates_in_text(elem.tail, emoji_replace_map, positions)
         for child in elem:
             recursive_xml(child)
     recursive_xml(root)
     tree.write(dst, encoding="utf-8", xml_declaration=True)
 
 class MyHTMLParser(HTMLParser):
-    def __init__(self, emoji_map):
+    def __init__(self, emoji_replace_map, positions):
         super().__init__()
-        self.emoji_map = emoji_map
+        self.emoji_replace_map = emoji_replace_map
+        self.positions = positions
         self.result = []
     def handle_starttag(self, tag, attrs):
         attr_str = ''.join([f' {k}="{v}"' for k, v in attrs])
@@ -205,7 +210,7 @@ class MyHTMLParser(HTMLParser):
     def handle_endtag(self, tag):
         self.result.append(f"</{tag}>")
     def handle_data(self, data):
-        self.result.append(replace_duplicate_emojis(data, self.emoji_map))
+        self.result.append(replace_duplicates_in_text(data, self.emoji_replace_map, self.positions))
     def handle_entityref(self, name):
         self.result.append(f"&{name};")
     def handle_charref(self, name):
@@ -213,10 +218,10 @@ class MyHTMLParser(HTMLParser):
     def get_html(self):
         return "".join(self.result)
 
-def process_html_file(src, dst, emoji_map):
+def process_html_file(src, dst, emoji_replace_map, positions):
     with open(src, "r", encoding="utf-8") as f:
         text = f.read()
-    parser = MyHTMLParser(emoji_map)
+    parser = MyHTMLParser(emoji_replace_map, positions)
     parser.feed(text)
     with open(dst, "w", encoding="utf-8") as f:
         f.write(parser.get_html())
@@ -239,20 +244,20 @@ def main():
         if not emoji_list:
             print(f"  No emojis found in {filename}, skip.")
             continue
-        emoji_map = build_emoji_replace_map(emoji_list, EMOJI_POOL)
+        emoji_replace_map, positions = build_duplicate_emoji_mapping(emoji_list, EMOJI_POOL)
         ext = os.path.splitext(filename)[-1].lower()
         if ext == ".json":
-            process_json_file(src_path, dst_path, emoji_map)
+            process_json_file(src_path, dst_path, emoji_replace_map, positions)
         elif ext == ".txt":
-            process_txt_file(src_path, dst_path, emoji_map)
+            process_txt_file(src_path, dst_path, emoji_replace_map, positions)
         elif ext == ".md":
-            process_md_file(src_path, dst_path, emoji_map)
+            process_md_file(src_path, dst_path, emoji_replace_map, positions)
         elif ext == ".csv":
-            process_csv_file(src_path, dst_path, emoji_map)
+            process_csv_file(src_path, dst_path, emoji_replace_map, positions)
         elif ext == ".xml":
-            process_xml_file(src_path, dst_path, emoji_map)
+            process_xml_file(src_path, dst_path, emoji_replace_map, positions)
         elif ext in (".html", ".htm"):
-            process_html_file(src_path, dst_path, emoji_map)
+            process_html_file(src_path, dst_path, emoji_replace_map, positions)
         else:
             print(f"  Unsupported file type: {filename}")