replica
/
llama.cpp
mirror of https://github.com/ggml-org/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

convert : only check for tokenizer folder if we need it (#14704)

This commit is contained in:
Sigbjørn Skjæret 2025-07-16 08:52:04 +02:00 committed by GitHub
parent cf91f217f1
commit 4b91d6f71f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
convert_hf_to_gguf_update.py
View File

@ -240,11 +240,6 @@ for model in [*pre_computed_hashes, *all_models]:
if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM: if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM:
continue continue
# Skip if the tokenizer folder does not exist or there are other download issues previously
if not os.path.exists(f"models/tokenizers/{name}"):
logger.warning(f"Directory for tokenizer {name} not found. Skipping...")
continue
# create the tokenizer # create the tokenizer
if chkhsh is not None: if chkhsh is not None:
# if the model has a pre-computed hash, use it # if the model has a pre-computed hash, use it
@ -254,6 +249,12 @@ for model in [*pre_computed_hashes, *all_models]:
chkhsh = existing_models[name] chkhsh = existing_models[name]
else: else:
# otherwise, compute the hash of the tokenizer # otherwise, compute the hash of the tokenizer
# Skip if the tokenizer folder does not exist or there are other download issues previously
if not os.path.exists(f"models/tokenizers/{name}"):
logger.warning(f"Directory for tokenizer {name} not found. Skipping...")
continue
try: try:
logger.info(f"Loading tokenizer from {f'models/tokenizers/{name}'}...") logger.info(f"Loading tokenizer from {f'models/tokenizers/{name}'}...")
if name == "t5": if name == "t5":