From 9c142e3a2a8f1c7415511bd9d24f4790ce2dac88 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 8 Jan 2026 09:29:53 +0100 Subject: [PATCH] model-conversion : add warn about transformers mismatch (#18691) This commit adds a check comparing the installed transformers library with the transformers version that the original model supports. This check will be performed upon a model verification failure and prints a warning/hint to the user suggesting to install the correct version of the transformers library. The motivation for this change is that it is possible for the model verification to fail due to differences in the transformers library used and it might not be obvious that this could be the cause of the failure. With this warning the correct version can be checked and hopefully save time troubleshooting the cause of the verification failure. --- examples/model-conversion/Makefile | 2 +- .../scripts/causal/compare-logits.py | 10 ++-- .../model-conversion/scripts/utils/common.py | 54 +++++++++++++++++++ .../scripts/utils/semantic_check.py | 7 ++- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/examples/model-conversion/Makefile b/examples/model-conversion/Makefile index 359b9cfd8e..3b0505911d 100644 --- a/examples/model-conversion/Makefile +++ b/examples/model-conversion/Makefile @@ -61,7 +61,7 @@ causal-run-converted-model: @CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh causal-verify-logits: causal-run-original-model causal-run-converted-model - @./scripts/causal/compare-logits.py + @MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/compare-logits.py @MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH} causal-run-original-embeddings: diff --git a/examples/model-conversion/scripts/causal/compare-logits.py b/examples/model-conversion/scripts/causal/compare-logits.py index 1a933207d5..83bd14c659 100755 --- a/examples/model-conversion/scripts/causal/compare-logits.py +++ b/examples/model-conversion/scripts/causal/compare-logits.py @@ -3,10 +3,11 @@ import sys import numpy as np from pathlib import Path +import os # Add utils directory to path for direct script execution sys.path.insert(0, str(Path(__file__).parent.parent / "utils")) -from common import get_model_name_from_env_path, compare_tokens # type: ignore[import-not-found] +from common import get_model_name_from_env_path, compare_tokens, exit_with_warning # type: ignore[import-not-found] def quick_logits_check(pytorch_file, llamacpp_file): """Lightweight sanity check before NMSE""" @@ -38,6 +39,7 @@ def quick_logits_check(pytorch_file, llamacpp_file): return True def main(): + model_path = os.environ.get('MODEL_PATH') model_name = get_model_name_from_env_path('MODEL_PATH') data_dir = Path("data") pytorch_file = data_dir / f"pytorch-{model_name}.bin" @@ -62,8 +64,7 @@ def main(): print("šŸ” Token Comparison Check") print("=" * 40) if not compare_tokens(f"pytorch-{model_name}", f"llamacpp-{llamacpp_model_name}"): - print("\nāŒ Token mismatch detected") - sys.exit(1) + exit_with_warning("\nāŒ Token mismatch detected", model_path) print() print("šŸ” GGML Model Validation for model ", model_name) @@ -80,8 +81,7 @@ def main(): print(" Ok to proceed with NMSE check...") sys.exit(0) else: - print(f"āŒ NOK: Top 10 predictions don't match - generation will differ") - sys.exit(1) + exit_with_warning(f"āŒ NOK: Top 10 predictions don't match - generation will differ", model_path) if __name__ == "__main__": main() diff --git a/examples/model-conversion/scripts/utils/common.py b/examples/model-conversion/scripts/utils/common.py index 71761127bb..aa4bab2601 100644 --- a/examples/model-conversion/scripts/utils/common.py +++ b/examples/model-conversion/scripts/utils/common.py @@ -3,6 +3,9 @@ import os import sys import torch +import transformers +import json +import textwrap import numpy as np from pathlib import Path @@ -243,3 +246,54 @@ def compare_tokens(original, converted, type_suffix="", output_dir="data"): print(f" ... and {len(mismatches) - num_to_show} more mismatches") return False + + +def show_version_warning(current_version, model_version): + if not model_version: + return False + + try: + from packaging.version import parse, InvalidVersion + try: + return parse(current_version) < parse(model_version) + except InvalidVersion: + return current_version != model_version + except ImportError: + return current_version != model_version + +def get_model_transformers_version(model_path): + if not model_path: + return None + + config_path = Path(model_path) / "config.json" + if not config_path.is_file(): + return None + + try: + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + return config.get("transformers_version") + except (IOError, json.JSONDecodeError) as e: + print(f"Warning: Could not read or parse {config_path}: {e}", file=sys.stderr) + return None + +def exit_with_warning(message, model_path): + print(message) + + if model_path and transformers is not None: + model_transformers_version = get_model_transformers_version(model_path) + transformers_version = transformers.__version__ + if show_version_warning(transformers_version, model_transformers_version): + warning_message = f""" + ===================================================================== + Verification failure might be due to a transformers version mismatch: + + Current transformers version: {transformers_version} + Model's required version : {model_transformers_version} + + Consider installing the version specified by the model's config: + pip install transformers=={model_transformers_version} + ===================================================================== + """ + print(textwrap.dedent(warning_message)) + sys.exit(1) diff --git a/examples/model-conversion/scripts/utils/semantic_check.py b/examples/model-conversion/scripts/utils/semantic_check.py index 38b03ce4d2..73e20ea489 100644 --- a/examples/model-conversion/scripts/utils/semantic_check.py +++ b/examples/model-conversion/scripts/utils/semantic_check.py @@ -7,7 +7,7 @@ import importlib from pathlib import Path from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, AutoModel -from common import compare_tokens # type: ignore[import-not-found] +from common import compare_tokens, exit_with_warning # type: ignore[import-not-found] unreleased_model_name = os.getenv('UNRELEASED_MODEL_NAME') @@ -174,8 +174,7 @@ def main(): print("=" * 70) data_dir = python_emb_path.parent if not compare_tokens(python_model_name, cpp_model_name, type_suffix="-embeddings", output_dir=str(data_dir)): - print("\nāŒ Token mismatch detected") - exit(1) + exit_with_warning("\nāŒ Token mismatch detected", args.model_path) print() # Single prompt detailed comparison @@ -237,7 +236,7 @@ def main(): elif avg_cross_sim > 0.70: print("āš ļø FAIR: Models have some differences") else: - print("āŒ POOR: Models are significantly different") + exit_with_warning("āŒ POOR: Models are significantly different", args.model_path) if __name__ == "__main__": main()