support updated test structure

app.py

@@ -1,12 +1,16 @@
+import importlib.metadata
+import inspect
 import re
 import subprocess
 import sys
 from datetime import datetime
-from importlib import import_module
+from importlib import reload
 from pathlib import Path
 
 import gradio as gr
+import optimum.intel.utils.import_utils as import_utils
 import pandas as pd
+from packaging.requirements import Requirement
 
 if Path("optimum-intel").is_dir():
     subprocess.run(["git", "pull"], cwd="optimum-intel")
@@ -15,44 +19,78 @@ else:
 test_path = Path(__file__).parent / "optimum-intel" / "tests" / "openvino"
 sys.path.append(str(test_path))
 
-from test_modeling import *
-from test_diffusion import *
-from test_decoder import *
+# import test files globally. Importing inside a function with import_module is extremely
+# slow inside a gradio app
+import test_decoder
+import test_diffusion
+import test_modeling
+import test_seq2seq
 
-RESULT_FILE = "supported_models.md"
 
-def generate_model_list():
-    tests = []
+def get_supported_models_for_version(version):
+    """
+    Get supported architectures for a particular transformers version. Uses mocking to set the
+    transformers version to `version`.
+    """
+    import_utils._transformers_version = version
+
+    # Patch _transformers version
+    test_seq2seq._transformers_version = version
+    test_modeling._transformers_version = version
+    test_diffusion._transformers_version = version
+    test_decoder._transformers_version = version
+
+    # Re-import to refresh SUPPORTED_ARCHITECTURES
+    seq2seq = reload(test_seq2seq)
+    decoder = reload(test_decoder)
+    modeling = reload(test_modeling)
+    diffusion = reload(test_diffusion)
+
+    # Get SUPPORTED_ARCHITECUTRES for all modules
     d = {}
-    for item in globals().copy():
-        match = re.match("(OVModelFor.*IntegrationTest)", item) or re.match("(OVPipelineFor.*Test)", item)
-        if match:
-            tests.append(match.group(1))
-
-    for test in tests:
-        task = test.replace("IntegrationTest", "").replace("Test", "")
-        if "OVModelForCausalLM" in task:
-            cls = getattr(import_module("test_decoder"), test)
-        elif "OVModelFor" in task:
-            cls = getattr(import_module("test_modeling"), test)
-        else:
-            cls = getattr(import_module("test_diffusion"), test)
-        try:
-            print(cls.SUPPORTED_ARCHITECTURES)
-            d[task] = cls.SUPPORTED_ARCHITECTURES
-        except AttributeError:
-            print(cls.SUPPORTED_ARCHITECTURES_WITH_HIDDEN_STATES)
-            print(cls.SUPPORTED_ARCHITECTURES_WITH_ATTENTION)
-            d[task] = cls.SUPPORTED_ARCHITECTURES_WITH_ATTENTION + cls.SUPPORTED_ARCHITECTURES_WITH_HIDDEN_STATES
+    modules = [seq2seq, decoder, modeling, diffusion]
+    for mod in modules:
+        for name, obj in inspect.getmembers(mod):
+            if inspect.isclass(obj):
+                if re.match(r"(OVModelFor.*IntegrationTest)", name) or re.match(r"(OVPipelineFor.*Test)", name):
+                    task = name.replace("IntegrationTest", "").replace("Test", "")
+                    if "CustomTasks" not in task:
+                        d[task] = obj.SUPPORTED_ARCHITECTURES
+    return d  # sorted(set(all_archs))
 
-    with open(RESULT_FILE, "w") as f:
-        f.write(f"Updated at {datetime.now().strftime('%d %B %Y')}\n\n")
 
+def get_min_max_transformers():
+    """
+    Get minumum and maximum supported transformers version by currently installed optimum-intel
+    """
+    meta = importlib.metadata.metadata("optimum-intel")
+    requires = meta.get_all("Requires-Dist") or []
+    transformers_versions = [item for item in requires if "transformers" in item and "extra" not in item][0]
+    req = Requirement(transformers_versions)
+    maxver, minver = [ver.version for ver in list(req.specifier)]
+    return (minver, maxver)
+
+
+def generate_model_list():
+    RESULT_FILE = "supported_models.md"
+    minver, maxver = get_min_max_transformers()  # This returns a too large maxver but that is not a problem
+    versions = [minver, "4.53.0", maxver]
+
+    model_classes_with_models = {}
+    for v in versions:
+        model_classes_with_models_version = get_supported_models_for_version(v)
+        for item in model_classes_with_models_version:
+            model_classes_with_models.setdefault(item, set())
+            model_classes_with_models[item].update(model_classes_with_models_version[item])
+
+    with open(RESULT_FILE, "w") as f:
+        optimum_intel_version = importlib.metadata.version("optimum-intel")
+        f.write(f"Updated at {datetime.now().strftime('%d %B %Y')} using optimum-intel {optimum_intel_version}\n\n")
         summary = []
         all_archs = []
-        for archs in d.values():
+        for archs in model_classes_with_models.values():
             all_archs += archs
-        for title, supported_models in d.items():
+        for title, supported_models in model_classes_with_models.items():
             f.write(f"## {title}\n\n")
             for item in supported_models:
                 f.write(f" - {item}\n")