Update evaluator.py

evaluator.py

@@ -3,13 +3,13 @@
 import numpy as np
 from collections import defaultdict
 from datasets import load_dataset
-from tqdm.auto import tqdm
+from transformers import pipeline
+import os
 
 from sacrebleu.metrics import BLEU, CHRF
 from rouge_score import rouge_scorer
 import Levenshtein
 
-from transformers import pipeline
 from transformers.models.whisper.english_normalizer import BasicTextNormalizer
 
 
@@ -100,8 +100,13 @@ def evaluate_model(
         "averages": {<metric1>: val, ...}
       }
     """
+    # Get Hugging Face token from environment variable
+    hf_token = os.getenv("HF_TOKEN")
+    if not hf_token:
+        raise ValueError("Hugging Face token (HF_TOKEN) is not set. Please set it as an environment variable.")
+
     # 1) load test split
-    ds = load_dataset(dataset_name, split=split)
+    ds = load_dataset(dataset_name, split=split, use_auth_token=hf_token)
 
     # 2) build pipeline
     nlp = pipeline(task, model=model_name, device=device)