something

.gitignore

@@ -7,7 +7,7 @@ __pycache__/
 .vscode/
 
 eval-queue/
-# eval-results/
+eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/

eval-results/.gitattributes

@@ -9,6 +9,7 @@
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
@@ -53,3 +54,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results/demo-leaderboard/gpt2-demo/results_2023-11-21T18-10-08.json

@@ -1,7 +1,6 @@
 {
     "config": {
-        "model_dtype": "torch.float16", 
-        "model_name": "demo-leaderboard/gpt2-demo",
+        "model_name": "H2O/Mistral-7B",
         "model_sha": "ac3299b02780836378b9e1e68c6eead546e89f90",
         "method": "H2O"
     },

eval-results/demo-leaderboard/gpt2-demo/results_2023-11-22 15:46:20.425378.json

@@ -32,7 +32,7 @@
     "bootstrap_iters": 100000,
     "description_dict": null,
     "model_dtype": "bfloat16",
-    "model_name": "demo-leaderboard/gpt2-demo",
+    "model_name": "H2O/Llama3.1-8B",
     "model_sha": "main"
   }
 }

src/about.py

@@ -14,6 +14,8 @@ class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task2 = Task("system", "latency", "E2E Latency")
+    task3 = Task("system", "throughput", "E2E Throughput")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -28,9 +28,9 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 auto_eval_column_dict.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
 
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-auto_eval_column_dict.append(["latency", ColumnContent, ColumnContent("E2E Latency", "number", True)])
-auto_eval_column_dict.append(["throughput", ColumnContent, ColumnContent("E2E Throughput", "number", True)])
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# auto_eval_column_dict.append(["latency", ColumnContent, ColumnContent("E2E Latency", "number", True)])
+# auto_eval_column_dict.append(["throughput", ColumnContent, ColumnContent("E2E Throughput", "number", True)])
 
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
@@ -67,8 +67,8 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="Llama3.1-8B", symbol="🟢")
-    FT = ModelDetails(name="Mistral-7B-Instruct-v0.1", symbol="🔶")
+    LLAMA8B = ModelDetails(name="Llama3.1-8B", symbol="🟢")
+    MISTRAL7B = ModelDetails(name="Mistral-7B-Instruct-v0.1", symbol="🔶")
     #Add more models
     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
     RL = ModelDetails(name="RL-tuned", symbol="🟦")
@@ -79,10 +79,11 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
+        print("wwww type", type)
+        if "Mistral-7B" in type or "🔶" in type:
+            return ModelType.MISTRAL7B
+        if "Llama3.1-8B" in type or "🟢" in type:
+            return ModelType.LLAMA8B
         if "RL-tuned" in type or "🟦" in type:
             return ModelType.RL
         if "instruction-tuned" in type or "⭕" in type:

src/leaderboard/read_evals.py

@@ -103,6 +103,7 @@ class EvalResult:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
+            print("WTF", self.model_type)
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
@@ -113,7 +114,6 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -124,7 +124,6 @@ class EvalResult:
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
@@ -132,7 +131,6 @@ class EvalResult:
         }
 
         for task in Tasks:
-            print(task.value.col_name)
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
@@ -144,6 +142,8 @@ def get_request_file_for_model(requests_path, model_name, method, precision):
         requests_path,
         f"{model_name}_eval_request_*.json",
     )
+    print(request_files)
+    
     request_files = glob.glob(request_files)
 
     # Select correct request file (precision)
@@ -154,7 +154,6 @@ def get_request_file_for_model(requests_path, model_name, method, precision):
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
     return request_file

src/populate.py

@@ -15,7 +15,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
     df = pd.DataFrame.from_records(all_data_json)
     print(df.columns)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced