update

app.py

@@ -17,17 +17,23 @@ CANDIDATE_MAX_LENGTH = 256
 DEFAULT_CANDIDATE_MAX_LENGTH = 128
 FUSER_MAX_NEW_TOKENS = 512
 DEFAULT_FUSER_MAX_NEW_TOKENS = 256
-EXAMPLES_DATASET = load_dataset("llm-blender/mix-instruct", split='validation', streaming=True)
-SHUFFLED_EXAMPLES_DATASET = EXAMPLES_DATASET.shuffle(seed=42, buffer_size=1000)
-EXAMPLES = []
-CANDIDATE_EXAMPLES = {}
-for example in SHUFFLED_EXAMPLES_DATASET.take(100):
-    EXAMPLES.append([
+
+
+# MIX-INSTRUCT
+EXAMPLES_DATASET = load_dataset("llm-blender/mix-instruct", split='validation')
+SHUFFLED_EXAMPLES_DATASET = EXAMPLES_DATASET.shuffle(seed=42)
+MIX_INSTRUCT_EXAMPLES = []
+CANDIDATE_MAP = {}
+for i, example in enumerate(SHUFFLED_EXAMPLES_DATASET):
+    MIX_INSTRUCT_EXAMPLES.append([
         example['instruction'],
         example['input'],
     ])
-    CANDIDATE_EXAMPLES[example['instruction']+example['input']] = example['candidates']
+    CANDIDATE_MAP[example['instruction']+example['input']] = example['candidates']
+    if i > 100:
+        break
 
+# HHH ALIGNMENT
 HHH_EXAMPLES = []
 subsets = ['harmless', 'helpful', 'honest', 'other']
 random.seed(42)
@@ -53,6 +59,7 @@ for subset in subsets:
 def get_hhh_examples(subset, instruction, response1, response2, dummy_text):
     return instruction, response1, response2
 
+# MT_BENCH_HUMAN_JUDGMENTS
 MT_BENCH_HUMAN_JUDGE_EXAMPLES = []
 dataset = load_dataset("lmsys/mt_bench_human_judgments")
 for example in dataset['human']:
@@ -101,15 +108,17 @@ def save_llm_output(selected_base_llm_name, selected_base_llm_output, llm_output
 
 def get_preprocess_examples(inst, input):
     # get the num_of_base_llms
-    candidates = CANDIDATE_EXAMPLES[inst+input]
+    candidates = CANDIDATE_MAP[inst+input]
     num_candiates = len(candidates)
     dummy_text = inst+input
     return inst, input, num_candiates, dummy_text
 
-def update_base_llm_dropdown_along_examples(dummy_text):
-    candidates = CANDIDATE_EXAMPLES[dummy_text]
+def update_base_llm_dropdown_along_examples(inst, input):
+    candidates = CANDIDATE_MAP[inst+input]
     ex_llm_outputs = {f"LLM-{i+1}": candidates[i]['text'] for i in range(len(candidates))}
-    return ex_llm_outputs, "", ""
+    k = len(candidates)
+    return ex_llm_outputs, "", "", \
+        gr.Dropdown(choices=[f"LLM-{i+1}" for i in range(k)], value=f"LLM-1" if k >= 1 else "", visible=True)
     
 def check_save_ranker_inputs(inst, input, llm_outputs, blender_config):
     if not inst and not input:
@@ -125,10 +134,11 @@ def check_save_ranker_inputs(inst, input, llm_outputs, blender_config):
     }
 
 def check_fuser_inputs(blender_state, blender_config, ranks):
-    if not (blender_state.get("inst", None) or blender_state.get("input", None)):
-        raise gr.Error("Please enter instruction or input context")
     if "candidates" not in blender_state or len(ranks)==0:
         raise gr.Error("Please rank LLM outputs first")
+    if not (blender_state.get("inst", None) or blender_state.get("input", None)):
+        raise gr.Error("Please enter instruction or input context")
+    
     return 
 
 def llms_rank(inst, input, llm_outputs, blender_config):
@@ -259,7 +269,7 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         
         examples_dummy_textbox = gr.Textbox(lines=1, label="", placeholder="", show_label=False, visible=False)     
         batch_examples = gr.Examples(
-            examples=EXAMPLES,
+            examples=MIX_INSTRUCT_EXAMPLES,
             fn=get_preprocess_examples,
             cache_examples=True,
             examples_per_page=5,
@@ -267,7 +277,7 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
             outputs=[inst_textbox, input_textbox, base_llms_num, examples_dummy_textbox],
         )
             
-        base_llms_num.change(
+        base_llms_num.input(
             fn=update_base_llms_num,
             inputs=[base_llms_num, saved_llm_outputs],
             outputs=[selected_base_llm_name_dropdown, saved_llm_outputs],
@@ -275,8 +285,8 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         
         examples_dummy_textbox.change(
             fn=update_base_llm_dropdown_along_examples,
-            inputs=[examples_dummy_textbox],
-            outputs=[saved_llm_outputs, rank_outputs, fuser_outputs],
+            inputs=[inst_textbox, input_textbox],
+            outputs=[saved_llm_outputs, rank_outputs, fuser_outputs, selected_base_llm_name_dropdown],
         ).then(
             fn=display_llm_output,
             inputs=[saved_llm_outputs, selected_base_llm_name_dropdown],

requirements.txt

@@ -1,2 +1 @@
-llm_blender @ git+https://github.com/yuchenlin/LLM-Blender.git@main
-gdown
+llm_blender @ git+https://github.com/yuchenlin/LLM-Blender.git@main