Add ability to check examples that would be filtered

app.py

@@ -1,10 +1,10 @@
 import os
+import random
 import gradio as gr
 import matplotlib.pyplot as plt
 import numpy as np
 from functools import partial
 from datasets import load_dataset
-from pathlib import Path
 
 dataset_names = [
     "AI4Code",
@@ -43,13 +43,13 @@ for name in dataset_names:
     dataset_data[name] = {
         "ds": ds,
         "word_rep_ratios": np.random.randn(len(ds)),
-        "char_rep_ratios": np.array(ds["check_char_repetition_criteria"]),
-        "flagged_word_ratios": np.array(ds["check_flagged_words_criteria"]),
+        "check_char_repetition_criteria": np.array(ds["check_char_repetition_criteria"]),
+        "check_flagged_words_criteria": np.array(ds["check_flagged_words_criteria"]),
     }
 
-def plt_plot(ratio, dataset, threshold):
+def plt_plot(criteria, dataset, threshold):
     plt.close("all")
-    x = dataset_data[dataset][ratio]
+    x = dataset_data[dataset][criteria]
     # calculate percentage of data that will be removed given threshold
     perc = np.sum(x > threshold) / len(x)
     # create a figure
@@ -69,35 +69,53 @@ def plt_plot(ratio, dataset, threshold):
     plt.tight_layout()
     return fig
 
-def check_filtered():
-    ...
+def check_filtered(criteria, dataset, threshold):
+    ds = dataset_data[dataset]["ds"]
+
+    filtered_ds = ds.filter(lambda x: x[criteria] > threshold)
+    if len(filtered_ds) == 0:
+        return "No examples found"
+    # get random sample of 1
+    sample = filtered_ds.select([random.randint(0, len(filtered_ds) - 1)])["text"][0]
+
+    return sample
 
 with gr.Blocks() as demo:
     dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
-    print(dataset.value)
 
-    with gr.Tab("Character Repetition Ratio"):
+    with gr.Tab("Character Repetition Criteria"):
         # plot some random data
         plot = gr.Plot()
         threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
         calculate = gr.Button("Calculate")
         check = gr.Button("Check Filtered Data")
-        plot_fn = partial(plt_plot, "char_rep_ratios")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
+        plot_fn = partial(plt_plot, "check_char_repetition_criteria")
         calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "check_char_repetition_criteria")
+        check.click(check_fn, [dataset, threshold], filtered_data)
     
-    with gr.Tab("Word Repetition Ratio"):# plot some random data
+    with gr.Tab("Word Repetition Criteria"):# plot some random data
         plot = gr.Plot()
         threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
         calculate = gr.Button("Calculate")
+        check = gr.Button("Check Filtered Data")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
         plot_fn = partial(plt_plot, "word_rep_ratios")
         calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "word_rep_ratios")
+        check.click(check_fn, [dataset, threshold], filtered_data)
     
-    with gr.Tab("Flagged Word Ratio"):# plot some random data
+    with gr.Tab("Flagged Word Criteria"):# plot some random data
         plot = gr.Plot()
         threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
         calculate = gr.Button("Calculate")
-        plot_fn = partial(plt_plot, "flagged_word_ratios")
+        check = gr.Button("Check Filtered Data")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
+        plot_fn = partial(plt_plot, "check_flagged_words_criteria")
         calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "check_flagged_words_criteria")
+        check.click(check_fn, [dataset, threshold], filtered_data)
 
 if __name__ == "__main__":
     demo.launch()