Add additional checks

app.py

@@ -42,9 +42,12 @@ for name in dataset_names:
     )
     dataset_data[name] = {
         "ds": ds,
-        "word_rep_ratios": np.random.randn(len(ds)),
+        "check_word_number_criteria": np.array(ds["check_word_number_criteria"]),
         "check_char_repetition_criteria": np.array(ds["check_char_repetition_criteria"]),
         "check_flagged_words_criteria": np.array(ds["check_flagged_words_criteria"]),
+        "check_stop_word_ratio_criteria": np.array(ds["check_stop_word_ratio_criteria"]),
+        "check_perplexity_criteria": np.array(ds["check_perplexity_criteria"]),
+        "check_language_criteria": np.array(ds["check_language_criteria"]),
     }
 
 def plt_plot(criteria, dataset, threshold):
@@ -83,6 +86,18 @@ def check_filtered(criteria, dataset, threshold):
 with gr.Blocks() as demo:
     dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
 
+    with gr.Tab("Number of Words Criteria"):
+        # plot some random data
+        plot = gr.Plot()
+        threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
+        calculate = gr.Button("Calculate")
+        check = gr.Button("Check Filtered Data")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
+        plot_fn = partial(plt_plot, "check_word_number_criteria")
+        calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "check_word_number_criteria")
+        check.click(check_fn, [dataset, threshold], filtered_data)
+
     with gr.Tab("Character Repetition Criteria"):
         # plot some random data
         plot = gr.Plot()
@@ -95,18 +110,18 @@ with gr.Blocks() as demo:
         check_fn = partial(check_filtered, "check_char_repetition_criteria")
         check.click(check_fn, [dataset, threshold], filtered_data)
     
-    with gr.Tab("Word Repetition Criteria"):# plot some random data
+    with gr.Tab("Stop Word Ratio Criteria"):
         plot = gr.Plot()
         threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
         calculate = gr.Button("Calculate")
         check = gr.Button("Check Filtered Data")
         filtered_data = gr.Textbox(lines=5, label="Filtered Data")
-        plot_fn = partial(plt_plot, "word_rep_ratios")
+        plot_fn = partial(plt_plot, "check_stop_word_ratio_criteria")
         calculate.click(plot_fn, [dataset, threshold], plot)
-        check_fn = partial(check_filtered, "word_rep_ratios")
+        check_fn = partial(check_filtered, "check_stop_word_ratio_criteria")
         check.click(check_fn, [dataset, threshold], filtered_data)
     
-    with gr.Tab("Flagged Word Criteria"):# plot some random data
+    with gr.Tab("Flagged Word Criteria"):
         plot = gr.Plot()
         threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
         calculate = gr.Button("Calculate")
@@ -117,5 +132,27 @@ with gr.Blocks() as demo:
         check_fn = partial(check_filtered, "check_flagged_words_criteria")
         check.click(check_fn, [dataset, threshold], filtered_data)
 
+    with gr.Tab("Perplexity Criteria"):
+        plot = gr.Plot()
+        threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
+        calculate = gr.Button("Calculate")
+        check = gr.Button("Check Filtered Data")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
+        plot_fn = partial(plt_plot, "check_perplexity_criteria")
+        calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "check_perplexity_criteria")
+        check.click(check_fn, [dataset, threshold], filtered_data)
+    
+    with gr.Tab("Language Detection Criteria"):
+        plot = gr.Plot()
+        threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
+        calculate = gr.Button("Calculate")
+        check = gr.Button("Check Filtered Data")
+        filtered_data = gr.Textbox(lines=5, label="Filtered Data")
+        plot_fn = partial(plt_plot, "check_language_criteria")
+        calculate.click(plot_fn, [dataset, threshold], plot)
+        check_fn = partial(check_filtered, "check_language_criteria")
+        check.click(check_fn, [dataset, threshold], filtered_data)
+
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -1,2 +1,2 @@
 scrubadub
-squeakily
+git+https://github.com/CarperAI/squeakily.git