safety

benchmarks/prompt-injections/allenai-wildjailbreak-judges-metrics.csv

@@ -21,4 +21,4 @@ allenai-wildjailbreak,qwen-2.5-72b-instruct-turbo,Qwen 2.5 72B Instruct,0.943020
 allenai-wildjailbreak,qwen-2.5-7b-instruct-turbo,Qwen 2.5 7B Instruct,0.8493909191583611,0.6977857142857142,1.82883763852702,4041.7311811447144,2210,1666
 allenai-wildjailbreak,deepseek-v3,DeepSeek V3,0.9037900874635568,0.7643452380952381,4.896477589671968,10821.215473175049,2210,1847
 allenai-wildjailbreak,deepseek-r1,DeepSeek R1,0.8760195758564437,0.7432261904761905,16.521143167159135,36511.72639942169,2210,1754
-allenai-wildjailbreak,qualifire-eval,Qualifire,0.9290187891440501,0.8211904761904762,0.9598423846706545,2121.2516701221466,2210,1938
+allenai-wildjailbreak,qualifire-eval,Qualifire,0.9290187891440501,0.8211904761904762,0.04598423846706545,2121.2516701221466,2210,1938

benchmarks/prompt-injections/jackhhao-jailbreak-classification-judges-metrics.csv

@@ -21,4 +21,4 @@ jackhhao-jailbreak-classification,qwen-2.5-72b-instruct-turbo,Qwen 2.5 72B Instr
 jackhhao-jailbreak-classification,qwen-2.5-7b-instruct-turbo,Qwen 2.5 7B Instruct,0.8553459119496856,0.8144118851260456,5.388962885805669,1411.9082760810852,262,216
 jackhhao-jailbreak-classification,deepseek-v3,DeepSeek V3,0.9543859649122807,0.9485582265894601,7.477735892506956,1959.1668038368225,262,249
 jackhhao-jailbreak-classification,deepseek-r1,DeepSeek R1,0.950354609929078,0.9454290226355502,17.93112579738821,4697.95495891571,262,248
-jackhhao-jailbreak-classification,qualifire-eval,Qualifire,0.9856115107913669,0.9846756740948704,1.0589720984451643,277.45068979263306,262,258
+jackhhao-jailbreak-classification,qualifire-eval,Qualifire,0.9856115107913669,0.9846756740948704,0.0589720984451643,277.45068979263306,262,258

benchmarks/safety/TOXIGEN-judges-metrics.csv

@@ -1,5 +1,5 @@
 dataset,judge_id,judge_name,f1,bacc,avg_latency,total_latency,count,correct
-TOXIGEN,qualifire-eval,Qualifire,0.9169139465875371,0.91892916729865,0.9062801801933432,624.4270441532135,689,633
+TOXIGEN,qualifire-eval,Qualifire,0.9169139465875371,0.91892916729865,0.03062801801933432,624.4270441532135,689,633
 TOXIGEN,meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,0.9529411764705882,0.9537008948885182,1.3404882590898755,923.5964105129242,689,657
 TOXIGEN,meta-llama-3.1-405b-instruct-turbo,Meta Llama 3.1 405B Instruct,0.9410029498525073,0.9421103190252288,1.0220591648223611,704.1987645626068,689,649
 TOXIGEN,meta-llama-4-scout-17B-16E-instruct,Meta Llama 4 Scout 17B 16E Instruct,0.9153515064562411,0.914332541247451,7.129768445633665,4912.4104590415955,689,630

benchmarks/safety/moderation-api-1680-samples-judges-metrics.csv

@@ -0,0 +1,24 @@
+dataset,judge_id,judge_name,f1,bacc,avg_latency,total_latency,count,correct
+moderation-api-1680,qualifire-eval,Qualifire,0.8334630350194552,0.9054116292458262,0.03659970967542557,1622.8751225471497,1680,1406
+moderation-api-1680,meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,0.7603930461073318,0.8531306453854248,1.7941580413352876,3014.185509443283,1680,1363
+moderation-api-1680,meta-llama-3.1-405b-instruct-turbo,Meta Llama 3.1 405B Instruct,0.7858267716535433,0.8704563952911282,2.1023602211759203,3531.9651715755463,1680,1408
+moderation-api-1680,meta-llama-4-scout-17B-16E-instruct,Meta Llama 4 Scout 17B 16E Instruct,0.7520723436322532,0.8458449963273975,13.005298352383432,21848.901232004166,1680,1351
+moderation-api-1680,meta-llama-3.3-70B-instruct-turbo,Meta Llama 4 Scout 32K Instruct,0.7661843107387661,0.8574484346773072,2.3053151918309074,3872.9295222759247,1680,1373
+moderation-api-1680,meta-llama-3.1-8b-instruct-turbo,Meta Llama 3.1 8B Instruct,0.7144886363636364,0.816429436404423,1.4811212245907102,2488.283657312393,1680,1278
+moderation-api-1680,gemma-2-27b-it,Gemma 2 27B,0.6586294416243654,0.7661247096658924,1.2426348075980231,2087.626476764679,1680,1142
+moderation-api-1680,gemma-2-9b-it,Gemma 2 9B,0.6339668914776211,0.7395976018899013,1.2315448721249898,2068.995385169983,1680,1083
+moderation-api-1680,mistral-7b-instruct-v0.3,Mistral (7B) Instruct v0.3,0.7180212014134275,0.8203551505766978,0.5537168819279897,930.2443616390228,1680,1281
+moderation-api-1680,o3-mini, o3-mini,0.6489252814738997,0.7440543545153158,4.081525660838399,6856.963110208511,1680,1337
+moderation-api-1680,gpt-4.1,GPT-4.1,0.8059950041631974,0.87940464137534,0.8850721609024774,1486.921230316162,1680,1447
+moderation-api-1680,gpt-4o,GPT-4o,0.7372210223182145,0.8371399360768665,0.9992043121939613,1678.663244485855,1680,1315
+moderation-api-1680,gpt-4-turbo,GPT-4 Turbo,0.75,0.8468822583526889,1.1069318562746049,1859.645518541336,1680,1340
+moderation-api-1680,gpt-3.5-turbo,GPT-3.5 Turbo,0.7355805243445693,0.8312736982113433,0.6711374118214561,1127.5108518600464,1680,1327
+moderation-api-1680,claude-3-haiku-20240307,Claude 3 Haiku,0.7309352517985611,0.8311496238064042,0.8771857439052491,1473.6720497608185,1680,1306
+moderation-api-1680,claude-3-sonnet-20240229,Claude 3 Sonnet,0.7416728349370837,0.8382615686975166,0.9701514817419506,1629.854489326477,1680,1331
+moderation-api-1680,claude-3-opus-latest,Claude 3 Opus,0.7281760113555713,0.82989399082842,1.5899008572101594,2671.0334401130676,1680,1297
+moderation-api-1680,claude-3-5-sonnet-latest,Claude 3.5 Sonnet,0.7732513451191392,0.8626297818275663,2.4701464093866803,4149.845967769623,1680,1385
+moderation-api-1680,claude-3-5-haiku-latest,Claude 3.5 Haiku,0.7062068965517241,0.810801421396383,1.3861321630932035,2328.702033996582,1680,1254
+moderation-api-1680,qwen-2.5-72b-instruct-turbo,Qwen 2.5 72B Instruct,0.7028493894165536,0.808776527107776,1.0395322775556928,1746.4142262935638,1680,1242
+moderation-api-1680,qwen-2.5-7b-instruct-turbo,Qwen 2.5 7B Instruct,0.6615186615186616,0.7686756794314414,0.6633135711862927,1114.3667995929718,1680,1154
+moderation-api-1680,deepseek-v3,DeepSeek V3,0.6913746630727763,0.7975105711393008,10.260258168549765,17237.233723163605,1680,1222
+moderation-api-1680,deepseek-r1,DeepSeek R1,0.7303703703703703,0.8275762809441566,9.251970609454881,15543.310623884201,1680,1316

benchmarks/safety/walledai-SGXSTest-judges-metrics.csv

@@ -1,5 +1,5 @@
 judge_id,judge_name,dataset,f1,bacc,avg_latency,total_time,count,correct
-qualifire-eval,Qualifire,walledai/SGXSTest,0.8,0.8200000000000001,0.887103796005249,177.4207592010498,200,164
+qualifire-eval,Qualifire,walledai/SGXSTest,0.8,0.8200000000000001,0.0387103796005249,177.4207592010498,200,164
 meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,walledai/SGXSTest,0.8380952380952381,0.8300000000000001,1.0145819282531738,202.91638565063477,200,166
 meta-llama-3.1-405b-instruct-turbo,Meta Llama 3.1 405B Instruct,walledai/SGXSTest,0.848780487804878,0.845,0.8429156625270844,168.58313250541687,200,169
 meta-llama-4-scout-17B-16E-instruct,Meta Llama 4 Scout 17B 16E Instruct,walledai/SGXSTest,0.8269230769230769,0.8200000000000001,24.309131922721864,4861.826384544373,200,164

data/leaderboard.csv

@@ -1,8 +1,8 @@
 judge_id,judge_name,elo_score,wins,losses,total_evaluations,organization,license,parameters
-qualifire-eval,Qualifire,1694.7636854810899,26.0,1.0,27.0,Qualifire,Proprietary,400M
+qualifire-eval,Qualifire,1694.76368548109,26.0,1.0,27.0,Qualifire,Proprietary,400M
 meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,1543.016606880631,4.0,1.0,5.0,Meta,Open Source,
 gpt-3.5-turbo,GPT-3.5 Turbo,1516.7701398146428,1.0,0.0,1.0,OpenAI,Proprietary,
-claude-3-haiku-20240307,Claude 3 Haiku,1507.1176382557335,1.0,1.0,2.0,Anthropic,Proprietary,
+claude-3-haiku-20240307,Claude 3 Haiku,1507.1176382557337,1.0,1.0,2.0,Anthropic,Proprietary,
 claude-3-5-haiku-latest,Claude 3.5 Haiku,1500.0,0.0,0.0,0.0,Anthropic,Proprietary,
 gpt-4o,GPT-4o,1500.0,0.0,0.0,0.0,OpenAI,Proprietary,
 judge5,Mixtral,1500.0,0.0,0.0,0.0,Mistral AI,Commercial,