refactoring

.cursor/rules/python.mdc

@@ -0,0 +1,124 @@
+---
+description:
+globs: **/*.py, src/**/*.py, tests/**/*.py
+alwaysApply: false
+---
+---
+description: Python best practices and patterns for modern software development with Flask and SQLite
+globs: **/*.py, src/**/*.py, tests/**/*.py
+---
+
+# Python Best Practices
+
+## Project Structure
+- Use src-layout with `src/your_package_name/`
+- Place tests in `tests/` directory parallel to `src/`
+- Keep configuration in `config/` or as environment variables
+- Store requirements in `requirements.txt` or `pyproject.toml`
+- Place static files in `static/` directory
+- Use `templates/` for Jinja2 templates
+
+## Code Style
+- Follow Black code formatting
+- Use isort for import sorting
+- Follow PEP 8 naming conventions:
+  - snake_case for functions and variables
+  - PascalCase for classes
+  - UPPER_CASE for constants
+- Maximum line length of 88 characters (Black default)
+- Use absolute imports over relative imports
+
+## Type Hints
+- Use type hints for all function parameters and returns
+- Import types from `typing` module
+- Use `Optional[Type]` instead of `Type | None`
+- Use `TypeVar` for generic types
+- Define custom types in `types.py`
+- Use `Protocol` for duck typing
+
+## Flask Structure
+- Use Flask factory pattern
+- Organize routes using Blueprints
+- Use Flask-SQLAlchemy for database
+- Implement proper error handlers
+- Use Flask-Login for authentication
+- Structure views with proper separation of concerns
+
+## Database
+- Use SQLAlchemy ORM
+- Implement database migrations with Alembic
+- Use proper connection pooling
+- Define models in separate modules
+- Implement proper relationships
+- Use proper indexing strategies
+## Authentication
+- Use Flask-Login for session management
+- Implement Google OAuth using Flask-OAuth
+- Hash passwords with bcrypt
+- Use proper session security
+- Implement CSRF protection
+- Use proper role-based access control
+
+## API Design
+- Use Flask-RESTful for REST APIs
+- Implement proper request validation
+- Use proper HTTP status codes
+- Handle errors consistently
+- Use proper response formats
+- Implement proper rate limiting
+
+## Testing
+- Use pytest for testing
+- Write tests for all routes
+- Use pytest-cov for coverage
+- Implement proper fixtures
+- Use proper mocking with pytest-mock
+- Test all error scenarios
+
+## Security
+- Use HTTPS in production
+- Implement proper CORS
+- Sanitize all user inputs
+- Use proper session configuration
+- Implement proper logging
+- Follow OWASP guidelines
+
+## Performance
+- Use proper caching with Flask-Caching
+- Implement database query optimization
+- Use proper connection pooling
+- Implement proper pagination
+- Use background tasks for heavy operations
+- Monitor application performance
+
+## Error Handling
+- Create custom exception classes
+- Use proper try-except blocks
+- Implement proper logging
+- Return proper error responses
+- Handle edge cases properly
+- Use proper error messages
+
+## Documentation
+- Use Google-style docstrings
+- Document all public APIs
+- Keep README.md updated
+- Use proper inline comments
+- Generate API documentation
+- Document environment setup
+
+## Development Workflow
+- Use virtual environments (venv)
+- Implement pre-commit hooks
+- Use proper Git workflow
+- Follow semantic versioning
+- Use proper CI/CD practices
+- Implement proper logging
+
+## Dependencies
+- Pin dependency versions
+- Use requirements.txt for production
+- Separate dev dependencies
+- Use proper package versions
+- Regularly update dependencies
+- Check for security vulnerabilities

data/leaderboard.csv

@@ -1,31 +1,31 @@
 judge_id,judge_name,elo_score,wins,losses,total_evaluations,organization,license
+qwen-2.5-72b-instruct-turbo,Qwen 2.5 72B Instruct,1649.8276600728695,21.0,2.0,23.0,Alibaba,Open Source
 claude-3-opus-latest,Claude 3 Opus,1531.9661669788793,2.0,0.0,2.0,Anthropic,Proprietary
 mistral-7b-instruct-v0.1,Mistral (7B) Instruct v0.1,1516.736306793522,1.0,0.0,1.0,Mistral AI,Open Source
 qwen-2.5-7b-instruct-turbo,Qwen 2.5 7B Instruct,1516.0,1.0,0.0,1.0,Alibaba,Open Source
 meta-llama-3.1-8b-instruct-turbo,Meta Llama 3.1 8B Instruct,1515.2298601853572,1.0,0.0,1.0,Meta,Open Source
 gpt-4-turbo,GPT-4 Turbo,1500.736306793522,1.0,1.0,2.0,OpenAI,Proprietary
-meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
-gemma-2-27b-it,Gemma 2 27B,1500.0,0.0,0.0,0.0,Google,Open Source
-gemma-2-9b-it,Gemma 2 9B,1500.0,0.0,0.0,0.0,Google,Open Source
-qwen-2-72b-instruct,Qwen 2 Instruct (72B),1500.0,0.0,0.0,0.0,Alibaba,Open Source
-mistral-7b-instruct-v0.3,Mistral (7B) Instruct v0.3,1500.0,0.0,0.0,0.0,Mistral AI,Open Source
-gpt-3.5-turbo,GPT-3.5 Turbo,1500.0,0.0,0.0,0.0,OpenAI,Proprietary
-atla-selene,Atla Selene,1500.0,0.0,0.0,0.0,Atla,Proprietary
-claude-3-5-haiku-latest,Claude 3.5 Haiku,1500.0,0.0,0.0,0.0,Anthropic,Proprietary
-claude-3-sonnet-20240229,Claude 3 Sonnet,1500.0,0.0,0.0,0.0,Anthropic,Proprietary
 deepseek-r1,DeepSeek R1,1500.0,0.0,0.0,0.0,DeepSeek,Open Source
-judge1,EvalGPT,1500.0,0.0,0.0,0.0,OpenAI,Commercial
-judge2,CritiqueBot,1500.0,0.0,0.0,0.0,OpenAI,Commercial
-judge3,GradeAssist,1500.0,0.0,0.0,0.0,Anthropic,Commercial
-judge4,PrecisionJudge,1500.0,0.0,0.0,0.0,Anthropic,Commercial
-judge5,Mixtral,1500.0,0.0,0.0,0.0,Mistral AI,Commercial
-meta-llama-4-scout-17B-16E-instruct,Meta Llama 4 Scout 17B 16E Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
-meta-llama-3.3-70B-instruct-turbo,Meta Llama 4 Scout 32K Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
-o3-mini, o3-mini,1500.0,0.0,0.0,0.0,OpenAI,Proprietary
 deepseek-v3,DeepSeek V3,1500.0,0.0,0.0,0.0,DeepSeek,Open Source
+o3-mini, o3-mini,1500.0,0.0,0.0,0.0,OpenAI,Proprietary
+meta-llama-3.3-70B-instruct-turbo,Meta Llama 4 Scout 32K Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
+meta-llama-4-scout-17B-16E-instruct,Meta Llama 4 Scout 17B 16E Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
+judge5,Mixtral,1500.0,0.0,0.0,0.0,Mistral AI,Commercial
+judge4,PrecisionJudge,1500.0,0.0,0.0,0.0,Anthropic,Commercial
+judge3,GradeAssist,1500.0,0.0,0.0,0.0,Anthropic,Commercial
+judge2,CritiqueBot,1500.0,0.0,0.0,0.0,OpenAI,Commercial
+claude-3-sonnet-20240229,Claude 3 Sonnet,1500.0,0.0,0.0,0.0,Anthropic,Proprietary
+claude-3-5-haiku-latest,Claude 3.5 Haiku,1500.0,0.0,0.0,0.0,Anthropic,Proprietary
+atla-selene,Atla Selene,1500.0,0.0,0.0,0.0,Atla,Proprietary
+mistral-7b-instruct-v0.3,Mistral (7B) Instruct v0.3,1500.0,0.0,0.0,0.0,Mistral AI,Open Source
+qwen-2-72b-instruct,Qwen 2 Instruct (72B),1500.0,0.0,0.0,0.0,Alibaba,Open Source
+gemma-2-9b-it,Gemma 2 9B,1500.0,0.0,0.0,0.0,Google,Open Source
+gemma-2-27b-it,Gemma 2 27B,1500.0,0.0,0.0,0.0,Google,Open Source
+meta-llama-3.1-70b-instruct-turbo,Meta Llama 3.1 70B Instruct,1500.0,0.0,0.0,0.0,Meta,Open Source
+judge1,EvalGPT,1500.0,0.0,0.0,0.0,OpenAI,Commercial
 meta-llama-3.1-405b-instruct-turbo,Meta Llama 3.1 405B Instruct,1499.263693206478,1.0,1.0,2.0,Meta,Open Source
 claude-3-haiku-20240307,Claude 3 Haiku,1499.263693206478,1.0,1.0,2.0,Anthropic,Proprietary
 gpt-4.1,GPT-4.1,1484.7701398146428,0.0,1.0,1.0,OpenAI,Proprietary
 claude-3-5-sonnet-latest,Claude 3.5 Sonnet,1484.0,0.0,1.0,1.0,Anthropic,Proprietary
 gpt-4o,GPT-4o,1484.0,0.0,1.0,1.0,OpenAI,Proprietary
-qwen-2.5-72b-instruct-turbo,Qwen 2.5 72B Instruct,1468.0338330211207,0.0,2.0,2.0,Alibaba,Open Source
+gpt-3.5-turbo,GPT-3.5 Turbo,1318.2061729482512,0.0,21.0,21.0,OpenAI,Proprietary

src/app.py

@@ -10,6 +10,7 @@ from src.ui import UI
 # Global state for evaluations
 eval1: Optional[Dict[str, Any]] = None
 eval2: Optional[Dict[str, Any]] = None
+selected_judges: list = []
 current_test_type: str = "grounding"
 
 
@@ -31,6 +32,18 @@ def initialize():
             test_type,
             judge_manager,
         ),
+        evaluate1_fn=lambda input_text, output_text, test_type: get_evaluation1(
+            input_text,
+            output_text,
+            test_type,
+            judge_manager,
+        ),
+        evaluate2_fn=lambda input_text, output_text, test_type: get_evaluation2(
+            input_text,
+            output_text,
+            test_type,
+            judge_manager,
+        ),
         winner1_fn=lambda: select_winner("Evaluation 1", judge_manager),
         winner2_fn=lambda: select_winner("Evaluation 2", judge_manager),
         refresh_leaderboard_fn=lambda: judge_manager.leaderboard_df,
@@ -57,50 +70,106 @@ def submit_example(
     output_text: str,
     test_type: str,
     judge_manager: JudgeManager,
-) -> Tuple[str, str, Any, Any]:
-    """Submit an example for evaluation."""
-    global eval1, eval2, current_test_type
+) -> Tuple[str, str, Any, Any, Any, Any]:
+    """Prepare for evaluation and select random judges."""
+    global selected_judges, current_test_type, eval1, eval2
 
     try:
-        logger.info(f"Submitting example for test type: {test_type}")
+        logger.info(f"Preparing evaluation for test type: {test_type}")
         current_test_type = test_type
-        selected_judges = judge_manager.pick_random_judges()
-        eval1 = judge_manager.get_random_judges_evaluations(
-            input_text,
-            output_text,
-            test_type,
-            selected_judges[0],
-        )
 
-        eval2 = judge_manager.get_random_judges_evaluations(
-            input_text,
-            output_text,
-            test_type,
-            selected_judges[1],
-        )
+        # Reset evaluations
+        eval1 = None
+        eval2 = None
 
-        if not eval1 or not eval2:
+        # Select random judges
+        selected_judges = judge_manager.pick_random_judges()
+
+        if len(selected_judges) < 2:
             return (
                 "Error: Not enough judges available",
                 "Error: Not enough judges available",
                 None,
                 None,
+                None,
+                gr.update(visible=False),
             )
 
+        # Show loading messages while evaluations are in progress
+        status_text = "Evaluations starting... Both judges will evaluate in parallel."
         return (
-            eval1["display_evaluation"],
-            eval2["display_evaluation"],
-            gr.update(visible=True),
-            gr.update(visible=True),
+            "Loading evaluation 1...",
+            "Loading evaluation 2...",
+            gr.update(value=input_text),
+            gr.update(value=output_text),
+            gr.update(value=test_type),
+            gr.update(visible=True, value=status_text),
         )
     except Exception as e:
-        logger.error(f"Error submitting example: {e}")
+        logger.error(f"Error preparing evaluation: {e}")
         return (
             f"Error: {str(e)}",
             f"Error: {str(e)}",
-            None,
-            None,
+            gr.update(value=input_text),
+            gr.update(value=output_text),
+            gr.update(value=test_type),
+            gr.update(visible=False),
+        )
+
+
+def get_evaluation1(
+    input_text: str,
+    output_text: str,
+    test_type: str,
+    judge_manager: JudgeManager,
+) -> Tuple[str, Any]:
+    """Get evaluation from the first judge."""
+    global eval1, selected_judges
+
+    try:
+        if not selected_judges or len(selected_judges) < 1:
+            return "No judges selected", gr.update(visible=False)
+
+        logger.info(f"Starting evaluation 1 with judge {selected_judges[0]['name']}")
+        # Get evaluation from the first judge
+        eval1 = judge_manager.get_evaluation(
+            selected_judges[0],
+            input_text,
+            output_text,
+            test_type,
         )
+        logger.info("Completed evaluation 1")
+
+        # Make the selection button visible once the evaluation is ready
+        return eval1["display_evaluation"], gr.update(visible=True)
+    except Exception as e:
+        logger.error(f"Error getting evaluation 1: {e}")
+        return f"Error: {str(e)}", gr.update(visible=False)
+
+
+def get_evaluation2(
+    input_text: str,
+    output_text: str,
+    test_type: str,
+    judge_manager: JudgeManager,
+) -> Tuple[str, Any]:
+    """Get evaluation from the second judge."""
+    global eval2, selected_judges
+
+    try:
+        if not selected_judges or len(selected_judges) < 2:
+            return "No judges selected", gr.update(visible=False)
+
+        logger.info(f"Starting evaluation 2 with judge {selected_judges[1]['name']}")
+        # Get evaluation from the second judge
+        eval2 = judge_manager.get_evaluation(selected_judges[1], input_text, output_text, test_type)
+        logger.info("Completed evaluation 2")
+
+        # Make the selection button visible once the evaluation is ready
+        return eval2["display_evaluation"], gr.update(visible=True)
+    except Exception as e:
+        logger.error(f"Error getting evaluation 2: {e}")
+        return f"Error: {str(e)}", gr.update(visible=False)
 
 
 def select_winner(choice: str, judge_manager: JudgeManager) -> str:

src/judge.py

@@ -101,8 +101,10 @@ class JudgeManager:
                     temperature=0.2,
                     max_tokens=500,
                 )
-                # Default fallback
                 evaluation = api_response.choices[0].message.content
+            else:
+                # Default fallback
+                evaluation = f"No evaluation provider for {judge['provider']}"
 
             # Format the evaluation
             eval_prefix = f"Evaluation by {judge['name']} (ID: {judge['id']}):\n\n"
@@ -137,8 +139,11 @@ AI RESPONSE:
 
 Please evaluate this response carefully and provide your assessment."""
 
-    def pick_random_judges(self) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    def pick_random_judges(self) -> List[Dict[str, Any]]:
         """Pick two random judges"""
+        if len(self.judges) < 2:
+            logger.error("Not enough judges available for comparison")
+            return []
         return random.sample(self.judges, 2)
 
     def get_random_judges_evaluations(
@@ -146,7 +151,7 @@ Please evaluate this response carefully and provide your assessment."""
         input_text: str,
         output_text: str,
         test_type: str,
-        selected_judges: List[Dict[str, Any]],
+        selected_judge: Dict[str, Any],
     ) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
         """Get evaluations from two random judges"""
         if len(self.judges) < 2:
@@ -154,12 +159,14 @@ Please evaluate this response carefully and provide your assessment."""
             return None, None
 
         # Get evaluations from the judges
-        evaluations = []
-        for judge in selected_judges:
-            evaluation = self.get_evaluation(judge, input_text, output_text, test_type)
-            evaluations.append(evaluation)
-
-        return evaluations[0], evaluations[1]
+        evaluation = self.get_evaluation(
+            selected_judge,
+            input_text,
+            output_text,
+            test_type,
+        )
+
+        return evaluation
 
     def update_leaderboard(self, winner_id: str, loser_id: str) -> pd.DataFrame:
         """Update the leaderboard after a comparison"""

src/ui.py

@@ -13,6 +13,8 @@ class UI:
         self,
         refresh_fn: Callable,
         submit_fn: Callable,
+        evaluate1_fn: Callable,
+        evaluate2_fn: Callable,
         winner1_fn: Callable,
         winner2_fn: Callable,
         refresh_leaderboard_fn: Callable,
@@ -20,6 +22,8 @@ class UI:
     ):
         self.refresh_fn = refresh_fn
         self.submit_fn = submit_fn
+        self.evaluate1_fn = evaluate1_fn
+        self.evaluate2_fn = evaluate2_fn
         self.winner1_fn = winner1_fn
         self.winner2_fn = winner2_fn
         self.refresh_leaderboard_fn = refresh_leaderboard_fn
@@ -97,6 +101,7 @@ class UI:
                         input_text = gr.Textbox(label="Input", lines=4)
                         output_text = gr.Textbox(label="Output", lines=6)
                         submit_button = gr.Button("Get Judge Evaluations")
+                        status_message = gr.Markdown(visible=False)
 
                 with gr.Row():
                     with gr.Column():
@@ -129,10 +134,26 @@ class UI:
                 [input_text, output_text],
             )
 
-            submit_button.click(
+            # Modified submit to prepare for evaluation and trigger both evaluations in parallel
+            submit_event = submit_button.click(
                 self.submit_fn,
                 [input_text, output_text, test_type_dropdown],
-                [evaluation1, evaluation2, select_eval1, select_eval2],
+                [evaluation1, evaluation2, input_text, output_text, test_type_dropdown, status_message],
+            )
+
+            # Start both evaluations simultaneously (in parallel) after submit completes
+            submit_event.then(
+                self.evaluate1_fn,
+                [input_text, output_text, test_type_dropdown],
+                [evaluation1, select_eval1],
+                queue=False,  # Run immediately without waiting in queue
+            )
+
+            submit_event.then(
+                self.evaluate2_fn,
+                [input_text, output_text, test_type_dropdown],
+                [evaluation2, select_eval2],
+                queue=False,  # Run immediately without waiting in queue
             )
 
             select_eval1.click(