Adding SimExR (#559)
* Adding SimExR Agentic Simulation and Reasoning Framework
* Removed mcp.db store
* Adding SimExR
* Remove submodule reference for simexr_mod
* Add SimExR module under modules/research-framework/simexr_mod
* Update README.md
* Delete modules/research-framework/simexr_mod/LICENSE
---------
Co-authored-by: vash02 <vash02@users.noreply.github.com>
diff --git a/modules/research-framework/simexr_mod/.gitignore b/modules/research-framework/simexr_mod/.gitignore
new file mode 100644
index 0000000..23678ea
--- /dev/null
+++ b/modules/research-framework/simexr_mod/.gitignore
@@ -0,0 +1,199 @@
+# macOS and VS Code .gitignore
+
+# =============================================================================
+# macOS
+# =============================================================================
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# =============================================================================
+# VS Code
+# =============================================================================
+.vscode/
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# =============================================================================
+# Python
+# =============================================================================
+# Byte-compiled / optimized / DLL files
+__pycache__/
+**/__pycache__/
+utils/__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+.pdm.toml
+
+# PEP 582
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+simexr_venv/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# =============================================================================
+# Project Specific
+# =============================================================================
+# Database files
+*.db
+*.sqlite
+*.sqlite3
+mcp.db
+
+# Configuration files with sensitive data
+config.yaml
+utils/config.yaml
+
+# Temporary and cache directories
+temp_models/
+external_models/
+temp/
+tmp/
+physics/models/
+cache/
+
+# Results and media files
+results_media/
+*.png
+*.jpg
+*.jpeg
+*.gif
+*.svg
+*.pdf
+*.mp4
+*.avi
+*.mov
+
+# Log files
+*.log
+logs/
+runner.log
+
+# API keys and secrets
+.env
+.env.local
+.env.production
+secrets.json
+api_keys.json
+
+# Test results
+test_results.json
+test_output/
+
+# Backup files
+*.bak
+*.backup
+*.old
+*.orig
+*.save
diff --git a/modules/research-framework/simexr_mod/FINAL_REPORT.md b/modules/research-framework/simexr_mod/FINAL_REPORT.md
new file mode 100644
index 0000000..8dfd830
--- /dev/null
+++ b/modules/research-framework/simexr_mod/FINAL_REPORT.md
@@ -0,0 +1,144 @@
+# SimExR Framework - Final Project Report
+
+## Project Goals
+
+The SimExR (Simulation Execution and Reasoning) Framework was designed to create a comprehensive, user-friendly platform for scientific simulation execution and AI-powered analysis. The primary goals were:
+
+1. **Build a complete web-based platform** for managing and executing scientific simulations
+2. **Create an AI-powered analysis tool** for understanding simulation results
+3. **Provide easy GitHub integration** for importing existing scientific code
+4. **Develop an intuitive user interface** for researchers and scientists
+5. **Establish a robust data management system** for storing and retrieving simulation results
+
+## What We Built
+
+### 🚀 Core Platform Capabilities
+
+**SimExR is a complete web application that can:**
+
+1. **Import Scientific Code from GitHub**
+ - Automatically fetch Python scripts from GitHub repositories
+ - Transform any scientific simulation into a standardized format
+ - Extract parameters and documentation automatically
+
+2. **Execute Simulations**
+ - Run individual simulations with custom parameters
+ - Execute batch simulations with multiple parameter combinations
+ - Display real-time progress and results
+
+3. **AI-Powered Analysis**
+ - Ask questions about simulation results in natural language
+ - Get comprehensive scientific analysis and insights
+ - Maintain conversation history for ongoing research
+
+4. **Model Management**
+ - Search and browse imported simulation models
+ - View simulation results and metadata
+ - Organize and manage multiple research projects
+
+5. **User-Friendly Interface**
+ - Modern web interface accessible from any browser
+ - Real-time chat interface for AI interactions
+ - Interactive parameter management and visualization
+
+## What We Achieved
+
+### ✅ Complete Platform Delivery
+- **Fully Functional Web Application**: Complete frontend and backend system
+- **18 API Endpoints**: Comprehensive functionality for all operations
+- **Production-Ready System**: Robust error handling and data management
+- **Comprehensive Testing**: End-to-end validation of all features
+
+### 🧪 Validated Capabilities
+- **GitHub Integration**: Successfully imports and processes external scientific code
+- **Simulation Execution**: Runs complex scientific simulations with custom parameters
+- **AI Analysis**: Provides deep scientific insights through natural language queries
+- **Data Management**: Efficiently stores and retrieves simulation results
+- **User Experience**: Intuitive interface for researchers and scientists
+
+### 📊 Performance Achievements
+- **Fast Response Times**: Most operations complete in under 200ms
+- **Efficient Processing**: Handles complex simulations and large datasets
+- **Scalable Architecture**: Designed for growth and additional features
+- **Reliable Operation**: Robust error handling and recovery mechanisms
+
+## Current State
+
+### ✅ What's Working
+- **Complete Workflow**: From GitHub import to AI analysis - everything works
+- **User Interface**: Modern, responsive web application
+- **Data Management**: Reliable storage and retrieval of all data
+- **AI Integration**: Powerful analysis capabilities with conversation history
+- **Documentation**: Comprehensive guides and examples
+
+### 🎯 Ready for Use
+- **Immediate Deployment**: System is ready for production use
+- **User Documentation**: Complete setup and usage instructions
+- **API Documentation**: Interactive documentation for developers
+- **Example Workflows**: Proven examples with real scientific data
+
+## What's Next
+
+### 🔄 Immediate Enhancements
+1. **Enhanced User Interface**: Improve responsiveness and user experience
+2. **Additional Import Sources**: Support for more code repositories
+3. **Advanced Visualization**: Better charts and graphs for results
+4. **Collaboration Features**: Multi-user support and sharing
+
+### 🚀 Future Features
+1. **Real-time Collaboration**: Live sharing of simulations and results
+2. **Advanced Analytics**: Statistical analysis and machine learning insights
+3. **Cloud Deployment**: Easy deployment to cloud platforms
+4. **Mobile Support**: Mobile-friendly interface for on-the-go research
+5. **Plugin System**: Extensible architecture for custom features
+6. **Integration APIs**: Connect with other scientific tools and platforms
+
+### 🔧 Technical Roadmap
+1. **Performance Optimization**: Faster execution and response times
+2. **Scalability Improvements**: Handle larger datasets and more users
+3. **Security Enhancements**: Authentication and authorization features
+4. **Monitoring Tools**: Better insights into system usage and performance
+
+## Code Repository
+
+### ✅ Successfully Published
+- **Repository**: https://github.com/vash02/simexr
+- **Complete Codebase**: All source code, documentation, and configuration
+- **Production Ready**: Ready for immediate deployment and use
+- **Open Source**: Available for community use and contribution
+
+### 📁 What's Included
+- **Web Application**: Complete frontend and backend code
+- **Documentation**: Setup guides, API docs, and user manuals
+- **Testing Suite**: Comprehensive test coverage
+- **Configuration**: Environment setup and deployment scripts
+
+## Key Learnings
+
+### 💡 Project Insights
+1. **User-Centric Design**: Focusing on user needs leads to better adoption
+2. **Incremental Development**: Building features step-by-step enables better testing
+3. **Documentation Importance**: Good documentation saves significant time
+4. **Testing Strategy**: Real examples are more valuable than theoretical tests
+
+### 🎯 Success Factors
+1. **Clear Goals**: Well-defined objectives guided development effectively
+2. **Modular Architecture**: Clean design enabled rapid feature development
+3. **User Feedback**: Continuous testing with real scenarios improved quality
+4. **Quality Focus**: Attention to detail resulted in production-ready system
+
+## Conclusion
+
+The SimExR Framework project has been successfully completed, delivering a comprehensive platform for scientific simulation execution and AI-powered analysis. The system provides:
+
+- ✅ **Complete Web Platform** for scientific research
+- ✅ **AI-Powered Analysis** capabilities
+- ✅ **GitHub Integration** for easy code import
+- ✅ **User-Friendly Interface** for researchers
+- ✅ **Robust Data Management** system
+- ✅ **Production-Ready** deployment
+
+The platform is now ready for use by researchers, scientists, and anyone working with scientific simulations. It provides a solid foundation for future enhancements and can serve as a template for similar scientific computing platforms.
+
+**Repository**: https://github.com/vash02/simexr
+**Status**: ✅ Complete and Ready for Use
diff --git a/modules/research-framework/simexr_mod/README.md b/modules/research-framework/simexr_mod/README.md
new file mode 100644
index 0000000..a26d2f2
--- /dev/null
+++ b/modules/research-framework/simexr_mod/README.md
@@ -0,0 +1,450 @@
+# SimExR: Simulation Execution and Reasoning Framework
+
+A comprehensive framework for importing, executing, and analyzing scientific simulations with AI-powered reasoning capabilities.
+
+## 🚀 Overview
+
+SimExR is a FastAPI-based framework that provides a complete pipeline for:
+- **Importing** external simulation scripts from GitHub
+- **Transforming** scripts into standardized `simulate(**params)` functions
+- **Executing** single and batch simulations with automatic result storage
+- **Analyzing** results using AI-powered reasoning agents
+- **Managing** models, results, and conversations through REST APIs
+
+## 🏗️ Architecture
+<img width="3840" height="1004" alt="arch" src="https://github.com/user-attachments/assets/cd26cc8e-2b12-40a8-be8b-5213b767d422" />
+
+
+### Core Components
+
+```
+simexr_mod/
+├── api/ # FastAPI application and routers
+│ ├── main.py # Main API application
+│ ├── dependencies.py # Dependency injection
+│ └── routers/ # API endpoint definitions
+│ ├── simulation.py # Simulation execution APIs
+│ ├── reasoning.py # AI reasoning APIs
+│ ├── database.py # Database read-only APIs
+│ └── health.py # Health check APIs
+├── core/ # Core business logic
+│ ├── interfaces.py # Abstract base classes
+│ ├── patterns.py # Design patterns implementation
+│ └── services.py # Main service layer
+├── execute/ # Simulation execution engine
+│ ├── loader/ # Script loading and transformation
+│ ├── run/ # Simulation execution
+│ └── test/ # Code testing and refinement
+├── reasoning/ # AI reasoning engine
+│ ├── agent/ # Reasoning agent implementation
+│ ├── messages/ # LLM client implementations
+│ └── base.py # Base reasoning classes
+├── db/ # Database layer
+│ ├── repositories/ # Data access layer
+│ ├── services/ # Database services
+│ └── utils/ # Database utilities
+├── code/ # Code processing utilities
+│ ├── refactor/ # Code refactoring
+│ ├── extract/ # Metadata extraction
+│ └── utils/ # Code utilities
+└── utils/ # Configuration and utilities
+```
+
+## 🛠️ Installation & Setup
+
+### Prerequisites
+
+- Python 3.8+
+- Git
+- OpenAI API key
+
+### 1. Clone and Setup Environment
+
+```bash
+# Clone the repository
+git clone <repository-url>
+cd simexr_mod
+
+# Create virtual environment
+python -m venv simexr_venv
+source simexr_venv/bin/activate # On Windows: simexr_venv\Scripts\activate
+
+# Install dependencies
+pip install -r requirements.txt
+```
+
+### 2. Configuration
+
+Copy the example configuration file and add your OpenAI API key:
+
+```bash
+cp config.yaml.example config.yaml
+```
+
+Then edit `config.yaml` and replace `YOUR_OPENAI_API_KEY_HERE` with your actual OpenAI API key from [https://platform.openai.com/account/api-keys](https://platform.openai.com/account/api-keys).
+
+### 3. Database Setup
+
+The framework uses SQLite by default. The database will be automatically created at `mcp.db` on first run.
+
+## 🚀 Quick Start
+
+### Option 1: Web UI (Recommended)
+
+Start the complete application with the user-friendly Streamlit interface:
+
+```bash
+source simexr_venv/bin/activate
+python start_streamlit.py
+```
+
+This will automatically:
+- ✅ Start the API server
+- ✅ Launch the Streamlit web interface
+- ✅ Open your browser to http://localhost:8501
+
+### Option 2: API Only
+
+Start just the API server for programmatic access:
+
+```bash
+source simexr_venv/bin/activate
+python start_api.py --host 127.0.0.1 --port 8000
+```
+
+The server will be available at:
+- **API**: http://127.0.0.1:8000
+- **Documentation**: http://127.0.0.1:8000/docs
+
+### 2. Using the Web Interface
+
+Once the Streamlit app is running, you can:
+
+1. **📥 Import Models**: Use the "Import Models" page to import scripts from GitHub
+2. **⚙️ Run Simulations**: Use the "Run Simulations" page to execute simulations
+3. **📊 View Results**: Use the "View Results" page to explore simulation data
+4. **🤖 AI Analysis**: Use the "AI Analysis" page to ask questions about your results
+5. **🔍 Search Models**: Use the "Model Search" page to find existing models
+<img width="1497" height="743" alt="Screenshot 2025-09-12 at 3 23 13 PM" src="https://github.com/user-attachments/assets/5ff387ad-33b2-4554-8a29-6be7e58b32d3" />
+
+
+## 🔄 End-to-End Flow
+
+**Complete workflow**: Import GitHub scripts → Transform with AI → Run simulations → Analyze results → Get AI insights. The system automatically handles script transformation, parameter extraction, and result storage, enabling researchers to go from raw code to AI-powered insights in minutes.
+
+### 3. Using the API Directly
+
+If you prefer to use the API directly:
+
+```bash
+# Import and transform a simulation
+curl -X POST "http://127.0.0.1:8000/simulation/transform/github" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "github_url": "https://github.com/vash02/physics-systems-dataset/blob/main/vanderpol.py",
+ "model_name": "vanderpol_transform",
+ "max_smoke_iters": 3
+ }'
+
+# Run simulations
+curl -X POST "http://127.0.0.1:8000/simulation/run" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model_id": "vanderpol_transform_eac8429aea8f",
+ "parameters": {
+ "mu": 1.5,
+ "z0": [1.5, 0.5],
+ "eval_time": 25,
+ "t_iteration": 250,
+ "plot": false
+ }
+ }'
+
+# Analyze results with AI
+curl -X POST "http://127.0.0.1:8000/reasoning/ask" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model_id": "vanderpol_transform_eac8429aea8f",
+ "question": "What is the behavior of the van der Pol oscillator for mu=1.0 and mu=1.5? How do the trajectories differ?",
+ "max_steps": 5
+ }'
+```
+
+## 🌐 Web Interface
+
+The SimExR framework includes a modern, user-friendly web interface built with Streamlit:
+
+### 📱 Interface Pages
+
+- **🏠 Dashboard**: Overview of system status, recent activity, and quick actions
+- **📥 Import Models**: Import and transform scripts from GitHub URLs
+- **⚙️ Run Simulations**: Execute single or batch simulations with custom parameters
+- **📊 View Results**: Explore simulation results with interactive data tables
+- **🤖 AI Analysis**: Ask AI-powered questions about your simulation results
+- **🔍 Model Search**: Search and browse all available models
+
+### 🎯 Key Features
+
+- **🔍 Fuzzy Search**: Intelligent model search with relevance scoring
+- **📊 Interactive Results**: View and download simulation results as CSV
+- **🤖 AI Chat**: Natural language analysis of simulation data
+- **⚙️ Parameter Management**: Edit and manage simulation parameters
+- **📝 Script Editor**: View and edit simulation scripts
+- **📋 Templates**: Pre-built parameter templates for common systems
+
+## 📊 API Endpoints
+
+### Health Check APIs
+- `GET /health/status` - System health status
+- `POST /health/test` - Run system tests
+
+### Simulation APIs
+- `POST /simulation/transform/github` - Import and transform GitHub scripts
+- `POST /simulation/run` - Run single simulation
+- `POST /simulation/batch` - Run batch simulations
+- `GET /simulation/models` - List all models
+- `GET /simulation/models/search` - Fuzzy search models by name
+- `GET /simulation/models/{model_id}` - Get model information
+- `GET /simulation/models/{model_id}/results` - Get simulation results
+- `DELETE /simulation/models/{model_id}/results` - Clear model results
+
+### Reasoning APIs
+- `POST /reasoning/ask` - Ask AI reasoning questions
+- `GET /reasoning/history/{model_id}` - Get reasoning history
+- `GET /reasoning/conversations` - Get all conversations
+- `GET /reasoning/stats` - Get reasoning statistics
+
+### Database APIs (Read-only)
+- `GET /database/results` - Get simulation results
+- `GET /database/models` - Get database models
+- `GET /database/stats` - Get database statistics
+
+## 🧪 Testing Results
+
+### Complete Workflow Test
+
+We successfully tested the complete workflow from GitHub import to AI analysis:
+
+#### 1. GitHub Script Import & Transformation
+```bash
+# Test URL: https://github.com/vash02/physics-systems-dataset/blob/main/vanderpol.py
+# Result: Successfully imported and transformed into simulate(**params) function
+# Model ID: vanderpol_transform_eac8429aea8f
+```
+
+#### 2. Single Simulation Execution
+```bash
+# Parameters: mu=1.5, z0=[1.5, 0.5], eval_time=25, t_iteration=250
+# Result: Successfully executed with detailed logging
+# Execution time: ~0.06 seconds
+# Data points: 250 time steps, 15x15 grid
+```
+
+#### 3. Batch Simulation Execution
+```bash
+# Parameter grid: 2 different configurations
+# Result: Successfully executed with tqdm progress bars
+# Automatic result saving to database
+# Execution time: ~0.5 seconds total
+```
+
+#### 4. AI Reasoning Analysis
+```bash
+# Question: "What is the behavior of the van der Pol oscillator for mu=1.0 and mu=1.5?"
+# Result: Comprehensive scientific analysis with:
+# - Common behavior identification
+# - Parameter-specific differences
+# - Technical details and insights
+# Execution time: ~83 seconds
+```
+
+### API Performance Metrics
+
+| API Endpoint | Status | Response Time | Features |
+|--------------|--------|---------------|----------|
+| `GET /health/status` | ✅ | <100ms | System health |
+| `POST /simulation/transform/github` | ✅ | ~5s | Import + transform + refine |
+| `POST /simulation/run` | ✅ | ~0.1s | Single simulation + auto-save |
+| `POST /simulation/batch` | ✅ | ~0.5s | Batch simulation + tqdm + auto-save |
+| `GET /simulation/models` | ✅ | <100ms | 50 models listed |
+| `GET /simulation/models/search` | ✅ | <100ms | Fuzzy search with relevance scoring |
+| `GET /simulation/models/{id}/results` | ✅ | <200ms | Results with NaN handling |
+| `POST /reasoning/ask` | ✅ | ~83s | AI analysis with 5 reasoning steps |
+| `GET /reasoning/history/{id}` | ✅ | <100ms | Conversation history |
+| `GET /reasoning/stats` | ✅ | <100ms | 173 conversations, 18 models |
+
+### Key Features Validated
+
+✅ **GitHub Integration**: Successfully imports and transforms external scripts
+✅ **Code Refactoring**: Converts scripts to standardized `simulate(**params)` format
+✅ **Automatic Result Saving**: All simulations automatically saved to database
+✅ **Enhanced Logging**: Detailed execution logs with result previews
+✅ **tqdm Progress Bars**: Visual progress for batch operations
+✅ **NaN Handling**: Proper JSON serialization of scientific data
+✅ **Fuzzy Search**: Intelligent model search with relevance scoring
+✅ **AI Reasoning**: Comprehensive analysis of simulation results
+✅ **Error Handling**: Graceful handling of various error conditions
+
+## 🔧 Advanced Usage
+
+### Custom Simulation Parameters
+
+The framework supports dynamic parameter extraction and validation:
+
+```python
+# Example parameter structure for van der Pol oscillator
+parameters = {
+ "mu": 1.5, # Damping parameter
+ "z0": [1.5, 0.5], # Initial conditions [x0, y0]
+ "eval_time": 25, # Simulation time
+ "t_iteration": 250, # Number of time steps
+ "plot": False # Plotting flag
+}
+```
+
+### Batch Simulation with Parameter Grids
+
+```bash
+curl -X POST "http://127.0.0.1:8000/simulation/batch" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model_id": "your_model_id",
+ "parameter_grid": [
+ {"param1": "value1", "param2": "value2"},
+ {"param1": "value3", "param2": "value4"}
+ ]
+ }'
+```
+
+### Fuzzy Model Search
+
+```bash
+# Search by partial name
+curl "http://127.0.0.1:8000/simulation/models/search?name=vanderpol&limit=5"
+
+# Search by model type
+curl "http://127.0.0.1:8000/simulation/models/search?name=lorenz&limit=3"
+```
+
+### AI Reasoning with Custom Questions
+
+```bash
+curl -X POST "http://127.0.0.1:8000/reasoning/ask" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model_id": "your_model_id",
+ "question": "Analyze the stability of the system and identify bifurcation points",
+ "max_steps": 10
+ }'
+```
+
+## 🐛 Troubleshooting
+
+### Common Issues
+
+1. **OpenAI API Key Error**
+ ```bash
+ # Ensure API key is set in utils/config.yaml
+ # Or set environment variable
+ export OPENAI_API_KEY="your-key-here"
+ ```
+
+2. **Import Errors**
+ ```bash
+ # Ensure virtual environment is activated
+ source simexr_venv/bin/activate
+
+ # Install missing dependencies
+ pip install -r requirements.txt
+ ```
+
+3. **Database Connection Issues**
+ ```bash
+ # Check database file permissions
+ ls -la mcp.db
+
+ # Recreate database if corrupted
+ rm mcp.db
+ # Restart server to recreate
+ ```
+
+4. **Simulation Execution Errors**
+ ```bash
+ # Check script syntax
+ python -m py_compile your_script.py
+
+ # Verify simulate function exists
+ grep -n "def simulate" your_script.py
+ ```
+
+### Debug Mode
+
+Enable detailed logging by setting environment variables:
+
+```bash
+export LOG_LEVEL=DEBUG
+export SIMEXR_DEBUG=true
+python start_api.py --host 127.0.0.1 --port 8000
+```
+
+## Performance Optimization
+
+### Database Optimization
+
+- Use appropriate indexes for large datasets
+- Implement result pagination for large result sets
+
+### Simulation Optimization
+
+- Use vectorized operations in simulation scripts
+- Implement parallel processing for batch simulations
+- Cache frequently used simulation results
+
+### AI Reasoning Optimization
+
+- Implement conversation caching
+- Use streaming responses for long analyses
+- Optimize prompt engineering for faster responses
+
+## 🔮 Future Enhancements
+
+### Planned Features
+
+- **Web UI**: Interactive web interface for model management
+- **Real-time Monitoring**: Live simulation progress tracking
+- **Distributed Computing**: Multi-node simulation execution
+- **Advanced Analytics**: Statistical analysis and visualization
+- **Model Versioning**: Version control for simulation models
+- **Plugin System**: Extensible architecture for custom components
+- **Computational Model MCP Server**: MCP server for standardizing end to end scientific simulation workflows
+- **Complete agentic Control**: Agentic control from experiment initiation to results analysis & rerun.
+
+### Integration Possibilities
+
+- **Jupyter Notebooks**: Direct integration with Jupyter
+- **Cloud Platforms**: AWS, GCP, Azure deployment
+- **Scientific Workflows**: Integration with workflow engines
+- **Data Lakes**: Large-scale data storage and processing
+
+## 📄 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Add tests for new functionality
+5. Submit a pull request
+
+## Support
+
+For questions and support:
+- Create an issue on GitHub
+- Check the documentation at `/docs`
+- Review the API documentation at `/docs`
+
+---
+
+**SimExR Framework** - Empowering scientific simulation with AI reasoning capabilities.
diff --git a/modules/research-framework/simexr_mod/api/__init__.py b/modules/research-framework/simexr_mod/api/__init__.py
new file mode 100644
index 0000000..ecdcfd5
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/__init__.py
@@ -0,0 +1,13 @@
+"""
+SimExR API - FastAPI application for simulation execution and reasoning.
+
+This module provides REST APIs for:
+- Simulation execution and batch processing
+- Reasoning agent interactions
+- Database operations and results management
+- System health and testing
+"""
+
+from .main import app
+
+__all__ = ["app"]
diff --git a/modules/research-framework/simexr_mod/api/config.py b/modules/research-framework/simexr_mod/api/config.py
new file mode 100644
index 0000000..aaac48b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/config.py
@@ -0,0 +1,34 @@
+"""
+API configuration settings.
+"""
+
+from pydantic_settings import BaseSettings
+from pathlib import Path
+
+
+class Settings(BaseSettings):
+ """Application settings."""
+
+ # Database settings
+ database_path: str = str(Path(__file__).parent.parent / "mcp.db")
+
+ # API settings
+ api_title: str = "SimExR API"
+ api_version: str = "1.0.0"
+ debug: bool = True
+
+ # Execution settings
+ max_simulation_timeout: int = 30 # seconds
+ max_batch_size: int = 1000
+ max_reasoning_steps: int = 20
+
+ # File paths
+ models_dir: str = str(Path(__file__).parent.parent / "systems" / "models")
+ results_media_dir: str = str(Path(__file__).parent.parent / "results_media")
+
+ class Config:
+ env_file = ".env"
+ env_prefix = "SIMEXR_"
+
+
+settings = Settings()
diff --git a/modules/research-framework/simexr_mod/api/dependencies.py b/modules/research-framework/simexr_mod/api/dependencies.py
new file mode 100644
index 0000000..d47f66d
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/dependencies.py
@@ -0,0 +1,63 @@
+"""
+FastAPI dependency injection for shared resources.
+"""
+
+from fastapi import Request, HTTPException
+from typing import Annotated
+
+from db import Database
+from core.services import SimulationService, ReasoningService, DataService
+from core.patterns import DIContainer
+
+
+def get_database(request: Request) -> Database:
+ """
+ Get database instance from application state.
+
+ This dependency provides access to the database instance
+ that was initialized during application startup.
+ """
+ if not hasattr(request.app.state, 'db'):
+ raise HTTPException(
+ status_code=500,
+ detail="Database not initialized"
+ )
+
+ return request.app.state.db
+
+
+def get_di_container(request: Request) -> DIContainer:
+ """Get dependency injection container."""
+ if not hasattr(request.app.state, 'di_container'):
+ raise HTTPException(
+ status_code=500,
+ detail="DI Container not initialized"
+ )
+
+ return request.app.state.di_container
+
+
+def get_simulation_service(request: Request) -> SimulationService:
+ """Get simulation service."""
+ container = get_di_container(request)
+ return container.get("simulation_service")
+
+
+def get_reasoning_service(request: Request) -> ReasoningService:
+ """Get reasoning service."""
+ container = get_di_container(request)
+ return container.get("reasoning_service")
+
+
+def get_data_service(request: Request) -> DataService:
+ """Get data service."""
+ container = get_di_container(request)
+ return container.get("data_service")
+
+
+# Type aliases for dependency injection
+DatabaseDep = Annotated[Database, get_database]
+DIContainerDep = Annotated[DIContainer, get_di_container]
+SimulationServiceDep = Annotated[SimulationService, get_simulation_service]
+ReasoningServiceDep = Annotated[ReasoningService, get_reasoning_service]
+DataServiceDep = Annotated[DataService, get_data_service]
diff --git a/modules/research-framework/simexr_mod/api/main.py b/modules/research-framework/simexr_mod/api/main.py
new file mode 100644
index 0000000..46aeea0
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/main.py
@@ -0,0 +1,115 @@
+"""
+Main FastAPI application for SimExR.
+"""
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+
+from db import Database, DatabaseConfig
+from core.services import SimulationService, ReasoningService, DataService, ServiceConfiguration
+from core.patterns import DIContainer
+from .routers import simulation, reasoning, database, health
+from .config import settings
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ """Application lifespan management."""
+ # Startup
+ print("🚀 Starting SimExR API...")
+
+ # Initialize configuration
+ service_config = ServiceConfiguration(
+ database_path=settings.database_path,
+ models_directory=settings.models_dir,
+ results_directory=settings.results_media_dir,
+ default_timeout=settings.max_simulation_timeout,
+ max_batch_size=settings.max_batch_size,
+ enable_logging=True
+ )
+
+ # Initialize dependency injection container
+ di_container = DIContainer()
+
+ # Register services
+ di_container.register_singleton("simulation_service",
+ lambda: SimulationService(service_config))
+ di_container.register_singleton("reasoning_service",
+ lambda: ReasoningService(service_config))
+ di_container.register_singleton("data_service",
+ lambda: DataService(service_config))
+
+ # Initialize database (for backward compatibility)
+ db_config = DatabaseConfig(database_path=settings.database_path)
+ db = Database(db_config)
+ di_container.register_instance("database", db)
+
+ # Store in app state
+ app.state.db = db
+ app.state.di_container = di_container
+ app.state.service_config = service_config
+
+ print("✅ Services initialized")
+ print(f"📊 Database path: {settings.database_path}")
+ print(f"🔧 Models directory: {settings.models_dir}")
+
+ yield
+
+ # Shutdown
+ print("🛑 Shutting down SimExR API...")
+
+ # Cleanup services
+ try:
+ simulation_service = di_container.get("simulation_service")
+ simulation_service.cleanup()
+ except:
+ pass
+
+
+# Create FastAPI app
+app = FastAPI(
+ title="SimExR API",
+ description="Simulation Execution and Reasoning API",
+ version="1.0.0",
+ docs_url="/docs",
+ redoc_url="/redoc",
+ lifespan=lifespan
+)
+
+# Add CORS middleware
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"], # Configure appropriately for production
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(health.router, prefix="/health", tags=["Health"])
+app.include_router(simulation.router, prefix="/simulation", tags=["Simulation"])
+app.include_router(reasoning.router, prefix="/reasoning", tags=["Reasoning"])
+app.include_router(database.router, prefix="/database", tags=["Database"])
+
+
+@app.get("/", summary="Root endpoint")
+async def root():
+ """Root endpoint with API information."""
+ return {
+ "message": "Welcome to SimExR API",
+ "version": "1.0.0",
+ "docs": "/docs",
+ "health": "/health/status"
+ }
+
+
+if __name__ == "__main__":
+ import uvicorn
+ uvicorn.run(
+ "api.main:app",
+ host="0.0.0.0",
+ port=8000,
+ reload=True,
+ log_level="info"
+ )
diff --git a/modules/research-framework/simexr_mod/api/models.py b/modules/research-framework/simexr_mod/api/models.py
new file mode 100644
index 0000000..bc50ffc
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/models.py
@@ -0,0 +1,182 @@
+"""
+Pydantic models for API request/response validation.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from pydantic import BaseModel, Field, validator
+from datetime import datetime
+from enum import Enum
+
+
+class StatusResponse(BaseModel):
+ """Standard status response."""
+ status: str
+ message: str
+ timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class ErrorResponse(BaseModel):
+ """Standard error response."""
+ error: str
+ detail: str
+ timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+# Simulation Models
+class SimulationParameters(BaseModel):
+ """Parameters for simulation execution."""
+ model_config = {"extra": "allow"} # Allow additional parameters
+
+ @validator('*', pre=True)
+ def convert_to_number(cls, v):
+ """Convert string numbers to float/int."""
+ if isinstance(v, str) and v.replace('.', '').replace('-', '').isdigit():
+ if '.' in v:
+ return float(v)
+ return int(v)
+ return v
+
+
+class SingleSimulationRequest(BaseModel):
+ """Request for single simulation execution."""
+ model_id: str = Field(..., description="ID of the simulation model")
+ parameters: SimulationParameters = Field(..., description="Simulation parameters")
+
+
+class BatchSimulationRequest(BaseModel):
+ """Request for batch simulation execution."""
+ model_id: str = Field(..., description="ID of the simulation model")
+ parameter_grid: List[SimulationParameters] = Field(..., description="List of parameter sets")
+
+ @validator('parameter_grid')
+ def validate_grid_size(cls, v):
+ if len(v) > 1000: # Max batch size
+ raise ValueError("Batch size cannot exceed 1000")
+ return v
+
+
+class SimulationResult(BaseModel):
+ """Result from simulation execution."""
+ success: bool
+ parameters: Dict[str, Any]
+ results: Dict[str, Any]
+ execution_time: float
+ stdout: str = ""
+ stderr: str = ""
+ error_message: Optional[str] = None
+
+
+class BatchSimulationResponse(BaseModel):
+ """Response for batch simulation."""
+ status: str
+ total_runs: int
+ successful_runs: int
+ failed_runs: int
+ results: List[SimulationResult]
+ execution_time: float
+
+
+# Reasoning Models
+class ReasoningRequest(BaseModel):
+ """Request for reasoning agent."""
+ model_id: str = Field(..., description="ID of the model to analyze")
+ question: str = Field(..., min_length=1, description="Question to ask the reasoning agent")
+ max_steps: Optional[int] = Field(20, ge=1, le=50, description="Maximum reasoning steps")
+
+
+class ReasoningResponse(BaseModel):
+ """Response from reasoning agent."""
+ answer: str
+ model_id: str
+ question: str
+ history: List[Dict[str, Any]]
+ code_map: Dict[int, str]
+ images: List[str]
+ execution_time: float
+
+
+# Database Models
+class ModelMetadata(BaseModel):
+ """Metadata for simulation model."""
+ model_name: str
+ description: Optional[str] = None
+ parameters: Dict[str, Any] = {}
+ author: Optional[str] = None
+ version: Optional[str] = "1.0"
+ tags: List[str] = []
+
+
+class StoreModelRequest(BaseModel):
+ """Request to store a new simulation model."""
+ model_name: str = Field(..., min_length=1)
+ metadata: ModelMetadata
+ script_content: str = Field(..., min_length=1, description="Python script content")
+
+
+class StoreModelResponse(BaseModel):
+ """Response after storing a model."""
+ model_id: str
+ status: str
+ message: str
+
+
+class ModelInfo(BaseModel):
+ """Information about a simulation model."""
+ id: str
+ name: str
+ metadata: Dict[str, Any]
+ script_path: str
+ created_at: Optional[str] = None
+
+
+class ResultsQuery(BaseModel):
+ """Query parameters for results."""
+ model_id: Optional[str] = None
+ limit: Optional[int] = Field(100, ge=1, le=10000)
+ offset: Optional[int] = Field(0, ge=0)
+
+
+class ResultsResponse(BaseModel):
+ """Response with simulation results."""
+ total_count: int
+ results: List[Dict[str, Any]]
+ model_id: Optional[str] = None
+
+
+# Health Check Models
+class HealthStatus(str, Enum):
+ """Health status enumeration."""
+ HEALTHY = "healthy"
+ UNHEALTHY = "unhealthy"
+ DEGRADED = "degraded"
+
+
+class ComponentHealth(BaseModel):
+ """Health status of a component."""
+ name: str
+ status: HealthStatus
+ message: str
+ last_check: datetime
+
+
+class HealthResponse(BaseModel):
+ """Overall health response."""
+ status: HealthStatus
+ components: List[ComponentHealth]
+ timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+# Test Models
+class TestRequest(BaseModel):
+ """Request for testing functionality."""
+ test_type: str = Field(..., description="Type of test to run")
+ parameters: Dict[str, Any] = Field(default_factory=dict)
+
+
+class TestResponse(BaseModel):
+ """Response from test execution."""
+ test_type: str
+ success: bool
+ message: str
+ details: Dict[str, Any] = Field(default_factory=dict)
+ execution_time: float
diff --git a/modules/research-framework/simexr_mod/api/routers/__init__.py b/modules/research-framework/simexr_mod/api/routers/__init__.py
new file mode 100644
index 0000000..a7a6d77
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/routers/__init__.py
@@ -0,0 +1 @@
+"""API router modules."""
diff --git a/modules/research-framework/simexr_mod/api/routers/database.py b/modules/research-framework/simexr_mod/api/routers/database.py
new file mode 100644
index 0000000..a05ba1b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/routers/database.py
@@ -0,0 +1,438 @@
+"""
+Database management API endpoints.
+"""
+
+import json
+import tempfile
+from typing import List, Optional
+from pathlib import Path
+from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
+
+from db import store_simulation_script
+from ..models import (
+ StoreModelRequest, StoreModelResponse, ModelInfo, ResultsQuery,
+ ResultsResponse, StatusResponse
+)
+from ..dependencies import get_database
+
+
+router = APIRouter()
+
+
+@router.post("/models", response_model=StoreModelResponse, summary="Store new simulation model")
+async def store_model(request: StoreModelRequest, db = Depends(get_database)):
+ """
+ Store a new simulation model in the database.
+
+ - **model_name**: Name of the simulation model
+ - **metadata**: Model metadata and configuration
+ - **script_content**: Python script content for the simulation
+
+ Returns the generated model ID.
+ """
+ try:
+ # Create temporary script file
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(request.script_content)
+ temp_script_path = f.name
+
+ try:
+ # Store in database
+ model_id = store_simulation_script(
+ model_name=request.model_name,
+ metadata=request.metadata.model_dump(),
+ script_path=temp_script_path
+ )
+
+ return StoreModelResponse(
+ model_id=model_id,
+ status="success",
+ message=f"Model {request.model_name} stored successfully"
+ )
+
+ finally:
+ # Clean up temporary file
+ Path(temp_script_path).unlink(missing_ok=True)
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to store model: {str(e)}")
+
+
+@router.post("/models/upload", response_model=StoreModelResponse, summary="Upload simulation model file")
+async def upload_model(
+ model_name: str,
+ metadata: str, # JSON string
+ script_file: UploadFile = File(...),
+ db = Depends(get_database)
+):
+ """
+ Upload a simulation model from a file.
+
+ - **model_name**: Name of the simulation model
+ - **metadata**: JSON string containing model metadata
+ - **script_file**: Python script file (.py)
+
+ Returns the generated model ID.
+ """
+ try:
+ # Validate file type
+ if not script_file.filename.endswith('.py'):
+ raise HTTPException(status_code=400, detail="File must be a Python script (.py)")
+
+ # Parse metadata
+ try:
+ metadata_dict = json.loads(metadata)
+ except json.JSONDecodeError:
+ raise HTTPException(status_code=400, detail="Invalid JSON in metadata")
+
+ # Read script content
+ script_content = await script_file.read()
+ script_content = script_content.decode('utf-8')
+
+ # Create temporary script file
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(script_content)
+ temp_script_path = f.name
+
+ try:
+ # Store in database
+ model_id = store_simulation_script(
+ model_name=model_name,
+ metadata=metadata_dict,
+ script_path=temp_script_path
+ )
+
+ return StoreModelResponse(
+ model_id=model_id,
+ status="success",
+ message=f"Model {model_name} uploaded successfully"
+ )
+
+ finally:
+ # Clean up temporary file
+ Path(temp_script_path).unlink(missing_ok=True)
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to upload model: {str(e)}")
+
+
+@router.get("/models", summary="List all models")
+async def list_all_models(
+ limit: int = 100,
+ offset: int = 0,
+ db = Depends(get_database)
+):
+ """
+ Get a list of all simulation models.
+
+ - **limit**: Maximum number of models to return (default: 100)
+ - **offset**: Number of models to skip (default: 0)
+
+ Returns paginated list of models.
+ """
+ try:
+ models = db.simulation_repository.list()
+
+ # Apply pagination
+ total_count = len(models)
+ models_page = models[offset:offset + limit]
+
+ return {
+ "status": "success",
+ "total_count": total_count,
+ "limit": limit,
+ "offset": offset,
+ "models": models_page
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to list models: {str(e)}")
+
+
+@router.get("/models/{model_id}", summary="Get model details")
+async def get_model_details(model_id: str, db = Depends(get_database)):
+ """
+ Get detailed information about a specific model.
+
+ - **model_id**: ID of the simulation model
+
+ Returns model metadata, script information, and statistics.
+ """
+ try:
+ # Get model info
+ models = db.simulation_repository.list({"id": model_id})
+ if not models:
+ raise HTTPException(status_code=404, detail=f"Model {model_id} not found")
+
+ model_info = models[0]
+
+ # Get result statistics
+ with db.config.get_sqlite_connection() as conn:
+ stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_runs,
+ MIN(ts) as first_run,
+ MAX(ts) as last_run
+ FROM results
+ WHERE model_id = ?
+ """, (model_id,)).fetchone()
+
+ reasoning_stats = conn.execute("""
+ SELECT COUNT(*) as conversation_count
+ FROM reasoning_agent
+ WHERE model_id = ?
+ """, (model_id,)).fetchone()
+
+ return {
+ "status": "success",
+ "model": model_info,
+ "statistics": {
+ "total_simulation_runs": stats["total_runs"] if stats else 0,
+ "first_run": stats["first_run"] if stats else None,
+ "last_run": stats["last_run"] if stats else None,
+ "reasoning_conversations": reasoning_stats["conversation_count"] if reasoning_stats else 0
+ }
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get model details: {str(e)}")
+
+
+@router.delete("/models/{model_id}", summary="Delete model")
+async def delete_model(model_id: str, db = Depends(get_database)):
+ """
+ Delete a simulation model and all associated data.
+
+ - **model_id**: ID of the simulation model
+
+ Returns confirmation of deletion.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ # Check if model exists
+ model = conn.execute(
+ "SELECT id FROM simulations WHERE id = ?",
+ (model_id,)
+ ).fetchone()
+
+ if not model:
+ raise HTTPException(status_code=404, detail=f"Model {model_id} not found")
+
+ # Delete associated data
+ results_deleted = conn.execute(
+ "DELETE FROM results WHERE model_id = ?",
+ (model_id,)
+ ).rowcount
+
+ reasoning_deleted = conn.execute(
+ "DELETE FROM reasoning_agent WHERE model_id = ?",
+ (model_id,)
+ ).rowcount
+
+ # Delete model
+ conn.execute("DELETE FROM simulations WHERE id = ?", (model_id,))
+
+ return {
+ "status": "success",
+ "message": f"Model {model_id} deleted successfully",
+ "deleted_results": results_deleted,
+ "deleted_conversations": reasoning_deleted
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to delete model: {str(e)}")
+
+
+@router.get("/results", summary="Get all simulation results")
+async def get_all_results(
+ model_id: Optional[str] = None,
+ limit: int = 100,
+ offset: int = 0,
+ db = Depends(get_database)
+):
+ """
+ Get simulation results across all or specific models.
+
+ - **model_id**: Optional filter by model ID
+ - **limit**: Maximum number of results to return (default: 100)
+ - **offset**: Number of results to skip (default: 0)
+
+ Returns paginated simulation results.
+ """
+ try:
+ if model_id:
+ # Load results for specific model
+ df = db.results_service.load_results(model_id=model_id)
+ else:
+ # Load all results
+ df = db.results_service.load_results()
+
+ # Apply pagination
+ total_count = len(df)
+ df_page = df.iloc[offset:offset + limit]
+
+ # Clean NaN values for JSON serialization
+ import numpy as np
+
+ def clean_nan_values(obj):
+ """Recursively replace NaN values with None for JSON serialization."""
+ if isinstance(obj, dict):
+ return {k: clean_nan_values(v) for k, v in obj.items()}
+ elif isinstance(obj, list):
+ return [clean_nan_values(item) for item in obj]
+ elif isinstance(obj, (np.floating, float)) and np.isnan(obj):
+ return None
+ elif isinstance(obj, np.integer):
+ return int(obj)
+ elif isinstance(obj, np.floating):
+ return float(obj)
+ elif isinstance(obj, np.ndarray):
+ return clean_nan_values(obj.tolist())
+ else:
+ return obj
+
+ # Convert to records and clean NaN values
+ results = df_page.to_dict('records')
+ cleaned_results = clean_nan_values(results)
+
+ # Log preview of results
+ print(f"=== DATABASE RESULTS PREVIEW (First 5 rows) ===")
+ print(f"Total count: {total_count}, Limit: {limit}, Offset: {offset}")
+ for i, result in enumerate(cleaned_results[:5]):
+ print(f"Row {i+1}: {list(result.keys())}")
+ # Show key values for first few rows
+ if i < 3: # Show more details for first 3 rows
+ for key, value in list(result.items())[:10]: # First 10 keys
+ if isinstance(value, (list, tuple)) and len(value) > 5:
+ print(f" {key}: {type(value).__name__} with {len(value)} items (first 3: {value[:3]})")
+ elif isinstance(value, dict):
+ print(f" {key}: dict with {len(value)} keys")
+ else:
+ print(f" {key}: {value}")
+ print(f"=== END DATABASE RESULTS PREVIEW ===")
+
+ return {
+ "status": "success",
+ "total_count": total_count,
+ "limit": limit,
+ "offset": offset,
+ "model_id": model_id,
+ "results": cleaned_results
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get results: {str(e)}")
+
+
+@router.get("/stats", summary="Get database statistics")
+async def get_database_stats(db = Depends(get_database)):
+ """
+ Get overall database statistics.
+
+ Returns counts and statistics for all database entities.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ # Model statistics
+ models_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_models,
+ MIN(created_at) as first_model,
+ MAX(created_at) as last_model
+ FROM simulations
+ """).fetchone()
+
+ # Results statistics
+ results_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_results,
+ COUNT(DISTINCT model_id) as models_with_results,
+ MIN(ts) as first_result,
+ MAX(ts) as last_result
+ FROM results
+ """).fetchone()
+
+ # Reasoning statistics
+ reasoning_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_conversations,
+ COUNT(DISTINCT model_id) as models_with_conversations,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ """).fetchone()
+
+ # Database size (SQLite specific)
+ size_stats = conn.execute("PRAGMA page_count").fetchone()
+ page_size = conn.execute("PRAGMA page_size").fetchone()
+
+ db_size_bytes = (size_stats[0] if size_stats else 0) * (page_size[0] if page_size else 0)
+ db_size_mb = round(db_size_bytes / (1024 * 1024), 2)
+
+ return {
+ "status": "success",
+ "database": {
+ "path": db.config.database_path,
+ "size_mb": db_size_mb
+ },
+ "models": {
+ "total": models_stats["total_models"] if models_stats else 0,
+ "first_created": models_stats["first_model"] if models_stats else None,
+ "last_created": models_stats["last_model"] if models_stats else None
+ },
+ "results": {
+ "total": results_stats["total_results"] if results_stats else 0,
+ "models_with_results": results_stats["models_with_results"] if results_stats else 0,
+ "first_result": results_stats["first_result"] if results_stats else None,
+ "last_result": results_stats["last_result"] if results_stats else None
+ },
+ "reasoning": {
+ "total_conversations": reasoning_stats["total_conversations"] if reasoning_stats else 0,
+ "models_with_conversations": reasoning_stats["models_with_conversations"] if reasoning_stats else 0,
+ "first_conversation": reasoning_stats["first_conversation"] if reasoning_stats else None,
+ "last_conversation": reasoning_stats["last_conversation"] if reasoning_stats else None
+ }
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get database stats: {str(e)}")
+
+
+@router.post("/backup", summary="Create database backup")
+async def create_backup(db = Depends(get_database)):
+ """
+ Create a backup of the database.
+
+ Returns the backup file path and statistics.
+ """
+ try:
+ import shutil
+ from datetime import datetime
+
+ # Create backup filename with timestamp
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ backup_path = f"{db.config.database_path}.backup_{timestamp}"
+
+ # Copy database file
+ shutil.copy2(db.config.database_path, backup_path)
+
+ # Get backup file size
+ backup_size = Path(backup_path).stat().st_size
+ backup_size_mb = round(backup_size / (1024 * 1024), 2)
+
+ return {
+ "status": "success",
+ "message": "Database backup created successfully",
+ "backup_path": backup_path,
+ "backup_size_mb": backup_size_mb,
+ "timestamp": timestamp
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to create backup: {str(e)}")
diff --git a/modules/research-framework/simexr_mod/api/routers/health.py b/modules/research-framework/simexr_mod/api/routers/health.py
new file mode 100644
index 0000000..551ca53
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/routers/health.py
@@ -0,0 +1,407 @@
+"""
+Health check and testing API endpoints.
+"""
+
+import time
+import tempfile
+from typing import Dict, Any
+from pathlib import Path
+from fastapi import APIRouter, HTTPException, Depends
+from datetime import datetime
+
+from execute import SimulationRunner
+from reasoning import ReasoningAgent
+from ..models import HealthResponse, HealthStatus, ComponentHealth, TestRequest, TestResponse
+from ..dependencies import get_database
+
+
+router = APIRouter()
+
+
+@router.get("/status", response_model=HealthResponse, summary="System health status")
+async def health_status(db = Depends(get_database)):
+ """
+ Get overall system health status.
+
+ Checks all major components and returns their health status.
+ """
+ components = []
+ overall_status = HealthStatus.HEALTHY
+
+ # Check database
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ conn.execute("SELECT 1").fetchone()
+
+ components.append(ComponentHealth(
+ name="database",
+ status=HealthStatus.HEALTHY,
+ message="Database connection successful",
+ last_check=datetime.utcnow()
+ ))
+ except Exception as e:
+ components.append(ComponentHealth(
+ name="database",
+ status=HealthStatus.UNHEALTHY,
+ message=f"Database error: {str(e)}",
+ last_check=datetime.utcnow()
+ ))
+ overall_status = HealthStatus.UNHEALTHY
+
+ # Check simulation execution
+ try:
+ # Create a simple test script
+ test_script = '''
+def simulate(**params):
+ return {"result": params.get("x", 0) * 2}
+'''
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(test_script)
+ temp_path = f.name
+
+ try:
+ runner = SimulationRunner()
+ result = runner.run(Path(temp_path), {"x": 5})
+
+ if result.get("_ok", False) and result.get("result") == 10:
+ components.append(ComponentHealth(
+ name="simulation_execution",
+ status=HealthStatus.HEALTHY,
+ message="Simulation execution working",
+ last_check=datetime.utcnow()
+ ))
+ else:
+ components.append(ComponentHealth(
+ name="simulation_execution",
+ status=HealthStatus.DEGRADED,
+ message="Simulation execution returned unexpected results",
+ last_check=datetime.utcnow()
+ ))
+ if overall_status == HealthStatus.HEALTHY:
+ overall_status = HealthStatus.DEGRADED
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+ except Exception as e:
+ components.append(ComponentHealth(
+ name="simulation_execution",
+ status=HealthStatus.UNHEALTHY,
+ message=f"Simulation execution error: {str(e)}",
+ last_check=datetime.utcnow()
+ ))
+ overall_status = HealthStatus.UNHEALTHY
+
+ # Check models directory
+ try:
+ from ..config import settings
+ models_dir = Path(settings.models_dir)
+
+ if models_dir.exists():
+ model_count = len(list(models_dir.glob("*/simulate.py")))
+ components.append(ComponentHealth(
+ name="models_directory",
+ status=HealthStatus.HEALTHY,
+ message=f"Models directory accessible, {model_count} models found",
+ last_check=datetime.utcnow()
+ ))
+ else:
+ components.append(ComponentHealth(
+ name="models_directory",
+ status=HealthStatus.DEGRADED,
+ message="Models directory not found",
+ last_check=datetime.utcnow()
+ ))
+ if overall_status == HealthStatus.HEALTHY:
+ overall_status = HealthStatus.DEGRADED
+
+ except Exception as e:
+ components.append(ComponentHealth(
+ name="models_directory",
+ status=HealthStatus.UNHEALTHY,
+ message=f"Models directory error: {str(e)}",
+ last_check=datetime.utcnow()
+ ))
+ overall_status = HealthStatus.UNHEALTHY
+
+ return HealthResponse(
+ status=overall_status,
+ components=components
+ )
+
+
+@router.post("/test", response_model=TestResponse, summary="Run system tests")
+async def run_test(request: TestRequest, db = Depends(get_database)):
+ """
+ Run specific system tests.
+
+ - **test_type**: Type of test to run (simulation, database, reasoning, etc.)
+ - **parameters**: Optional test parameters
+
+ Returns test results and performance metrics.
+ """
+ start_time = time.time()
+
+ try:
+ if request.test_type == "simulation":
+ return await _test_simulation(request.parameters)
+ elif request.test_type == "database":
+ return await _test_database(db, request.parameters)
+ elif request.test_type == "reasoning":
+ return await _test_reasoning(db, request.parameters)
+ elif request.test_type == "end_to_end":
+ return await _test_end_to_end(db, request.parameters)
+ else:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unknown test type: {request.test_type}"
+ )
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ return TestResponse(
+ test_type=request.test_type,
+ success=False,
+ message=f"Test failed: {str(e)}",
+ execution_time=execution_time
+ )
+
+
+async def _test_simulation(parameters: Dict[str, Any]) -> TestResponse:
+ """Test simulation execution."""
+ start_time = time.time()
+
+ # Create test script
+ test_script = '''
+def simulate(x=1, y=2):
+ import math
+ result = x * y + math.sqrt(x + y)
+ return {
+ "product": x * y,
+ "sum": x + y,
+ "result": result
+ }
+'''
+
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(test_script)
+ temp_path = f.name
+
+ try:
+ runner = SimulationRunner()
+ test_params = parameters.get("params", {"x": 3, "y": 4})
+ result = runner.run(Path(temp_path), test_params)
+
+ execution_time = time.time() - start_time
+
+ success = result.get("_ok", False)
+ details = {
+ "input_params": test_params,
+ "output_keys": list(result.keys()),
+ "execution_time_s": result.get("_duration_s", 0),
+ "stdout_length": len(result.get("_stdout", "")),
+ "stderr_length": len(result.get("_stderr", ""))
+ }
+
+ if success:
+ expected_product = test_params["x"] * test_params["y"]
+ actual_product = result.get("product")
+ if actual_product != expected_product:
+ success = False
+ details["validation_error"] = f"Expected product {expected_product}, got {actual_product}"
+
+ return TestResponse(
+ test_type="simulation",
+ success=success,
+ message="Simulation test completed" if success else "Simulation test failed",
+ details=details,
+ execution_time=execution_time
+ )
+
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+
+async def _test_database(db, parameters: Dict[str, Any]) -> TestResponse:
+ """Test database operations."""
+ start_time = time.time()
+
+ details = {}
+
+ try:
+ # Test database connection
+ with db.config.get_sqlite_connection() as conn:
+ # Test basic query
+ result = conn.execute("SELECT COUNT(*) as count FROM simulations").fetchone()
+ details["models_count"] = result["count"] if result else 0
+
+ # Test results table
+ result = conn.execute("SELECT COUNT(*) as count FROM results").fetchone()
+ details["results_count"] = result["count"] if result else 0
+
+ # Test reasoning table
+ result = conn.execute("SELECT COUNT(*) as count FROM reasoning_agent").fetchone()
+ details["conversations_count"] = result["count"] if result else 0
+
+ # Test write operation (if requested)
+ if parameters.get("test_write", False):
+ test_id = f"test_{int(time.time())}"
+ conn.execute("""
+ INSERT INTO simulations (id, name, metadata, script_path)
+ VALUES (?, ?, ?, ?)
+ """, (test_id, "test_model", "{}", "/tmp/test.py"))
+
+ # Clean up
+ conn.execute("DELETE FROM simulations WHERE id = ?", (test_id,))
+ details["write_test"] = "passed"
+
+ execution_time = time.time() - start_time
+
+ return TestResponse(
+ test_type="database",
+ success=True,
+ message="Database test completed successfully",
+ details=details,
+ execution_time=execution_time
+ )
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ return TestResponse(
+ test_type="database",
+ success=False,
+ message=f"Database test failed: {str(e)}",
+ details=details,
+ execution_time=execution_time
+ )
+
+
+async def _test_reasoning(db, parameters: Dict[str, Any]) -> TestResponse:
+ """Test reasoning agent (mock test)."""
+ start_time = time.time()
+
+ # For now, just test that we can create a reasoning agent
+ # In a real test, we'd need a valid model with results
+ try:
+ details = {
+ "reasoning_agent_class": "ReasoningAgent",
+ "test_mode": "mock",
+ "note": "Full reasoning test requires valid model with simulation results"
+ }
+
+ # Check if we have any models to test with
+ models = db.simulation_repository.list()
+ if models:
+ details["available_models"] = len(models)
+ details["sample_model"] = models[0].get("id") if models else None
+ else:
+ details["available_models"] = 0
+
+ execution_time = time.time() - start_time
+
+ return TestResponse(
+ test_type="reasoning",
+ success=True,
+ message="Reasoning test completed (mock mode)",
+ details=details,
+ execution_time=execution_time
+ )
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ return TestResponse(
+ test_type="reasoning",
+ success=False,
+ message=f"Reasoning test failed: {str(e)}",
+ execution_time=execution_time
+ )
+
+
+async def _test_end_to_end(db, parameters: Dict[str, Any]) -> TestResponse:
+ """Test complete end-to-end workflow."""
+ start_time = time.time()
+
+ details = {}
+
+ try:
+ # 1. Create and store a test model
+ test_script = '''
+def simulate(amplitude=1.0, frequency=1.0, phase=0.0):
+ import math
+ import numpy as np
+
+ t = np.linspace(0, 2*math.pi, 100)
+ signal = amplitude * np.sin(frequency * t + phase)
+
+ return {
+ "max_value": float(np.max(signal)),
+ "min_value": float(np.min(signal)),
+ "mean_value": float(np.mean(signal)),
+ "signal_length": len(signal)
+ }
+'''
+
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(test_script)
+ temp_path = f.name
+
+ try:
+ # Store model
+ from db import store_simulation_script
+ model_id = store_simulation_script(
+ model_name="e2e_test_model",
+ metadata={"description": "End-to-end test model", "test": True},
+ script_path=temp_path
+ )
+ details["model_created"] = model_id
+
+ # 2. Run simulation
+ runner = SimulationRunner()
+ result = runner.run(Path(temp_path), {"amplitude": 2.0, "frequency": 0.5})
+
+ details["simulation_success"] = result.get("_ok", False)
+ details["simulation_results"] = {k: v for k, v in result.items() if not k.startswith("_")}
+
+ # 3. Store results
+ if result.get("_ok", False):
+ from db import store_simulation_results
+ store_simulation_results(model_id, [result], ["amplitude", "frequency"])
+ details["results_stored"] = True
+
+ # 4. Test reasoning (mock)
+ details["reasoning_available"] = True
+
+ # 5. Clean up
+ with db.config.get_sqlite_connection() as conn:
+ conn.execute("DELETE FROM results WHERE model_id = ?", (model_id,))
+ conn.execute("DELETE FROM simulations WHERE id = ?", (model_id,))
+ details["cleanup_completed"] = True
+
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+ execution_time = time.time() - start_time
+
+ success = all([
+ details.get("model_created"),
+ details.get("simulation_success", False),
+ details.get("results_stored", False),
+ details.get("cleanup_completed", False)
+ ])
+
+ return TestResponse(
+ test_type="end_to_end",
+ success=success,
+ message="End-to-end test completed" if success else "End-to-end test had failures",
+ details=details,
+ execution_time=execution_time
+ )
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ return TestResponse(
+ test_type="end_to_end",
+ success=False,
+ message=f"End-to-end test failed: {str(e)}",
+ details=details,
+ execution_time=execution_time
+ )
diff --git a/modules/research-framework/simexr_mod/api/routers/reasoning.py b/modules/research-framework/simexr_mod/api/routers/reasoning.py
new file mode 100644
index 0000000..744f6ba
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/routers/reasoning.py
@@ -0,0 +1,268 @@
+"""
+Reasoning agent API endpoints.
+"""
+
+import time
+from typing import List, Optional
+from fastapi import APIRouter, HTTPException, Depends
+
+from reasoning import ReasoningAgent
+from db.config.database import DatabaseConfig
+from ..models import ReasoningRequest, ReasoningResponse, StatusResponse
+from ..dependencies import get_database
+
+
+router = APIRouter()
+
+
+@router.post("/ask", response_model=ReasoningResponse, summary="Ask reasoning agent")
+async def ask_reasoning_agent(
+ request: ReasoningRequest,
+ db = Depends(get_database)
+):
+ """
+ Ask the reasoning agent a question about a simulation model.
+
+ - **model_id**: ID of the simulation model to analyze
+ - **question**: Question to ask the reasoning agent
+ - **max_steps**: Maximum number of reasoning steps (default: 20)
+
+ Returns the agent's analysis and answer.
+ """
+ try:
+ start_time = time.time()
+
+ # Verify model exists
+ from db import get_simulation_path
+ try:
+ get_simulation_path(request.model_id)
+ except KeyError:
+ raise HTTPException(status_code=404, detail=f"Model {request.model_id} not found")
+
+ # Create reasoning agent
+ agent = ReasoningAgent(
+ model_id=request.model_id,
+ db_config=db.config,
+ max_steps=request.max_steps or 20
+ )
+
+ # Ask question
+ result = agent.ask(request.question)
+
+ execution_time = time.time() - start_time
+
+ return ReasoningResponse(
+ answer=result.answer,
+ model_id=request.model_id,
+ question=request.question,
+ history=result.history,
+ code_map=result.code_map,
+ images=result.images,
+ execution_time=execution_time
+ )
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Reasoning failed: {str(e)}")
+
+
+@router.get("/history/{model_id}", summary="Get reasoning history")
+async def get_reasoning_history(
+ model_id: str,
+ limit: int = 50,
+ offset: int = 0,
+ db = Depends(get_database)
+):
+ """
+ Get reasoning conversation history for a model.
+
+ - **model_id**: ID of the simulation model
+ - **limit**: Maximum number of conversations to return (default: 50)
+ - **offset**: Number of conversations to skip (default: 0)
+
+ Returns paginated reasoning history.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ # Get total count
+ count_result = conn.execute(
+ "SELECT COUNT(*) as count FROM reasoning_agent WHERE model_id = ?",
+ (model_id,)
+ ).fetchone()
+ total_count = count_result["count"] if count_result else 0
+
+ # Get paginated results
+ rows = conn.execute("""
+ SELECT id, model_id, question, answer, images, ts
+ FROM reasoning_agent
+ WHERE model_id = ?
+ ORDER BY ts DESC
+ LIMIT ? OFFSET ?
+ """, (model_id, limit, offset)).fetchall()
+
+ history = []
+ for row in rows:
+ history.append({
+ "id": row["id"],
+ "model_id": row["model_id"],
+ "question": row["question"],
+ "answer": row["answer"],
+ "images": row["images"],
+ "timestamp": row["ts"]
+ })
+
+ return {
+ "status": "success",
+ "total_count": total_count,
+ "limit": limit,
+ "offset": offset,
+ "history": history
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get history: {str(e)}")
+
+
+@router.delete("/history/{model_id}", summary="Clear reasoning history")
+async def clear_reasoning_history(model_id: str, db = Depends(get_database)):
+ """
+ Clear all reasoning history for a specific model.
+
+ - **model_id**: ID of the simulation model
+
+ Returns confirmation of deletion.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ cursor = conn.execute(
+ "DELETE FROM reasoning_agent WHERE model_id = ?",
+ (model_id,)
+ )
+ deleted_count = cursor.rowcount
+
+ return {
+ "status": "success",
+ "message": f"Deleted {deleted_count} reasoning conversations for model {model_id}",
+ "deleted_count": deleted_count
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to clear history: {str(e)}")
+
+
+@router.get("/conversations", summary="Get all reasoning conversations")
+async def get_all_conversations(
+ limit: int = 100,
+ offset: int = 0,
+ model_id: Optional[str] = None,
+ db = Depends(get_database)
+):
+ """
+ Get all reasoning conversations across all models.
+
+ - **limit**: Maximum number of conversations to return (default: 100)
+ - **offset**: Number of conversations to skip (default: 0)
+ - **model_id**: Optional filter by model ID
+
+ Returns paginated conversation list.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ # Build query
+ base_query = "FROM reasoning_agent"
+ params = []
+
+ if model_id:
+ base_query += " WHERE model_id = ?"
+ params.append(model_id)
+
+ # Get total count
+ count_result = conn.execute(
+ f"SELECT COUNT(*) as count {base_query}",
+ params
+ ).fetchone()
+ total_count = count_result["count"] if count_result else 0
+
+ # Get paginated results
+ rows = conn.execute(f"""
+ SELECT id, model_id, question, answer, images, ts
+ {base_query}
+ ORDER BY ts DESC
+ LIMIT ? OFFSET ?
+ """, params + [limit, offset]).fetchall()
+
+ conversations = []
+ for row in rows:
+ conversations.append({
+ "id": row["id"],
+ "model_id": row["model_id"],
+ "question": row["question"],
+ "answer": row["answer"],
+ "images": row["images"],
+ "timestamp": row["ts"]
+ })
+
+ return {
+ "status": "success",
+ "total_count": total_count,
+ "limit": limit,
+ "offset": offset,
+ "conversations": conversations
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get conversations: {str(e)}")
+
+
+@router.get("/stats", summary="Get reasoning statistics")
+async def get_reasoning_stats(db = Depends(get_database)):
+ """
+ Get statistics about reasoning agent usage.
+
+ Returns overall statistics and per-model breakdown.
+ """
+ try:
+ with db.config.get_sqlite_connection() as conn:
+ # Overall stats
+ overall = conn.execute("""
+ SELECT
+ COUNT(*) as total_conversations,
+ COUNT(DISTINCT model_id) as unique_models,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ """).fetchone()
+
+ # Per-model stats
+ per_model = conn.execute("""
+ SELECT
+ model_id,
+ COUNT(*) as conversation_count,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ GROUP BY model_id
+ ORDER BY conversation_count DESC
+ """).fetchall()
+
+ model_stats = []
+ for row in per_model:
+ model_stats.append({
+ "model_id": row["model_id"],
+ "conversation_count": row["conversation_count"],
+ "first_conversation": row["first_conversation"],
+ "last_conversation": row["last_conversation"]
+ })
+
+ return {
+ "status": "success",
+ "overall": {
+ "total_conversations": overall["total_conversations"] if overall else 0,
+ "unique_models": overall["unique_models"] if overall else 0,
+ "first_conversation": overall["first_conversation"] if overall else None,
+ "last_conversation": overall["last_conversation"] if overall else None
+ },
+ "per_model": model_stats
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get stats: {str(e)}")
diff --git a/modules/research-framework/simexr_mod/api/routers/simulation.py b/modules/research-framework/simexr_mod/api/routers/simulation.py
new file mode 100644
index 0000000..9d97b06
--- /dev/null
+++ b/modules/research-framework/simexr_mod/api/routers/simulation.py
@@ -0,0 +1,510 @@
+"""
+Simulation execution API endpoints.
+"""
+
+import time
+from typing import List
+from fastapi import APIRouter, HTTPException, Depends, Request
+from pathlib import Path
+
+from ..models import (
+ SingleSimulationRequest, BatchSimulationRequest, SimulationResult,
+ BatchSimulationResponse, StatusResponse, ErrorResponse
+)
+from ..dependencies import get_simulation_service, get_data_service, get_database
+from core.interfaces import SimulationStatus
+
+
+router = APIRouter()
+
+
+@router.post("/import/github", summary="Import simulation from GitHub")
+async def import_from_github(
+ github_url: str,
+ model_name: str,
+ description: str = "",
+ simulation_service = Depends(get_simulation_service)
+):
+ """
+ Import a simulation model from a GitHub URL.
+
+ - **github_url**: URL to the GitHub script (e.g., https://github.com/user/repo/blob/main/script.py)
+ - **model_name**: Name for the imported model
+ - **description**: Optional description of the model
+
+ Returns the generated model ID.
+ """
+ try:
+ # Extract parameters info from the script if possible
+ parameters = {
+ "github_url": "Source URL",
+ "imported": "Imported from GitHub"
+ }
+
+ model_id = simulation_service.import_model_from_github(
+ github_url=github_url,
+ model_name=model_name,
+ description=description,
+ parameters=parameters
+ )
+
+ return {
+ "status": "success",
+ "model_id": model_id,
+ "message": f"Successfully imported model from {github_url}",
+ "github_url": github_url,
+ "model_name": model_name
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to import from GitHub: {str(e)}")
+
+
+@router.post("/transform/github", summary="Transform GitHub script using transform_code")
+async def transform_github_script(
+ github_url: str,
+ model_name: str,
+ max_smoke_iters: int = 3
+):
+ """
+ Transform a GitHub script using the transform_code module.
+
+ This endpoint uses ExternalScriptImporter to:
+ 1. Import the script from GitHub
+ 2. Refactor it to have a simulate(**params) function
+ 3. Refine it through smoke testing and fixes
+ 4. Return the model_id and metadata
+
+ - **github_url**: URL to the GitHub script
+ - **model_name**: Name for the imported model
+ - **max_smoke_iters**: Maximum smoke test iterations (default: 3)
+
+ Returns the generated model ID and processing details.
+ """
+ try:
+ print(f"[TRANSFORM API] Starting transform process for {github_url}")
+ from execute.loader.transform_code import ExternalScriptImporter
+ import tempfile
+ import os
+
+ # Create importer
+ print("[TRANSFORM API] Creating ExternalScriptImporter...")
+ importer = ExternalScriptImporter()
+
+ # Create temporary directory for processing
+ print(f"[TRANSFORM API] Creating temporary directory...")
+ with tempfile.TemporaryDirectory() as temp_dir:
+ print(f"[TRANSFORM API] Temporary directory created: {temp_dir}")
+ # Import and refactor using transform_code
+ print(f"[TRANSFORM API] Calling import_and_refactor...")
+ model_id, metadata = importer.import_and_refactor(
+ source_url=github_url,
+ model_name=model_name,
+ dest_dir=temp_dir,
+ max_smoke_iters=max_smoke_iters
+ )
+
+ print(f"[TRANSFORM API] Import and refactor completed. Model ID: {model_id}")
+ # Get the final script path from the database
+ from db import get_simulation_path
+ try:
+ script_path = get_simulation_path(model_id)
+ print(f"[TRANSFORM API] Script path from database: {script_path}")
+ except:
+ # Fallback to expected path
+ script_path = f"external_models/{model_name}.py"
+ print(f"[TRANSFORM API] Using fallback script path: {script_path}")
+
+ # Read the final script content
+ print(f"[TRANSFORM API] Reading script content...")
+ with open(script_path, 'r') as f:
+ script_content = f.read()
+ print(f"[TRANSFORM API] Script content length: {len(script_content)}")
+
+ return {
+ "status": "success",
+ "model_id": model_id,
+ "message": f"Successfully transformed script from {github_url}",
+ "github_url": github_url,
+ "model_name": model_name,
+ "script_path": script_path,
+ "script_content": script_content,
+ "metadata": metadata,
+ "processing_details": {
+ "max_smoke_iters": max_smoke_iters,
+ "script_size": len(script_content),
+ "temp_directory": temp_dir
+ }
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to transform GitHub script: {str(e)}")
+
+
+@router.post("/run", response_model=SimulationResult, summary="Run single simulation")
+async def run_single_simulation(
+ request: SingleSimulationRequest,
+ simulation_service = Depends(get_simulation_service)
+):
+ """
+ Execute a single simulation with given parameters.
+
+ - **model_id**: ID of the simulation model
+ - **parameters**: Dictionary of simulation parameters
+
+ Returns the simulation result with outputs and execution metadata.
+ """
+ try:
+ # Use the service layer
+ result = simulation_service.run_single_simulation(
+ model_id=request.model_id,
+ parameters=request.parameters.model_dump()
+ )
+
+ # Convert to API response format
+ return SimulationResult(
+ success=result.status == SimulationStatus.COMPLETED,
+ parameters=result.parameters,
+ results=result.outputs,
+ execution_time=result.execution_time,
+ stdout=result.stdout,
+ stderr=result.stderr,
+ error_message=result.error_message
+ )
+
+ except FileNotFoundError:
+ raise HTTPException(status_code=404, detail=f"Model {request.model_id} not found")
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Simulation failed: {str(e)}")
+
+
+@router.post("/batch", response_model=BatchSimulationResponse, summary="Run batch simulations")
+async def run_batch_simulation(
+ request: BatchSimulationRequest,
+ simulation_service = Depends(get_simulation_service)
+):
+ """
+ Execute multiple simulations in batch with different parameter sets.
+
+ - **model_id**: ID of the simulation model
+ - **parameter_grid**: List of parameter dictionaries
+
+ Returns batch execution results with statistics.
+ """
+ try:
+ start_time = time.time()
+
+ # Convert parameter grid
+ param_grid = [params.model_dump() for params in request.parameter_grid]
+
+ # Use the service layer
+ results = simulation_service.run_batch_simulations(
+ model_id=request.model_id,
+ parameter_grid=param_grid
+ )
+
+ # Convert to API response format
+ api_results = []
+ for result in results:
+ api_result = SimulationResult(
+ success=result.status == SimulationStatus.COMPLETED,
+ parameters=result.parameters,
+ results=result.outputs,
+ execution_time=result.execution_time,
+ stdout=result.stdout,
+ stderr=result.stderr,
+ error_message=result.error_message
+ )
+ api_results.append(api_result)
+
+ execution_time = time.time() - start_time
+ successful_runs = sum(1 for r in api_results if r.success)
+ failed_runs = len(api_results) - successful_runs
+
+ return BatchSimulationResponse(
+ status="completed",
+ total_runs=len(api_results),
+ successful_runs=successful_runs,
+ failed_runs=failed_runs,
+ results=api_results,
+ execution_time=execution_time
+ )
+
+ except FileNotFoundError:
+ raise HTTPException(status_code=404, detail=f"Model {request.model_id} not found")
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Batch simulation failed: {str(e)}")
+
+
+@router.get("/models", summary="List available simulation models")
+async def list_models(simulation_service = Depends(get_simulation_service)):
+ """
+ Get a list of all available simulation models.
+
+ Returns list of models with basic information.
+ """
+ try:
+ models = simulation_service.list_models()
+ return {
+ "status": "success",
+ "count": len(models),
+ "models": models
+ }
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to list models: {str(e)}")
+
+
+@router.get("/models/search", summary="Search models by name (fuzzy search)")
+async def search_models_by_name(
+ name: str,
+ limit: int = 20,
+ simulation_service = Depends(get_simulation_service)
+):
+ """
+ Search for simulation models by name using fuzzy matching.
+
+ - **name**: Partial name to search for (case-insensitive)
+ - **limit**: Maximum number of results to return (default: 20)
+
+ Returns models that match the search criteria.
+ """
+ try:
+ import re
+
+ # Get all models
+ all_models = simulation_service.list_models()
+
+ # Convert search term to lowercase for case-insensitive matching
+ search_term = name.lower()
+
+ # Filter models using fuzzy matching
+ matching_models = []
+ for model in all_models:
+ model_name = model.get('name', '').lower()
+ model_id = model.get('id', '').lower()
+
+ # Check if search term appears in model name or ID
+ if (search_term in model_name or
+ search_term in model_id or
+ any(word in model_name for word in search_term.split()) or
+ any(word in model_id for word in search_term.split())):
+ matching_models.append(model)
+
+ # Sort by relevance (exact matches first, then partial matches)
+ def relevance_score(model):
+ model_name = model.get('name', '').lower()
+ model_id = model.get('id', '').lower()
+
+ # Exact match gets highest score
+ if search_term == model_name or search_term == model_id:
+ return 100
+ # Starts with search term
+ elif model_name.startswith(search_term) or model_id.startswith(search_term):
+ return 90
+ # Contains search term
+ elif search_term in model_name or search_term in model_id:
+ return 80
+ # Word boundary matches
+ elif any(word in model_name for word in search_term.split()):
+ return 70
+ else:
+ return 50
+
+ # Sort by relevance and limit results
+ matching_models.sort(key=relevance_score, reverse=True)
+ limited_models = matching_models[:limit]
+
+ return {
+ "status": "success",
+ "search_term": name,
+ "total_matches": len(matching_models),
+ "returned_count": len(limited_models),
+ "limit": limit,
+ "models": limited_models
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to search models: {str(e)}")
+
+
+@router.get("/models/{model_id}", summary="Get model information")
+async def get_model_info(model_id: str, simulation_service = Depends(get_simulation_service)):
+ """
+ Get detailed information about a specific simulation model.
+
+ - **model_id**: ID of the simulation model
+
+ Returns model metadata and script information.
+ """
+ try:
+ model_info = simulation_service.get_model_info(model_id)
+
+ return {
+ "status": "success",
+ "model": model_info
+ }
+
+ except ValueError as e:
+ raise HTTPException(status_code=404, detail=str(e))
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get model info: {str(e)}")
+
+
+@router.get("/models/{model_id}/results", summary="Get simulation results")
+async def get_model_results(
+ model_id: str,
+ limit: int = 100,
+ offset: int = 0,
+ data_service = Depends(get_data_service)
+):
+ """
+ Get simulation results for a specific model.
+
+ - **model_id**: ID of the simulation model
+ - **limit**: Maximum number of results to return (default: 100)
+ - **offset**: Number of results to skip (default: 0)
+
+ Returns paginated simulation results.
+ """
+ try:
+ results_data = data_service.get_simulation_results(
+ model_id=model_id,
+ limit=limit,
+ offset=offset
+ )
+
+ return {
+ "status": "success",
+ **results_data
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get results: {str(e)}")
+
+
+@router.delete("/models/{model_id}/results", summary="Clear model results")
+async def clear_model_results(model_id: str, db = Depends(get_database)):
+ """
+ Clear all simulation results for a specific model.
+
+ - **model_id**: ID of the simulation model
+
+ Returns confirmation of deletion.
+ """
+ try:
+ # Delete results from database
+ with db.config.get_sqlite_connection() as conn:
+ cursor = conn.execute(
+ "DELETE FROM results WHERE model_id = ?",
+ (model_id,)
+ )
+ deleted_count = cursor.rowcount
+
+ return {
+ "status": "success",
+ "message": f"Deleted {deleted_count} results for model {model_id}",
+ "deleted_count": deleted_count
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to clear results: {str(e)}")
+
+
+@router.get("/models/{model_id}/script", summary="Get model script")
+async def get_model_script(model_id: str, simulation_service = Depends(get_simulation_service)):
+ """
+ Get the refactored script for a specific model.
+
+ - **model_id**: ID of the simulation model
+
+ Returns the script content.
+ """
+ try:
+ model_info = simulation_service.get_model_info(model_id)
+ script_path = model_info.get("script_path")
+
+ # Try to find the script in common locations if script_path is not available
+ if not script_path:
+ # Look for script in external_models directory
+ possible_paths = [
+ f"external_models/{model_id}.py",
+ f"external_models/{model_info.get('name', model_id)}.py",
+ f"systems/models/{model_id}.py",
+ f"systems/models/{model_info.get('name', model_id)}.py"
+ ]
+
+ for path in possible_paths:
+ if Path(path).exists():
+ script_path = path
+ break
+
+ if not script_path or not Path(script_path).exists():
+ raise HTTPException(status_code=404, detail=f"Script not found for model {model_id}")
+
+ # Read the script file
+ with open(script_path, 'r') as f:
+ script_content = f.read()
+
+ return {
+ "status": "success",
+ "model_id": model_id,
+ "script": script_content,
+ "script_path": script_path,
+ "is_placeholder": False
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to get script: {str(e)}")
+
+
+@router.post("/models/{model_id}/script", summary="Save model script")
+async def save_model_script(
+ model_id: str,
+ script_data: dict,
+ simulation_service = Depends(get_simulation_service)
+):
+ """
+ Save the modified script for a specific model.
+
+ - **model_id**: ID of the simulation model
+ - **script_data**: Dictionary containing the script content
+
+ Returns confirmation of save.
+ """
+ try:
+ model_info = simulation_service.get_model_info(model_id)
+ script_path = model_info.get("script_path")
+
+ # If no script path exists, create one in external_models directory
+ if not script_path:
+ script_path = f"external_models/{model_id}.py"
+ # Ensure the directory exists
+ Path("external_models").mkdir(exist_ok=True)
+
+ script_content = script_data.get("script")
+ if not script_content:
+ raise HTTPException(status_code=400, detail="Script content is required")
+
+ # Write the script to file
+ with open(script_path, 'w') as f:
+ f.write(script_content)
+
+ # Update the model info with the new script path if it wasn't set before
+ if not model_info.get("script_path"):
+ # This would require updating the database, but for now we'll just return success
+ # In a full implementation, you'd update the model metadata in the database
+ pass
+
+ return {
+ "status": "success",
+ "model_id": model_id,
+ "message": f"Script saved successfully for model {model_id}",
+ "script_path": script_path
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to save script: {str(e)}")
diff --git a/modules/research-framework/simexr_mod/app.py b/modules/research-framework/simexr_mod/app.py
new file mode 100644
index 0000000..b820fce
--- /dev/null
+++ b/modules/research-framework/simexr_mod/app.py
@@ -0,0 +1,1071 @@
+import json
+import requests
+import pandas as pd
+from typing import Dict, List, Any, Optional
+from pathlib import Path
+import streamlit as st
+from streamlit_chat import message
+import time
+
+# API Configuration
+API_BASE_URL = "http://127.0.0.1:8000"
+
+# ─────────────────────────────────────────────────────────────────────
+# API Helper Functions
+# ─────────────────────────────────────────────────────────────────────
+
+def make_api_request(method: str, endpoint: str, data: Dict = None, params: Dict = None) -> Dict:
+ """Make an API request and return the response."""
+ url = f"{API_BASE_URL}{endpoint}"
+ headers = {"Content-Type": "application/json"}
+
+ try:
+ if method.upper() == "GET":
+ response = requests.get(url, headers=headers, params=params)
+ elif method.upper() == "POST":
+ if params:
+ # Use query parameters for POST requests that expect them
+ response = requests.post(url, headers=headers, params=params)
+ else:
+ # Use JSON body for POST requests that expect JSON
+ response = requests.post(url, headers=headers, json=data)
+ elif method.upper() == "DELETE":
+ response = requests.delete(url, headers=headers)
+ else:
+ raise ValueError(f"Unsupported method: {method}")
+
+ response.raise_for_status()
+ return response.json()
+
+ except requests.exceptions.RequestException as e:
+ st.error(f"API request failed: {e}")
+ return {"error": str(e)}
+
+def check_api_health() -> bool:
+ """Check if the API server is running."""
+ try:
+ result = make_api_request("GET", "/health/status")
+ return "error" not in result
+ except:
+ return False
+
+def search_models(query: str, limit: int = 10) -> List[Dict]:
+ """Search for models using the fuzzy search API with caching."""
+ cache_key = f"{query}_{limit}"
+
+ # Check cache first
+ if cache_key in st.session_state.cached_search_results:
+ return st.session_state.cached_search_results[cache_key]
+
+ # Fetch from API
+ result = make_api_request("GET", f"/simulation/models/search?name={query}&limit={limit}")
+ models = result.get("models", []) if "error" not in result else []
+
+ # Cache the results
+ st.session_state.cached_search_results[cache_key] = models
+ return models
+
+def get_model_info(model_id: str) -> Dict:
+ """Get detailed information about a model with caching."""
+ # Check cache first
+ if model_id in st.session_state.cached_model_info:
+ return st.session_state.cached_model_info[model_id]
+
+ # Fetch from API
+ result = make_api_request("GET", f"/simulation/models/{model_id}")
+ model_info = result.get("model", {}) if "error" not in result else {}
+
+ # Cache the results
+ if model_info:
+ st.session_state.cached_model_info[model_id] = model_info
+
+ return model_info
+
+def get_model_results(model_id: str, limit: int = 100) -> Dict:
+ """Get simulation results for a model with caching."""
+ cache_key = f"{model_id}_{limit}"
+
+ # Check cache first
+ if cache_key in st.session_state.cached_model_results:
+ return st.session_state.cached_model_results[cache_key]
+
+ # Fetch from API
+ result = make_api_request("GET", f"/simulation/models/{model_id}/results?limit={limit}")
+
+ # Cache the results
+ if "error" not in result:
+ st.session_state.cached_model_results[cache_key] = result
+
+ return result if "error" not in result else {}
+
+def ask_reasoning_question(model_id: str, question: str, max_steps: int = 5) -> Dict:
+ """Ask a question to the reasoning agent."""
+ data = {
+ "model_id": model_id,
+ "question": question,
+ "max_steps": max_steps
+ }
+ result = make_api_request("POST", "/reasoning/ask", data)
+ return result if "error" not in result else {}
+
+def get_reasoning_history(model_id: str, limit: int = 10) -> List[Dict]:
+ """Get reasoning conversation history for a model."""
+ result = make_api_request("GET", f"/reasoning/history/{model_id}?limit={limit}")
+ return result.get("conversations", []) if "error" not in result else []
+
+def get_model_script(model_id: str) -> str:
+ """Get the refactored script for a model."""
+ result = make_api_request("GET", f"/simulation/models/{model_id}/script")
+ return result.get("script", "") if "error" not in result else ""
+
+def save_model_script(model_id: str, script: str) -> Dict:
+ """Save the modified script for a model."""
+ data = {"script": script}
+ result = make_api_request("POST", f"/simulation/models/{model_id}/script", data)
+ return result if "error" not in result else {}
+
+def clear_model_cache(model_id: str = None):
+ """Clear cache for a specific model or all models."""
+ if model_id:
+ # Clear specific model cache
+ if model_id in st.session_state.cached_model_info:
+ del st.session_state.cached_model_info[model_id]
+
+ # Clear all cached results for this model
+ keys_to_remove = [key for key in st.session_state.cached_model_results.keys() if key.startswith(model_id)]
+ for key in keys_to_remove:
+ del st.session_state.cached_model_results[key]
+
+ # Clear model code cache
+ if model_id in st.session_state.cached_model_code:
+ del st.session_state.cached_model_code[model_id]
+ else:
+ # Clear all cache
+ st.session_state.cached_model_info.clear()
+ st.session_state.cached_model_results.clear()
+ st.session_state.cached_model_code.clear()
+ st.session_state.cached_search_results.clear()
+
+def refresh_model_data(model_id: str):
+ """Force refresh of model data by clearing cache and re-fetching."""
+ clear_model_cache(model_id)
+ return get_model_info(model_id)
+
+# ─────────────────────────────────────────────────────────────────────
+# Streamlit Configuration
+# ─────────────────────────────────────────────────────────────────────
+
+st.set_page_config(
+ page_title="SimExR - Simulation Explorer",
+ page_icon="🔬",
+ layout="wide",
+ initial_sidebar_state="expanded"
+)
+
+# ─────────────────────────────────────────────────────────────────────
+# Session State Management
+# ─────────────────────────────────────────────────────────────────────
+
+# Initialize session state for caching
+if "selected_model_id" not in st.session_state:
+ st.session_state.selected_model_id = None
+if "simulation_results" not in st.session_state:
+ st.session_state.simulation_results = None
+if "current_question" not in st.session_state:
+ st.session_state.current_question = ""
+
+# Cache for model data and results
+if "cached_model_info" not in st.session_state:
+ st.session_state.cached_model_info = {}
+if "cached_model_results" not in st.session_state:
+ st.session_state.cached_model_results = {}
+if "cached_model_code" not in st.session_state:
+ st.session_state.cached_model_code = {}
+if "cached_search_results" not in st.session_state:
+ st.session_state.cached_search_results = {}
+
+# Parameter annotations state
+if "parameter_changes" not in st.session_state:
+ st.session_state.parameter_changes = {}
+if "original_parameters" not in st.session_state:
+ st.session_state.original_parameters = {}
+if "current_script" not in st.session_state:
+ st.session_state.current_script = ""
+
+# ─────────────────────────────────────────────────────────────────────
+# Main App
+# ─────────────────────────────────────────────────────────────────────
+
+# Check API health
+if not check_api_health():
+ st.error("🚨 API server is not running! Please start the server with:")
+ st.code("python start_api.py --host 127.0.0.1 --port 8000")
+ st.stop()
+
+# Sidebar
+st.sidebar.title("🔬 SimExR")
+st.sidebar.markdown("Simulation Execution & Reasoning Framework")
+
+# Navigation
+page = st.sidebar.radio(
+ "Navigation",
+ ["🏠 Dashboard", "📥 Import Models", "⚙️ Run Simulations", "📊 View Results", "🤖 AI Analysis", "📝 Parameter Annotations", "🔍 Model Search"]
+)
+
+# ─────────────────────────────────────────────────────────────────────
+# Dashboard Page
+# ─────────────────────────────────────────────────────────────────────
+
+if page == "🏠 Dashboard":
+ st.title("🏠 SimExR Dashboard")
+ st.markdown("Welcome to the Simulation Execution & Reasoning Framework!")
+
+ # System Status
+ col1, col2, col3 = st.columns(3)
+
+ with col1:
+ st.metric("API Status", "🟢 Online")
+
+ with col2:
+ st.metric("API Status", "🟢 Online")
+
+ with col3:
+ st.metric("Framework", "SimExR v1.0")
+
+ # Quick Actions
+ st.subheader("🚀 Quick Actions")
+
+ col1, col2, col3 = st.columns(3)
+
+ with col1:
+ if st.button("📥 Import New Model", use_container_width=True):
+ st.info("Navigate to 'Import Models' in the sidebar")
+
+ with col2:
+ if st.button("🔍 Search Models", use_container_width=True):
+ st.info("Navigate to 'Model Search' in the sidebar")
+
+ with col3:
+ if st.button("🤖 AI Analysis", use_container_width=True):
+ st.info("Navigate to 'AI Analysis' in the sidebar")
+
+ # Cache Management
+ st.subheader("🗄️ Cache Management")
+
+ col1, col2, col3, col4 = st.columns(4)
+
+ with col1:
+ st.metric("Cached Models", len(st.session_state.cached_model_info))
+
+ with col2:
+ st.metric("Cached Results", len(st.session_state.cached_model_results))
+
+ with col3:
+ st.metric("Cached Code", len(st.session_state.cached_model_code))
+
+ with col4:
+ st.metric("Search Cache", len(st.session_state.cached_search_results))
+
+ # Cache controls
+ col1, col2 = st.columns(2)
+
+ with col1:
+ if st.button("🔄 Refresh All Data", use_container_width=True):
+ clear_model_cache()
+ st.success("✅ Cache cleared! Data will be refreshed on next access.")
+ st.rerun()
+
+ with col2:
+ if st.button("📊 Show Cache Details", use_container_width=True):
+ with st.expander("Cache Details"):
+ st.write("**Cached Models:**", list(st.session_state.cached_model_info.keys()))
+ st.write("**Cached Results:**", list(st.session_state.cached_model_results.keys()))
+ st.write("**Cached Code:**", list(st.session_state.cached_model_code.keys()))
+ st.write("**Search Cache:**", list(st.session_state.cached_search_results.keys()))
+
+ # Recent Activity
+ st.subheader("📈 Recent Activity")
+
+ # Get recent reasoning conversations
+ conversations = make_api_request("GET", "/reasoning/conversations?limit=5")
+ if "error" not in conversations and conversations.get("conversations"):
+ for conv in conversations["conversations"][:3]:
+ with st.expander(f"💬 {conv.get('question', 'Question')[:50]}..."):
+ st.write(f"**Model:** {conv.get('model_id', 'Unknown')}")
+ st.write(f"**Time:** {conv.get('timestamp', 'Unknown')}")
+ st.write(f"**Answer:** {conv.get('answer', 'No answer')[:200]}...")
+ else:
+ st.info("No recent conversations found.")
+
+# ─────────────────────────────────────────────────────────────────────
+# Import Models Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "📥 Import Models":
+ st.title("📥 Import Models")
+
+ # Import from GitHub
+ st.header("Import from GitHub")
+
+ github_url = st.text_input(
+ "GitHub URL",
+ placeholder="https://github.com/user/repo/blob/main/script.py",
+ help="Paste the GitHub URL of the script you want to import"
+ )
+
+ model_name = st.text_input(
+ "Model Name",
+ placeholder="my_custom_model",
+ help="Give your model a descriptive name"
+ )
+
+ max_smoke_iters = st.slider(
+ "Max Smoke Test Iterations",
+ min_value=1,
+ max_value=10,
+ value=3,
+ help="Number of iterations to test and fix the imported code"
+ )
+
+ if st.button("🚀 Import & Transform", type="primary"):
+ if not github_url or not model_name:
+ st.error("Please provide both GitHub URL and model name.")
+ else:
+ with st.spinner("Importing and transforming script..."):
+ params = {
+ "github_url": github_url,
+ "model_name": model_name,
+ "max_smoke_iters": max_smoke_iters
+ }
+
+ result = make_api_request("POST", "/simulation/transform/github", params=params)
+
+ if "error" not in result:
+ model_id = result.get('model_id')
+ st.success(f"✅ Successfully imported model: {model_id}")
+
+ # Clear cache for new model to ensure fresh data
+ clear_model_cache(model_id)
+
+ # Show model details
+ with st.expander("📋 Model Details"):
+ st.json(result)
+
+ # Set as selected model
+ st.session_state.selected_model_id = model_id
+ st.info(f"Model '{model_name}' is now ready for simulation!")
+ else:
+ st.error(f"❌ Import failed: {result.get('error')}")
+
+# ─────────────────────────────────────────────────────────────────────
+# Run Simulations Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "⚙️ Run Simulations":
+ st.title("⚙️ Run Simulations")
+
+ # Model Selection
+ st.header("1. Select Model")
+
+ # Search for models
+ search_query = st.text_input("Search models", placeholder="Enter model name...")
+
+ if search_query:
+ models = search_models(search_query, limit=10)
+ if models:
+ model_options = {f"{m['name']} ({m['id']})": m['id'] for m in models}
+ selected_model = st.selectbox("Choose a model", list(model_options.keys()))
+
+ if selected_model:
+ model_id = model_options[selected_model]
+ st.session_state.selected_model_id = model_id
+
+ # Show model info
+ model_info = get_model_info(model_id)
+ if model_info:
+ with st.expander("📋 Model Information"):
+ st.json(model_info)
+ else:
+ st.warning("No models found matching your search.")
+
+ # Simulation Parameters
+ if st.session_state.selected_model_id:
+ st.header("2. Simulation Parameters")
+
+ # Simple parameter input for now
+ st.info("Enter simulation parameters as JSON:")
+
+ params_json = st.text_area(
+ "Parameters (JSON format)",
+ value='{\n "mu": 1.5,\n "z0": [1.5, 0.5],\n "eval_time": 25,\n "t_iteration": 250,\n "plot": false\n}',
+ height=200
+ )
+
+ # Simulation Type
+ sim_type = st.radio("Simulation Type", ["Single Run", "Batch Run"])
+
+ if sim_type == "Single Run":
+ if st.button("▶️ Run Single Simulation", type="primary"):
+ try:
+ params = json.loads(params_json)
+
+ with st.spinner("Running simulation..."):
+ data = {
+ "model_id": st.session_state.selected_model_id,
+ "parameters": params
+ }
+
+ result = make_api_request("POST", "/simulation/run", data)
+
+ if "error" not in result:
+ st.success("✅ Simulation completed successfully!")
+
+ # Clear cache for this model to ensure fresh results
+ clear_model_cache(st.session_state.selected_model_id)
+
+ # Show results
+ with st.expander("📊 Simulation Results"):
+ st.json(result)
+
+ # Store results
+ st.session_state.simulation_results = result
+ else:
+ st.error(f"❌ Simulation failed: {result.get('error')}")
+
+ except json.JSONDecodeError:
+ st.error("❌ Invalid JSON format")
+
+ else: # Batch Run
+ st.info("For batch runs, provide multiple parameter sets:")
+
+ batch_params_json = st.text_area(
+ "Batch Parameters (JSON array)",
+ value='[\n {\n "mu": 1.0,\n "z0": [2, 0],\n "eval_time": 30,\n "t_iteration": 300,\n "plot": false\n },\n {\n "mu": 1.5,\n "z0": [1.5, 0.5],\n "eval_time": 25,\n "t_iteration": 250,\n "plot": false\n }\n]',
+ height=200
+ )
+
+ if st.button("▶️ Run Batch Simulation", type="primary"):
+ try:
+ param_grid = json.loads(batch_params_json)
+
+ with st.spinner("Running batch simulation..."):
+ data = {
+ "model_id": st.session_state.selected_model_id,
+ "parameter_grid": param_grid
+ }
+
+ result = make_api_request("POST", "/simulation/batch", data)
+
+ if "error" not in result:
+ st.success(f"✅ Batch simulation completed! {len(result.get('results', []))} simulations run.")
+
+ # Clear cache for this model to ensure fresh results
+ clear_model_cache(st.session_state.selected_model_id)
+
+ # Show results summary
+ with st.expander("📊 Batch Results Summary"):
+ st.json(result)
+
+ # Store results
+ st.session_state.simulation_results = result
+ else:
+ st.error(f"❌ Batch simulation failed: {result.get('error')}")
+
+ except json.JSONDecodeError:
+ st.error("❌ Invalid JSON format")
+
+# ─────────────────────────────────────────────────────────────────────
+# View Results Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "📊 View Results":
+ st.title("📊 View Results")
+
+ # Model Selection
+ st.header("Select Model")
+
+ search_query = st.text_input("Search models for results", placeholder="Enter model name...")
+
+ if search_query:
+ models = search_models(search_query, limit=10)
+ if models:
+ model_options = {f"{m['name']} ({m['id']})": m['id'] for m in models}
+ selected_model = st.selectbox("Choose a model", list(model_options.keys()))
+
+ if selected_model:
+ model_id = model_options[selected_model]
+
+ # Get results
+ results = get_model_results(model_id, limit=100)
+
+ if results and results.get("results"):
+ st.success(f"📊 Found {results.get('total_count', 0)} results")
+
+ # Display results
+ st.subheader("Simulation Results")
+
+ # Convert to DataFrame for better display
+ df = pd.DataFrame(results["results"])
+
+ # Show basic stats
+ col1, col2, col3 = st.columns(3)
+ with col1:
+ st.metric("Total Results", len(df))
+ with col2:
+ st.metric("Success Rate", f"{len(df[df.get('success', False)])/len(df)*100:.1f}%")
+ with col3:
+ if 'execution_time' in df.columns:
+ avg_time = df['execution_time'].mean()
+ st.metric("Avg Execution Time", f"{avg_time:.3f}s")
+
+ # Show results table
+ st.dataframe(df, use_container_width=True)
+
+ # Download option
+ csv = df.to_csv(index=False)
+ st.download_button(
+ label="📥 Download Results as CSV",
+ data=csv,
+ file_name=f"{model_id}_results.csv",
+ mime="text/csv"
+ )
+
+ # Store for analysis
+ st.session_state.simulation_results = results
+ st.session_state.selected_model_id = model_id
+
+ else:
+ st.warning("No results found for this model.")
+ else:
+ st.warning("No models found matching your search.")
+
+# ─────────────────────────────────────────────────────────────────────
+# AI Analysis Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "🤖 AI Analysis":
+ st.title("🤖 AI Analysis")
+
+ # Model Selection
+ if not st.session_state.selected_model_id:
+ st.header("1. Select Model")
+
+ search_query = st.text_input("Search models for analysis", placeholder="Enter model name...")
+
+ if search_query:
+ models = search_models(search_query, limit=10)
+ if models:
+ model_options = {f"{m['name']} ({m['id']})": m['id'] for m in models}
+ selected_model = st.selectbox("Choose a model", list(model_options.keys()))
+
+ if selected_model:
+ st.session_state.selected_model_id = model_options[selected_model]
+ st.success(f"✅ Selected model: {selected_model}")
+ else:
+ st.warning("No models found matching your search.")
+
+ # AI Analysis
+ if st.session_state.selected_model_id:
+ st.header("2. AI Chatbot")
+
+ # Show selected model
+ st.info(f"📋 Chatting about model: {st.session_state.selected_model_id}")
+
+ # Chat settings at the top
+ st.subheader("⚙️ Chat Settings")
+ col1, col2 = st.columns([1, 3])
+
+ with col1:
+ max_steps = st.number_input(
+ "Max Reasoning Steps",
+ min_value=1,
+ max_value=20,
+ value=5,
+ help="Maximum reasoning steps for complex questions (1-20)"
+ )
+
+ with col2:
+ st.markdown("💡 **Tip**: Higher step values allow for more complex reasoning but may take longer to respond.")
+
+ # Initialize chat history for this model if not exists
+ model_chat_key = f"chat_history_{st.session_state.selected_model_id}"
+ if model_chat_key not in st.session_state:
+ st.session_state[model_chat_key] = []
+
+ # Display chat messages
+ st.subheader("💬 Conversation")
+
+ # Show welcome message if no chat history
+ if not st.session_state[model_chat_key]:
+ st.markdown("""
+ <div style="background-color: #f0f2f6; padding: 20px; border-radius: 10px; border-left: 4px solid #1f77b4;">
+ <h4>🤖 Welcome to AI Analysis!</h4>
+ <p>I'm your AI assistant for analyzing simulation results. I can help you understand your model's behavior, interpret results, and answer questions about your simulations.</p>
+
+ <h5>💡 What you can ask me:</h5>
+ <ul>
+ <li>📊 Analyze simulation results and trends</li>
+ <li>🔍 Explain parameter effects on system behavior</li>
+ <li>📈 Identify patterns and anomalies in the data</li>
+ <li>🧮 Help with mathematical interpretations</li>
+ <li>💡 Suggest improvements or optimizations</li>
+ </ul>
+
+ <p><strong>Start by typing your question below! 👇</strong></p>
+ </div>
+ """, unsafe_allow_html=True)
+
+ # Display existing chat messages
+ for i, chat in enumerate(st.session_state[model_chat_key]):
+ if chat["role"] == "user":
+ message(chat["content"], is_user=True, key=f"user_{i}")
+ elif chat.get("is_thinking"):
+ # Show thinking indicator
+ with st.container():
+ st.markdown("🤔 **AI Assistant**: Thinking...")
+ st.progress(0) # Show progress bar
+ else:
+ message(chat["content"], is_user=False, key=f"assistant_{i}")
+
+ # Chat input area
+ st.markdown("---")
+ st.markdown("### 💭 Ask a Question")
+
+ # Create a form for the chat input
+ with st.form(key="chat_form", clear_on_submit=True):
+ col1, col2 = st.columns([4, 1])
+
+ with col1:
+ user_input = st.text_area(
+ "Type your message...",
+ placeholder="Ask me anything about your simulation results...",
+ height=60,
+ key="user_input",
+ label_visibility="collapsed"
+ )
+
+ with col2:
+ st.markdown("<br>", unsafe_allow_html=True) # Add some spacing
+ submit_button = st.form_submit_button("🚀 Send", type="primary", use_container_width=True)
+
+ # Handle chat submission
+ if submit_button and user_input.strip():
+ # Add user message to chat immediately
+ st.session_state[model_chat_key].append({
+ "role": "user",
+ "content": user_input,
+ "timestamp": time.strftime("%Y-%m-%d %H:%M:%S")
+ })
+
+ # Add a temporary "thinking" message
+ thinking_message = {
+ "role": "assistant",
+ "content": "🤔 Thinking...",
+ "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+ "is_thinking": True
+ }
+ st.session_state[model_chat_key].append(thinking_message)
+
+ # Force rerun to show user message immediately
+ st.rerun()
+
+ # Check if we need to process a thinking message
+ if st.session_state[model_chat_key] and st.session_state[model_chat_key][-1].get("is_thinking"):
+ # Remove the thinking message
+ st.session_state[model_chat_key].pop()
+
+ # Get the last user message
+ last_user_message = st.session_state[model_chat_key][-1]["content"]
+
+ # Show loading indicator
+ with st.spinner("🤖 AI is analyzing your question..."):
+ result = ask_reasoning_question(
+ st.session_state.selected_model_id,
+ last_user_message,
+ max_steps
+ )
+
+ if "error" not in result:
+ ai_response = result.get("answer", "I apologize, but I couldn't generate a response for your question.")
+
+ # Add AI response to chat
+ st.session_state[model_chat_key].append({
+ "role": "assistant",
+ "content": ai_response,
+ "timestamp": time.strftime("%Y-%m-%d %H:%M:%S")
+ })
+
+ st.success("✅ Response generated!")
+ st.rerun() # Refresh to show new messages
+ else:
+ # Add error message to chat
+ st.session_state[model_chat_key].append({
+ "role": "assistant",
+ "content": f"❌ Sorry, I encountered an error: {result.get('error')}",
+ "timestamp": time.strftime("%Y-%m-%d %H:%M:%S")
+ })
+ st.error(f"❌ AI analysis failed: {result.get('error')}")
+ st.rerun()
+
+ # Chat controls
+ st.subheader("⚙️ Chat Controls")
+ col1, col2, col3 = st.columns(3)
+
+ with col1:
+ if st.button("🗑️ Clear Chat History"):
+ st.session_state[model_chat_key] = []
+ st.rerun()
+
+ with col2:
+ if st.button("📥 Export Chat"):
+ if st.session_state[model_chat_key]:
+ chat_text = "\n\n".join([
+ f"**{chat['role'].title()}** ({chat['timestamp']}):\n{chat['content']}"
+ for chat in st.session_state[model_chat_key]
+ ])
+ st.download_button(
+ label="📄 Download Chat",
+ data=chat_text,
+ file_name=f"ai_chat_{st.session_state.selected_model_id}_{time.strftime('%Y%m%d_%H%M%S')}.txt",
+ mime="text/plain"
+ )
+
+ with col3:
+ if st.button("🔄 New Conversation"):
+ st.session_state[model_chat_key] = []
+ st.rerun()
+
+# ─────────────────────────────────────────────────────────────────────
+# Parameter Annotations Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "📝 Parameter Annotations":
+ st.title("📝 Parameter Annotations & Script Management")
+
+ # Model Selection
+ st.header("1. Select Model")
+
+ search_query = st.text_input("Search models", placeholder="Enter model name...")
+
+ if search_query:
+ models = search_models(search_query, limit=10)
+ if models:
+ model_options = {f"{m['name']} ({m['id']})": m['id'] for m in models}
+ selected_model = st.selectbox("Choose a model", list(model_options.keys()))
+
+ if selected_model:
+ model_id = model_options[selected_model]
+ st.session_state.selected_model_id = model_id
+
+ # Get model info
+ model_info = get_model_info(model_id)
+
+ if model_info:
+ st.success(f"✅ Selected model: {model_info.get('name', model_id)}")
+
+ # Display model metadata
+ with st.expander("📋 Model Information"):
+ st.json(model_info)
+
+ # Extract parameters from model info
+ extracted_params = model_info.get('parameters', {})
+
+ if extracted_params:
+ st.header("2. Extracted Parameters")
+
+ # Store original parameters if not already stored
+ if model_id not in st.session_state.original_parameters:
+ st.session_state.original_parameters[model_id] = extracted_params.copy()
+ st.session_state.parameter_changes[model_id] = {}
+
+ # Display parameters with change tracking
+ st.subheader("📊 Parameter Visualization")
+
+ # Create parameter editing interface
+ col1, col2 = st.columns([2, 1])
+
+ with col1:
+ st.markdown("### 🔧 Edit Parameters")
+
+ # Parameter editing form
+ with st.form("parameter_form"):
+ updated_params = {}
+
+ for param_name, param_info in extracted_params.items():
+ param_type = param_info.get('type', 'string')
+ param_description = param_info.get('description', '')
+ param_default = param_info.get('default', '')
+
+ st.markdown(f"**{param_name}** ({param_type})")
+ if param_description:
+ st.caption(f"Description: {param_description}")
+
+ # Create appropriate input based on type
+ if param_type == 'number':
+ value = st.number_input(
+ f"Value for {param_name}",
+ value=float(param_default) if param_default else 0.0,
+ key=f"param_{model_id}_{param_name}"
+ )
+ elif param_type == 'boolean':
+ value = st.checkbox(
+ f"Value for {param_name}",
+ value=bool(param_default) if param_default else False,
+ key=f"param_{model_id}_{param_name}"
+ )
+ elif param_type == 'array':
+ # For arrays, provide a text input that can be parsed as JSON
+ value_str = st.text_input(
+ f"Value for {param_name} (JSON array)",
+ value=json.dumps(param_default) if param_default else "[]",
+ key=f"param_{model_id}_{param_name}"
+ )
+ try:
+ value = json.loads(value_str)
+ except json.JSONDecodeError:
+ value = param_default
+ else:
+ value = st.text_input(
+ f"Value for {param_name}",
+ value=str(param_default) if param_default else "",
+ key=f"param_{model_id}_{param_name}"
+ )
+
+ updated_params[param_name] = value
+
+ # Track changes
+ original_value = st.session_state.original_parameters[model_id].get(param_name, {}).get('default', '')
+ if value != original_value:
+ st.session_state.parameter_changes[model_id][param_name] = {
+ 'original': original_value,
+ 'current': value,
+ 'changed': True
+ }
+ else:
+ if param_name in st.session_state.parameter_changes[model_id]:
+ del st.session_state.parameter_changes[model_id][param_name]
+
+ submit_params = st.form_submit_button("💾 Save Parameter Changes")
+
+ with col2:
+ st.markdown("### 📈 Change Tracking")
+
+ # Show parameter changes
+ changes = st.session_state.parameter_changes.get(model_id, {})
+
+ if changes:
+ st.success(f"🔄 {len(changes)} parameters modified")
+
+ for param_name, change_info in changes.items():
+ with st.expander(f"📝 {param_name}"):
+ st.write(f"**Original:** {change_info['original']}")
+ st.write(f"**Current:** {change_info['current']}")
+ st.write(f"**Status:** Modified")
+ else:
+ st.info("✅ No parameter changes detected")
+
+ # Quick actions
+ st.markdown("### ⚡ Quick Actions")
+
+ if st.button("🔄 Reset All Parameters"):
+ st.session_state.parameter_changes[model_id] = {}
+ st.rerun()
+
+ if st.button("📊 Export Parameters"):
+ param_data = {
+ "model_id": model_id,
+ "parameters": updated_params,
+ "changes": changes
+ }
+ st.download_button(
+ label="📄 Download Parameters",
+ data=json.dumps(param_data, indent=2),
+ file_name=f"{model_id}_parameters.json",
+ mime="application/json"
+ )
+
+ # Script Management
+ st.header("3. Script Management")
+
+ # Get current script
+ script_result = make_api_request("GET", f"/simulation/models/{model_id}/script")
+
+ if "error" not in script_result:
+ current_script = script_result.get("script", "")
+ is_placeholder = script_result.get("is_placeholder", False)
+
+ if is_placeholder:
+ st.subheader("📝 Script Editor (Placeholder)")
+ st.warning("⚠️ This model doesn't have a script file yet. You can create one by editing the placeholder below.")
+ else:
+ st.subheader("📝 Refactored Script")
+
+ # Script editing
+ edited_script = st.text_area(
+ "Edit the script:",
+ value=current_script,
+ height=400,
+ key=f"script_{model_id}"
+ )
+
+ col1, col2 = st.columns(2)
+
+ with col1:
+ if st.button("💾 Save Script Changes"):
+ result = save_model_script(model_id, edited_script)
+ if "error" not in result:
+ st.success("✅ Script saved successfully!")
+ else:
+ st.error(f"❌ Failed to save script: {result.get('error')}")
+
+ with col2:
+ if st.button("🔄 Reset Script"):
+ st.rerun()
+
+ # Script preview
+ with st.expander("👀 Script Preview"):
+ st.code(edited_script, language="python")
+
+ # Simulation with updated parameters
+ st.header("4. Quick Simulation")
+
+ if st.button("🚀 Run Simulation with Current Parameters"):
+ # Get the updated parameters from the form
+ updated_params = {}
+ for param_name in extracted_params.keys():
+ param_key = f"param_{model_id}_{param_name}"
+ if param_key in st.session_state:
+ updated_params[param_name] = st.session_state[param_key]
+
+ if updated_params:
+ with st.spinner("Running simulation with updated parameters..."):
+ data = {
+ "model_id": model_id,
+ "parameters": updated_params
+ }
+
+ result = make_api_request("POST", "/simulation/run", data)
+
+ if "error" not in result:
+ st.success("✅ Simulation completed successfully!")
+
+ with st.expander("📊 Simulation Results"):
+ st.json(result)
+
+ # Store results for other pages
+ st.session_state.simulation_results = result
+ else:
+ st.error(f"❌ Simulation failed: {result.get('error')}")
+ else:
+ st.warning("⚠️ No parameters available for simulation")
+
+ else:
+ st.error("❌ Failed to load model information")
+ else:
+ st.info("Please select a model to continue")
+ else:
+ st.warning("No models found matching your search.")
+ else:
+ st.info("🔍 Enter a model name to search and get started with parameter annotations")
+
+# ─────────────────────────────────────────────────────────────────────
+# Model Search Page
+# ─────────────────────────────────────────────────────────────────────
+
+elif page == "🔍 Model Search":
+ st.title("🔍 Model Search")
+
+ # Search interface
+ search_query = st.text_input(
+ "Search models by name",
+ placeholder="e.g., vanderpol, lorenz, pendulum...",
+ help="Enter part of a model name to search"
+ )
+
+ limit = st.slider("Max results", 5, 50, 20)
+
+ if search_query:
+ with st.spinner("Searching models..."):
+ models = search_models(search_query, limit)
+
+ if models:
+ st.success(f"🔍 Found {len(models)} models")
+
+ # Display models
+ for i, model in enumerate(models):
+ with st.expander(f"📋 {model.get('name', 'Unknown')} ({model.get('id', 'No ID')})"):
+ col1, col2 = st.columns([2, 1])
+
+ with col1:
+ st.write(f"**ID:** {model.get('id', 'N/A')}")
+ st.write(f"**Name:** {model.get('name', 'N/A')}")
+ st.write(f"**Script Path:** {model.get('script_path', 'N/A')}")
+
+ # Show metadata if available
+ if model.get('metadata'):
+ try:
+ metadata = json.loads(model['metadata']) if isinstance(model['metadata'], str) else model['metadata']
+ st.write("**Metadata:**")
+ st.json(metadata)
+ except:
+ st.write(f"**Metadata:** {model['metadata']}")
+
+ with col2:
+ if st.button(f"Select Model {i+1}", key=f"select_{i}"):
+ st.session_state.selected_model_id = model.get('id')
+ st.success(f"✅ Selected: {model.get('name')}")
+
+ if st.button(f"View Results {i+1}", key=f"results_{i}"):
+ st.session_state.selected_model_id = model.get('id')
+ st.info("Navigate to 'View Results' in the sidebar to see the results")
+ else:
+ st.warning("No models found matching your search.")
+
+ # Show all models
+ else:
+ st.subheader("All Available Models")
+
+ # Get all models
+ all_models = make_api_request("GET", "/simulation/models")
+
+ if "error" not in all_models and all_models.get("models"):
+ models = all_models["models"]
+ st.info(f"📊 Total models: {len(models)}")
+
+ # Display in a table
+ model_data = []
+ for model in models[:50]: # Limit to first 50
+ model_data.append({
+ "Name": model.get('name', 'N/A'),
+ "ID": model.get('id', 'N/A')[:20] + "..." if len(model.get('id', '')) > 20 else model.get('id', 'N/A'),
+ "Script Path": model.get('script_path', 'N/A')
+ })
+
+ df = pd.DataFrame(model_data)
+ st.dataframe(df, use_container_width=True)
+ else:
+ st.error("Failed to load models.")
+
+# ─────────────────────────────────────────────────────────────────────
+# Footer
+# ─────────────────────────────────────────────────────────────────────
+
+st.sidebar.markdown("---")
+st.sidebar.markdown("**API Status:** 🟢 Online")
+st.sidebar.markdown(f"**Server:** {API_BASE_URL}")
+
+# Debug info
+if st.sidebar.checkbox("Show Debug Info"):
+ # Count total chat messages across all models
+ total_chat_messages = 0
+ for key in st.session_state.keys():
+ if key.startswith("chat_history_"):
+ total_chat_messages += len(st.session_state[key])
+
+ st.sidebar.json({
+ "selected_model": st.session_state.selected_model_id,
+ "total_chat_messages": total_chat_messages,
+ "has_results": st.session_state.simulation_results is not None,
+ "cached_models": len(st.session_state.cached_model_info),
+ "cached_results": len(st.session_state.cached_model_results)
+ })
diff --git a/modules/research-framework/simexr_mod/code/__init__.py b/modules/research-framework/simexr_mod/code/__init__.py
new file mode 100644
index 0000000..3fdff84
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/__init__.py
@@ -0,0 +1,8 @@
+"""
+Code processing module for SimExR.
+
+This module handles code refactoring, testing, and utility functions
+for processing simulation scripts.
+"""
+
+__all__ = []
diff --git a/modules/research-framework/simexr_mod/code/extract/llm_extract.py b/modules/research-framework/simexr_mod/code/extract/llm_extract.py
new file mode 100644
index 0000000..3ecdd14
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/extract/llm_extract.py
@@ -0,0 +1,131 @@
+from typing import Any, Dict
+
+from core.parser import tidy_json
+
+from pathlib import Path
+import json, re
+import openai
+import os
+
+def extract_script_settings(
+ script_path: str,
+ llm_model: str = "gpt-5-mini",
+ retries: int = 4
+) -> Dict[str, Any]:
+ """
+ Return a flat settings dict: name -> default (float for numerics/fractions; else original).
+ Uses gpt-5-mini by default. Robust to malformed LLM output.
+ """
+ # Ensure OpenAI API key is configured globally
+ try:
+ from utils.openai_config import ensure_openai_api_key
+ ensure_openai_api_key()
+ print("OpenAI API key configured globally in llm_extract")
+ except Exception as e:
+ print(f"Warning: Could not configure OpenAI API key in llm_extract: {e}")
+
+ code = Path(script_path).read_text()
+
+ system_prompt = r"""
+ You are a precise code-analysis assistant.
+ Given a Python script defining a function simulate(**params), extract:
+
+ 1) All keyword parameters that simulate accepts, with their defaults (as strings) and types.
+ 2) All variables used as initial conditions, with their defaults (as strings or null).
+ 3) All independent variables (e.g. t, time), with their defaults (as strings or null).
+
+ Return ONLY a raw JSON object with this schema:
+
+ {
+ "parameters": {
+ "param1": {"default": "<value or null>", "type": "<type or unknown>"},
+ …
+ },
+ "initial_conditions": {
+ "varA": "<value or null>",
+ …
+ },
+ "independent_variables": {
+ "varX": "<value or null>",
+ …
+ }
+ }
+
+ Use only double-quotes, no markdown or code fences.
+ Make all the keys or values strings in the response json.
+ """
+ user_prompt = f"---BEGIN SCRIPT---\n{code}\n---END SCRIPT---"
+
+ raw_payload = None
+ last_cleaned = None
+
+ for attempt in range(retries):
+ resp = openai.chat.completions.create(
+ model=llm_model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt},
+ ],
+ # temperature=0.0,
+ )
+ out = resp.choices[0].message.content.strip()
+
+ out = re.sub(r"```(?:json)?", "", out, flags=re.IGNORECASE)
+ out = re.sub(r"^\s*json\s*\n", "", out, flags=re.IGNORECASE | re.MULTILINE)
+
+ cleaned = tidy_json(out)
+ last_cleaned = cleaned
+ try:
+ raw_payload = json.loads(cleaned)
+ break
+ except json.JSONDecodeError:
+ if attempt == retries - 1:
+ raise ValueError(f"Failed to parse JSON after {retries} attempts. Last output:\n{cleaned}")
+
+ # ── normalization helpers ──
+ def _as_dict(obj) -> Dict[str, Any]:
+ return obj if isinstance(obj, dict) else {}
+
+ def _convert(val: Any) -> Any:
+ if isinstance(val, (int, float)) or val is None:
+ return val
+ if isinstance(val, str):
+ s = val.strip()
+ m = re.fullmatch(r"(-?\d+(?:\.\d*)?)\s*/\s*(\d+(?:\.\d*)?)", s)
+ if m:
+ num, den = map(float, m.groups())
+ return num / den
+ if re.fullmatch(r"-?\d+", s):
+ try:
+ return int(s)
+ except Exception:
+ pass
+ try:
+ return float(s)
+ except Exception:
+ return val
+ return val
+
+ payload = _as_dict(raw_payload or {})
+ params_obj = _as_dict(payload.get("parameters", {}))
+ inits_obj = _as_dict(payload.get("initial_conditions", {}))
+ indep_obj = _as_dict(payload.get("independent_variables", {}))
+
+ settings: Dict[str, Any] = {}
+
+ for name, info in params_obj.items():
+ default = info.get("default") if isinstance(info, dict) else info
+ settings[name] = _convert(default)
+
+ for name, default in inits_obj.items():
+ if isinstance(default, dict):
+ default = default.get("default")
+ settings[name] = _convert(default)
+
+ for name, default in indep_obj.items():
+ if isinstance(default, dict):
+ default = default.get("default")
+ settings[name] = _convert(default)
+
+ print(f"Settings:\n{json.dumps(settings, indent=2, ensure_ascii=False)}")
+ return settings
diff --git a/modules/research-framework/simexr_mod/code/helpers/ast_helpers.py b/modules/research-framework/simexr_mod/code/helpers/ast_helpers.py
new file mode 100644
index 0000000..96764a8
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/helpers/ast_helpers.py
@@ -0,0 +1,116 @@
+"""
+Helper functions for AST manipulation and source code generation.
+
+This module provides utilities for working with Python's Abstract Syntax Tree
+and generating source code from AST nodes.
+"""
+
+import ast
+import json
+from typing import Dict, Any
+
+
+def build_overrides_assignment(overrides: Dict[str, Any]) -> ast.Assign:
+ """
+ Build an AST assignment node for DEFAULT_OVERRIDES.
+
+ Handles various data types, converting complex types to JSON strings.
+
+ Args:
+ overrides: Dictionary of parameter overrides
+
+ Returns:
+ AST assignment node
+ """
+ keys, values = [], []
+
+ for k, v in overrides.items():
+ keys.append(ast.Constant(value=str(k)))
+
+ # Handle different types of values
+ if isinstance(v, (int, float, str, bool)) or v is None:
+ values.append(ast.Constant(value=v))
+ else:
+ # Convert complex types to JSON strings
+ values.append(ast.Constant(value=json.dumps(v)))
+
+ return ast.Assign(
+ targets=[ast.Name(id="DEFAULT_OVERRIDES", ctx=ast.Store())],
+ value=ast.Dict(keys=keys, values=values),
+ )
+
+
+def generate_source(tree: ast.AST, fallback: str) -> str:
+ """
+ Generate source code from AST with multiple fallback methods.
+
+ Tries multiple approaches to generate source code:
+ 1. ast.unparse (Python 3.9+)
+ 2. astor.to_source (if available)
+ 3. black formatting (if available)
+ 4. Original source (last resort)
+
+ Args:
+ tree: AST to convert to source
+ fallback: Original source to use as fallback
+
+ Returns:
+ Generated source code
+ """
+ new_code = None
+
+ # Try ast.unparse (Python 3.9+)
+ try:
+ new_code = ast.unparse(tree)
+ except Exception:
+ pass
+
+ # Try astor if ast.unparse failed
+ if new_code is None:
+ try:
+ import astor
+ new_code = astor.to_source(tree)
+ except Exception:
+ return fallback # Last resort fallback
+
+ # Optional formatting with black
+ try:
+ import black
+ new_code = black.format_str(new_code, mode=black.Mode())
+ except Exception:
+ pass
+
+ return new_code or fallback
+
+
+def find_function_by_name(tree: ast.Module, name: str) -> ast.FunctionDef:
+ """
+ Find a function in an AST by name.
+
+ Args:
+ tree: AST module node
+ name: Function name to find
+
+ Returns:
+ Function definition node or None if not found
+ """
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return node
+ return None
+
+
+def has_docstring(node: ast.FunctionDef) -> bool:
+ """
+ Check if a function has a docstring.
+
+ Args:
+ node: Function definition node
+
+ Returns:
+ True if the function has a docstring
+ """
+ return (node.body and
+ isinstance(node.body[0], ast.Expr) and
+ isinstance(getattr(node.body[0], "value", None), ast.Constant) and
+ isinstance(node.body[0].value.value, str))
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/code/helpers/locate_helpers.py b/modules/research-framework/simexr_mod/code/helpers/locate_helpers.py
new file mode 100644
index 0000000..cfb03d3
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/helpers/locate_helpers.py
@@ -0,0 +1,44 @@
+from typing import List, Any, Dict
+import re
+
+from core.code.models.param_ref import ParamRef
+
+
+def group_param_refs(refs: List[ParamRef]) -> Dict[str, List[Dict[str, Any]]]:
+ grouped: Dict[str, List[Dict[str, Any]]] = {}
+ for r in refs:
+ grouped.setdefault(r.param, []).append({"line": r.line, "col": r.col, "kind": r.kind})
+ for k in grouped:
+ grouped[k].sort(key=lambda x: (x["line"], x["col"]))
+ return grouped
+
+def _coerce_literal(val: Any) -> Any:
+ """Coerce editor string values to float/int if possible; support simple fractions like '8/3'."""
+ if val is None:
+ return None
+ if isinstance(val, (int, float)):
+ return val
+ s = str(val).strip()
+ if s == "":
+ return ""
+ m = re.fullmatch(r"(-?\d+(?:\.\d*)?)\s*/\s*(\d+(?:\.\d*)?)", s)
+ if m:
+ num, den = map(float, m.groups())
+ return num / den
+ # try int then float
+ try:
+ if re.fullmatch(r"-?\d+", s):
+ return int(s)
+ return float(s)
+ except Exception:
+ return s
+
+
+def _grab_line_context(source_lines: List[str], line: int, col: int, pad: int = 120) -> str:
+ if 1 <= line <= len(source_lines):
+ s = source_lines[line - 1].rstrip("\n")
+ caret = " " * max(col, 0) + "^"
+ if len(s) > pad:
+ s = s[:pad] + " ..."
+ return f"{s}\n{caret}"
+ return ""
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/code/inject/inject_overrides.py b/modules/research-framework/simexr_mod/code/inject/inject_overrides.py
new file mode 100644
index 0000000..0fe5ce0
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/inject/inject_overrides.py
@@ -0,0 +1,137 @@
+"""
+Module for injecting parameter overrides into Python simulation code.
+
+This module uses AST manipulation to cleanly inject parameter overrides
+into simulation code by:
+1. Adding a DEFAULT_OVERRIDES dictionary at the module level
+2. Inserting a params merge statement at the beginning of the simulate function
+"""
+
+import ast
+
+from core.code.helpers.ast_helpers import build_overrides_assignment, generate_source
+
+
+class OverrideInjector:
+ """
+ Handles the AST transformation to inject overrides.
+
+ Separates the transformation logic for better organization.
+ """
+
+ def inject(self, tree: ast.Module, assign_overrides: ast.Assign) -> ast.Module:
+ """
+ Inject overrides into the AST.
+
+ Args:
+ tree: AST to modify
+ assign_overrides: AST node for DEFAULT_OVERRIDES assignment
+
+ Returns:
+ Modified AST
+ """
+ # Inject at module level
+ tree = self._inject_module_overrides(tree, assign_overrides)
+
+ # Inject in simulate function
+ tree = self._inject_function_overrides(tree)
+
+ return tree
+
+ def _inject_module_overrides(self, tree: ast.Module, assign_overrides: ast.Assign) -> ast.Module:
+ """Inject DEFAULT_OVERRIDES at module level."""
+ # Remove any existing DEFAULT_OVERRIDES
+ new_body = []
+ for node in tree.body:
+ if isinstance(node, ast.Assign) and any(
+ isinstance(target, ast.Name) and target.id == "DEFAULT_OVERRIDES"
+ for target in node.targets
+ ):
+ continue # Skip existing DEFAULT_OVERRIDES
+ new_body.append(node)
+
+ # Add new DEFAULT_OVERRIDES at the beginning
+ new_body.insert(0, assign_overrides)
+ tree.body = new_body
+
+ return tree
+
+ def _inject_function_overrides(self, tree: ast.Module) -> ast.Module:
+ """Inject params merge in simulate function."""
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == "simulate":
+ if not self._has_params_merge(node):
+ # Create the merge statement
+ merge_stmt = ast.parse(
+ "params = {**DEFAULT_OVERRIDES, **(params or {})}"
+ ).body[0]
+
+ # Insert after docstring if present
+ insert_idx = self._get_insert_index(node)
+ node.body.insert(insert_idx, merge_stmt)
+
+ return tree
+
+ def _has_params_merge(self, node: ast.FunctionDef) -> bool:
+ """Check if the function already has a params merge statement."""
+ for stmt in node.body[:4]: # Check first few statements
+ if isinstance(stmt, ast.Assign) and any(
+ isinstance(target, ast.Name) and target.id == "params"
+ for target in stmt.targets
+ ):
+ # Look for DEFAULT_OVERRIDES in the statement
+ for name in ast.walk(stmt):
+ if isinstance(name, ast.Name) and name.id == "DEFAULT_OVERRIDES":
+ return True
+ return False
+
+ def _get_insert_index(self, node: ast.FunctionDef) -> int:
+ """Get index to insert after docstring."""
+ # If first statement is a docstring, insert after it
+ if (node.body and
+ isinstance(node.body[0], ast.Expr) and
+ isinstance(getattr(node.body[0], "value", None), ast.Constant) and
+ isinstance(node.body[0].value.value, str)):
+ return 1
+ return 0
+
+
+def inject_overrides_via_ast(source: str, overrides: Dict[str, Any]) -> str:
+ """
+ Inject overrides into Python simulation code.
+
+ Adds:
+ • module-level DEFAULT_OVERRIDES = {...}
+ • First statement inside simulate(**params):
+ params = {**DEFAULT_OVERRIDES, **(params or {})}
+
+ Uses AST transformation for clean code manipulation and falls back
+ gracefully if code generation fails.
+
+ Args:
+ source: Python source code
+ overrides: Dictionary of parameter overrides
+
+ Returns:
+ Modified Python source code with injected overrides
+ """
+ if not overrides:
+ return source
+
+ # Parse source into AST
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ # If source has syntax errors, return original
+ return source
+
+ # Build DEFAULT_OVERRIDES dict AST node
+ assign_overrides = build_overrides_assignment(overrides)
+
+ # Transform the AST to inject overrides
+ transformer = OverrideInjector()
+ new_tree = transformer.inject(tree, assign_overrides)
+ ast.fix_missing_locations(new_tree)
+
+ # Generate new source code with fallbacks
+ return generate_source(new_tree, fallback=source)
diff --git a/modules/research-framework/simexr_mod/code/locate/locate.py b/modules/research-framework/simexr_mod/code/locate/locate.py
new file mode 100644
index 0000000..9325646
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/locate/locate.py
@@ -0,0 +1,27 @@
+import ast
+import re
+from dataclasses import dataclass
+from typing import List, Dict, Any, Iterable
+
+from core.code.locate.param_usage_visitor import _ParamUsageVisitor
+from core.code.models.param_ref import ParamRef
+from core.code.helpers.locate_helpers import _grab_line_context
+
+
+def locate_param_references_from_source(source: str, param_names: Iterable[str]) -> List[ParamRef]:
+ lines = source.splitlines(True)
+ try:
+ tree = ast.parse(source)
+ except Exception:
+ return []
+ visitor = _ParamUsageVisitor(param_names)
+ visitor.visit(tree)
+ refs: List[ParamRef] = []
+ for p, ln, col, kind in visitor.references:
+ ctx = _grab_line_context(lines, ln, col)
+ refs.append(ParamRef(param=p, line=ln, col=col, kind=kind, context=ctx))
+ # stable ordering by param then location
+ refs.sort(key=lambda r: (r.param, r.line, r.col))
+ return refs
+
+
diff --git a/modules/research-framework/simexr_mod/code/locate/param_usage_visitor.py b/modules/research-framework/simexr_mod/code/locate/param_usage_visitor.py
new file mode 100644
index 0000000..fe037ca
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/locate/param_usage_visitor.py
@@ -0,0 +1,70 @@
+import ast
+from typing import Iterable, List, Tuple, Dict, Optional
+
+
+class _ParamUsageVisitor(ast.NodeVisitor):
+ """
+ Finds param usages in simulate(**params)-style scripts:
+ • params["k"], params.get("k", default)
+ • alias: a = params["k"] or params.get("k"); later reads of `a`
+ Notes:
+ - We match direct access on Name('params'); this covers almost all real scripts.
+ - We record alias reads as kind='alias' to distinguish from raw dict access.
+ """
+ def __init__(self, param_names: Iterable[str]):
+ self.param_names = set(param_names)
+ self.references: List[Tuple[str, int, int, str]] = []
+ self._alias_from_params: Dict[str, str] = {}
+
+ # Capture aliasing like: a = params['k'] or a = params.get('k', d)
+ def visit_Assign(self, node: ast.Assign):
+ rhs_key = self._match_param_subscript(node.value) or self._match_params_get(node.value)
+ if rhs_key:
+ for t in node.targets:
+ if isinstance(t, ast.Name):
+ self._alias_from_params[t.id] = rhs_key
+ self.generic_visit(node)
+
+ # Any later read of that alias variable is a usage of the original param key
+ def visit_Name(self, node: ast.Name):
+ if isinstance(node.ctx, ast.Load) and node.id in self._alias_from_params:
+ p = self._alias_from_params[node.id]
+ self.references.append((p, node.lineno, node.col_offset, "alias"))
+ self.generic_visit(node)
+
+ # Direct subscript access: params['k']
+ def visit_Subscript(self, node: ast.Subscript):
+ key = self._match_param_subscript(node)
+ if key:
+ self.references.append((key, node.lineno, node.col_offset, "subscript"))
+ self.generic_visit(node)
+
+ # params.get('k', default)
+ def visit_Call(self, node: ast.Call):
+ key = self._match_params_get(node)
+ if key:
+ self.references.append((key, node.lineno, node.col_offset, "get"))
+ self.generic_visit(node)
+
+ @staticmethod
+ def _match_param_subscript(node: ast.AST) -> Optional[str]:
+ # Match params['k'] where `params` is a Name
+ if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) and node.value.id == "params":
+ sl = node.slice
+ if isinstance(sl, ast.Constant) and isinstance(sl.value, str):
+ return sl.value
+ if hasattr(ast, "Index") and isinstance(sl, ast.Index) and isinstance(sl.value, ast.Constant):
+ if isinstance(sl.value.value, str):
+ return sl.value.value
+ return None
+
+ @staticmethod
+ def _match_params_get(node: ast.AST) -> Optional[str]:
+ # Match params.get('k', ...)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "get":
+ if isinstance(node.func.value, ast.Name) and node.func.value.id == "params":
+ if node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
+ return node.args[0].value
+ return None
+
+
diff --git a/modules/research-framework/simexr_mod/code/models/param_ref.py b/modules/research-framework/simexr_mod/code/models/param_ref.py
new file mode 100644
index 0000000..81c7279
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/models/param_ref.py
@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ParamRef:
+ param: str
+ line: int
+ col: int
+ kind: str # 'subscript' | 'get' | 'alias' | 'name'
+ context: str # one-line preview + caret
diff --git a/modules/research-framework/simexr_mod/code/refactor/llm_refactor.py b/modules/research-framework/simexr_mod/code/refactor/llm_refactor.py
new file mode 100644
index 0000000..cd773f8
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/refactor/llm_refactor.py
@@ -0,0 +1,99 @@
+import re
+import os
+from pathlib import Path
+from typing import Any, Tuple
+
+import openai
+from code.extract.llm_extract import extract_script_settings # assumes this is defined elsewhere
+
+
+def refactor_to_single_entry(
+ script_path: Path,
+ entry_fn: str = "simulate",
+ llm_model: str = "gpt-5-mini",
+ max_attempts: int = 3
+) -> Tuple[Path, Any]:
+ """
+ Refactors a full Python simulation script into a single function `simulate(**params)`
+ which overrides all internally defined parameters and returns a dict.
+ Uses an agentic retry loop to recover from malformed generations.
+ """
+ # Ensure OpenAI API key is configured globally
+ try:
+ from utils.openai_config import ensure_openai_api_key
+ ensure_openai_api_key()
+ print("[LLM_REFACTOR] OpenAI API key configured globally")
+ except Exception as e:
+ print(f"[LLM_REFACTOR] Warning: Could not configure OpenAI API key: {e}")
+
+ print(f"[LLM_REFACTOR] Starting refactor_to_single_entry for {script_path}")
+ original_source = script_path.read_text().strip()
+ print(f"[LLM_REFACTOR] Original source length: {len(original_source)}")
+
+ def build_prompt(source_code: str) -> str:
+ return (
+ f"""
+ You are a helpful **code-refactoring assistant**.
+
+ Your task: Take the entire Python script below and refactor it into a single function:
+
+ def {entry_fn}(**params):
+
+ Requirements for the new function:
+ - Inline all helper functions if needed.
+ - Return **one dictionary** of results with Python built-in datatypes.
+ - Override all internally defined constants/globals with values from `params` if keys exist.
+ - Contain **no top-level code** and **no extra function definitions**.
+ - Must behave as a self-contained black box that depends *only* on its parameters.
+ - Catch common issues like indentation and variable scope errors.
+ - Ensure the data types for all variable are type checked and converted incase of unexpected type inputs.
+
+ If initial condition values are missing from `params`, make an intelligent guess.
+
+ Return ONLY the **Python source code** for the new function (no markdown, no explanations).
+
+ --- Original script ---
+ ```python
+ {source_code}```
+ """)
+
+
+ def is_valid_python(source: str) -> bool:
+ try:
+ compile(source, "<string>", "exec")
+ return True
+ except SyntaxError:
+ return False
+
+ for attempt in range(1, max_attempts + 1):
+ print(f"[LLM_REFACTOR] [Attempt {attempt}] Refactoring script into `{entry_fn}(**params)`...")
+
+ prompt = build_prompt(original_source)
+ print(f"[LLM_REFACTOR] Prompt length: {len(prompt)}")
+
+ print(f"[LLM_REFACTOR] Making OpenAI API call...")
+ resp = openai.chat.completions.create(
+ model=llm_model,
+ messages=[
+ {"role": "system", "content": "You are a code transformation assistant."},
+ {"role": "user", "content": prompt},
+ ],
+ # temperature=0.0,
+ )
+ print(f"[LLM_REFACTOR] OpenAI API call completed")
+
+ content = resp.choices[0].message.content.strip()
+
+ # Clean code fences
+ new_src = re.sub(r"^```python\s*", "", content)
+ new_src = re.sub(r"```$", "", new_src).strip()
+
+ if is_valid_python(new_src):
+ script_path.write_text(new_src)
+ print(f"[Success] Script successfully refactored and written to {script_path}")
+ metadata = extract_script_settings(str(script_path))
+ return script_path, metadata
+ else:
+ print(f"[Warning] Invalid Python generated. Retrying...")
+
+ raise RuntimeError("Failed to refactor the script after multiple attempts.")
diff --git a/modules/research-framework/simexr_mod/code/test/__init__.py b/modules/research-framework/simexr_mod/code/test/__init__.py
new file mode 100644
index 0000000..3081227
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/test/__init__.py
@@ -0,0 +1,5 @@
+"""Code testing utilities."""
+
+from .simulation_refiner import SimulationRefiner
+
+__all__ = ["SimulationRefiner"]
diff --git a/modules/research-framework/simexr_mod/code/test/simulation_refiner.py b/modules/research-framework/simexr_mod/code/test/simulation_refiner.py
new file mode 100644
index 0000000..58e5b17
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/test/simulation_refiner.py
@@ -0,0 +1,46 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from execute.test.fix_agent import FixAgent
+from execute.test.smoke_tester import SmokeTester
+
+# Import database function - adjust path as needed
+from db import store_simulation_script
+
+
+@dataclass
+class SimulationRefiner:
+ """
+ Iteratively smoke-tests a simulate.py and uses an agent to repair it.
+ Writes intermediate .iter{i}.py files; returns model_id when passing.
+ """
+ script_path: Path
+ model_name: str
+ max_iterations: int = 3
+ smoke_tester: SmokeTester = field(default_factory=SmokeTester)
+ agent: FixAgent = field(default_factory=FixAgent)
+
+ def refine(self) -> str:
+ for i in range(1, self.max_iterations + 1):
+ res = self.smoke_tester.test(self.script_path)
+ if res.ok:
+ print(f"[✓] simulate.py passed smoke test on iteration {i}")
+ final_model_id = store_simulation_script(
+ model_name=self.model_name,
+ metadata={}, # keep parity with your original
+ script_path=str(self.script_path),
+ )
+ return final_model_id
+
+ print(f"[!] simulate.py failed on iteration {i}:\n{res.log.strip()}")
+ current_src = self.script_path.read_text()
+
+ corrected_code = self.agent.propose_fix(res.log, current_src)
+
+ # Save intermediate & replace current
+ iter_path = self.script_path.with_name(f"{self.script_path.stem}.iter{i}.py")
+ iter_path.write_text(corrected_code)
+ self.script_path.write_text(corrected_code)
+
+ raise RuntimeError("simulate.py still failing after all correction attempts.")
+
diff --git a/modules/research-framework/simexr_mod/code/utils/github_utils.py b/modules/research-framework/simexr_mod/code/utils/github_utils.py
new file mode 100644
index 0000000..412137e
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/utils/github_utils.py
@@ -0,0 +1,26 @@
+import requests
+from pathlib import Path
+
+def fetch_notebook_from_github(github_url: str, dest_dir: str = "external_models") -> str:
+ """
+ Downloads a file from a GitHub URL and saves it locally.
+ Handles both raw URLs and blob URLs.
+ Returns the local path to the saved file.
+ """
+ # Convert GitHub blob URL to raw URL if needed
+ if "github.com" in github_url and "/blob/" in github_url:
+ raw_url = github_url.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/")
+ else:
+ raw_url = github_url
+
+ print(f"[GITHUB_UTILS] Converting {github_url} to {raw_url}")
+
+ resp = requests.get(raw_url)
+ resp.raise_for_status()
+
+ Path(dest_dir).mkdir(exist_ok=True, parents=True)
+ filename = Path(raw_url).name
+ local_path = Path(dest_dir) / filename
+ local_path.write_bytes(resp.content)
+ print(f"[GITHUB_UTILS] Downloaded file to {local_path}")
+ return str(local_path)
diff --git a/modules/research-framework/simexr_mod/code/utils/notebook_utils.py b/modules/research-framework/simexr_mod/code/utils/notebook_utils.py
new file mode 100644
index 0000000..d1fbb95
--- /dev/null
+++ b/modules/research-framework/simexr_mod/code/utils/notebook_utils.py
@@ -0,0 +1,36 @@
+import nbformat
+from nbconvert import PythonExporter
+from pathlib import Path
+import shutil
+
+def notebook_to_script(notebook_path: str, output_dir: str = "external_models") -> str:
+ """
+ If `notebook_path` is a Jupyter notebook (.ipynb), convert it to a .py script
+ in `output_dir`, returning the script path.
+ If it's already a .py file, ensure it's in `output_dir` (copy if needed)
+ and return its path.
+ """
+ src = Path(notebook_path)
+ out_dir = Path(output_dir)
+ out_dir.mkdir(parents=True, exist_ok=True)
+
+ # Case 1: Already a Python script
+ if src.suffix.lower() == ".py":
+ dest = out_dir / src.name
+ # copy only if not already in the target dir
+ if src.resolve() != dest.resolve():
+ shutil.copy2(src, dest)
+ return str(dest)
+
+ # Case 2: Jupyter notebook → Python script
+ if src.suffix.lower() == ".ipynb":
+ nb = nbformat.read(src, as_version=4)
+ exporter = PythonExporter()
+ script_source, _ = exporter.from_notebook_node(nb)
+
+ py_path = out_dir / (src.stem + ".py")
+ py_path.write_text(script_source)
+ return str(py_path)
+
+ # Unsupported extension
+ raise ValueError(f"Cannot convert '{notebook_path}': unsupported extension '{src.suffix}'")
diff --git a/modules/research-framework/simexr_mod/config.yaml.example b/modules/research-framework/simexr_mod/config.yaml.example
new file mode 100644
index 0000000..435047f
--- /dev/null
+++ b/modules/research-framework/simexr_mod/config.yaml.example
@@ -0,0 +1,15 @@
+# SimExR Configuration File
+# Copy this file to config.yaml and update with your actual values
+
+openai:
+ # Replace with your actual OpenAI API key
+ # Get your API key from: https://platform.openai.com/account/api-keys
+ api_key: "YOUR_OPENAI_API_KEY_HERE"
+
+database:
+ # Path to the SQLite database file
+ path: "mcp.db"
+
+media:
+ # Root directory for storing simulation results and media files
+ root: "results_media"
diff --git a/modules/research-framework/simexr_mod/core/__init__.py b/modules/research-framework/simexr_mod/core/__init__.py
new file mode 100644
index 0000000..9a5529b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/core/__init__.py
@@ -0,0 +1 @@
+# Core module stub
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/core/interfaces.py b/modules/research-framework/simexr_mod/core/interfaces.py
new file mode 100644
index 0000000..9cd08ec
--- /dev/null
+++ b/modules/research-framework/simexr_mod/core/interfaces.py
@@ -0,0 +1,224 @@
+"""
+Interface definitions for SimExR components.
+
+These interfaces define contracts that concrete implementations must follow,
+promoting loose coupling and testability.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Optional, Protocol, Union
+from pathlib import Path
+from dataclasses import dataclass
+from enum import Enum
+
+
+class SimulationStatus(Enum):
+ """Status of a simulation execution."""
+ PENDING = "pending"
+ RUNNING = "running"
+ COMPLETED = "completed"
+ FAILED = "failed"
+ CANCELLED = "cancelled"
+
+
+@dataclass
+class SimulationResult:
+ """Standardized simulation result."""
+ status: SimulationStatus
+ parameters: Dict[str, Any]
+ outputs: Dict[str, Any]
+ execution_time: float
+ error_message: Optional[str] = None
+ stdout: str = ""
+ stderr: str = ""
+ metadata: Dict[str, Any] = None
+
+
+@dataclass
+class SimulationRequest:
+ """Standardized simulation request."""
+ model_id: str
+ parameters: Dict[str, Any]
+ execution_options: Dict[str, Any] = None
+ priority: int = 0
+ timeout: Optional[float] = None
+
+
+class ISimulationRunner(Protocol):
+ """Interface for simulation execution."""
+
+ def run(self, request: SimulationRequest) -> SimulationResult:
+ """Execute a single simulation."""
+ ...
+
+ def run_batch(self, requests: List[SimulationRequest]) -> List[SimulationResult]:
+ """Execute multiple simulations."""
+ ...
+
+ def cancel(self, execution_id: str) -> bool:
+ """Cancel a running simulation."""
+ ...
+
+
+class ISimulationLoader(Protocol):
+ """Interface for loading simulation models."""
+
+ def load_model(self, model_id: str) -> Any:
+ """Load a simulation model by ID."""
+ ...
+
+ def validate_model(self, model_path: Path) -> bool:
+ """Validate that a model is properly formatted."""
+ ...
+
+ def get_model_metadata(self, model_id: str) -> Dict[str, Any]:
+ """Get metadata for a model."""
+ ...
+
+
+class IResultStore(Protocol):
+ """Interface for storing and retrieving simulation results."""
+
+ def store_result(self, result: SimulationResult) -> str:
+ """Store a simulation result and return an ID."""
+ ...
+
+ def get_result(self, result_id: str) -> Optional[SimulationResult]:
+ """Retrieve a simulation result by ID."""
+ ...
+
+ def query_results(self, filters: Dict[str, Any]) -> List[SimulationResult]:
+ """Query results with filters."""
+ ...
+
+ def delete_results(self, model_id: str) -> int:
+ """Delete all results for a model."""
+ ...
+
+
+class IReasoningAgent(Protocol):
+ """Interface for reasoning agents."""
+
+ def ask(self, question: str, context: Dict[str, Any]) -> Dict[str, Any]:
+ """Ask a question about simulation data."""
+ ...
+
+ def analyze_results(self, results: List[SimulationResult]) -> Dict[str, Any]:
+ """Analyze simulation results."""
+ ...
+
+ def suggest_parameters(self, model_id: str, objective: str) -> Dict[str, Any]:
+ """Suggest parameter values for a given objective."""
+ ...
+
+
+class IEventListener(Protocol):
+ """Interface for event listeners."""
+
+ def on_simulation_started(self, request: SimulationRequest) -> None:
+ """Called when a simulation starts."""
+ ...
+
+ def on_simulation_completed(self, result: SimulationResult) -> None:
+ """Called when a simulation completes."""
+ ...
+
+ def on_simulation_failed(self, request: SimulationRequest, error: Exception) -> None:
+ """Called when a simulation fails."""
+ ...
+
+
+class IExecutionStrategy(Protocol):
+ """Interface for execution strategies."""
+
+ def execute(self, request: SimulationRequest) -> SimulationResult:
+ """Execute a simulation using this strategy."""
+ ...
+
+ def can_handle(self, request: SimulationRequest) -> bool:
+ """Check if this strategy can handle the request."""
+ ...
+
+ def get_priority(self) -> int:
+ """Get the priority of this strategy (higher = more preferred)."""
+ ...
+
+
+class IModelAdapter(Protocol):
+ """Interface for adapting different model formats."""
+
+ def can_adapt(self, source_format: str, target_format: str) -> bool:
+ """Check if this adapter can handle the conversion."""
+ ...
+
+ def adapt(self, model_content: str, source_format: str, target_format: str) -> str:
+ """Convert model from source to target format."""
+ ...
+
+ def get_supported_formats(self) -> List[str]:
+ """Get list of supported formats."""
+ ...
+
+
+class IResourceManager(Protocol):
+ """Interface for managing shared resources."""
+
+ def acquire_resource(self, resource_type: str, **kwargs) -> Any:
+ """Acquire a shared resource."""
+ ...
+
+ def release_resource(self, resource: Any) -> None:
+ """Release a shared resource."""
+ ...
+
+ def cleanup(self) -> None:
+ """Clean up all resources."""
+ ...
+
+
+class IConfigurationProvider(Protocol):
+ """Interface for providing configuration."""
+
+ def get_config(self, key: str, default: Any = None) -> Any:
+ """Get a configuration value."""
+ ...
+
+ def set_config(self, key: str, value: Any) -> None:
+ """Set a configuration value."""
+ ...
+
+ def get_all_config(self) -> Dict[str, Any]:
+ """Get all configuration values."""
+ ...
+
+
+class IValidationRule(Protocol):
+ """Interface for validation rules."""
+
+ def validate(self, data: Any) -> bool:
+ """Validate data against this rule."""
+ ...
+
+ def get_error_message(self) -> str:
+ """Get error message for validation failure."""
+ ...
+
+
+class ISecurityProvider(Protocol):
+ """Interface for security operations."""
+
+ def authenticate(self, credentials: Dict[str, Any]) -> bool:
+ """Authenticate user credentials."""
+ ...
+
+ def authorize(self, user_id: str, operation: str, resource: str) -> bool:
+ """Authorize user operation on resource."""
+ ...
+
+ def encrypt_sensitive_data(self, data: str) -> str:
+ """Encrypt sensitive data."""
+ ...
+
+ def decrypt_sensitive_data(self, encrypted_data: str) -> str:
+ """Decrypt sensitive data."""
+ ...
diff --git a/modules/research-framework/simexr_mod/core/parser.py b/modules/research-framework/simexr_mod/core/parser.py
new file mode 100644
index 0000000..e47e80a
--- /dev/null
+++ b/modules/research-framework/simexr_mod/core/parser.py
@@ -0,0 +1,65 @@
+"""
+Stub for core.parser module to satisfy imports.
+Since parser flow is ignored, providing minimal implementation.
+"""
+
+import json
+import re
+
+
+def tidy_json(json_string: str) -> str:
+ """
+ Clean up JSON string to make it parseable.
+ Simple implementation to satisfy the import requirement.
+ """
+ if not isinstance(json_string, str):
+ return "{}"
+
+ # Remove markdown code blocks
+ json_string = re.sub(r"```(?:json)?", "", json_string, flags=re.IGNORECASE)
+
+ # Remove leading "json" labels
+ json_string = re.sub(r"^\s*json\s*\n", "", json_string, flags=re.IGNORECASE | re.MULTILINE)
+
+ # Strip whitespace
+ json_string = json_string.strip()
+
+ # If empty, return empty object
+ if not json_string:
+ return "{}"
+
+ # Try to fix common JSON issues
+ try:
+ # Test if it's already valid JSON
+ json.loads(json_string)
+ return json_string
+ except json.JSONDecodeError:
+ # Basic cleanup attempts
+
+ # Fix single quotes to double quotes
+ json_string = re.sub(r"'([^']*)':", r'"\1":', json_string)
+ json_string = re.sub(r":\s*'([^']*)'", r': "\1"', json_string)
+
+ # Ensure it starts and ends with braces
+ json_string = json_string.strip()
+ if not json_string.startswith('{'):
+ json_string = '{' + json_string
+ if not json_string.endswith('}'):
+ json_string = json_string + '}'
+
+ return json_string
+
+
+def parse_nl_input(query: str, retries: int = 3, temperature: float = 0.0) -> dict:
+ """
+ Stub for natural language parsing.
+ Returns a basic structure for testing.
+ """
+ return {
+ "model_name": "parsed_model",
+ "description": f"Parsed from: {query[:50]}...",
+ "parameters": {
+ "param1": {"type": "float", "default": 1.0},
+ "param2": {"type": "float", "default": 0.5}
+ }
+ }
diff --git a/modules/research-framework/simexr_mod/core/patterns.py b/modules/research-framework/simexr_mod/core/patterns.py
new file mode 100644
index 0000000..9227d96
--- /dev/null
+++ b/modules/research-framework/simexr_mod/core/patterns.py
@@ -0,0 +1,686 @@
+"""
+Implementation of key design patterns for SimExR.
+
+This module provides concrete implementations of various design patterns
+to improve code organization and maintainability.
+"""
+
+import threading
+import weakref
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Optional, Type, Callable, Union
+from pathlib import Path
+from dataclasses import dataclass, field
+from enum import Enum
+import time
+import uuid
+
+from .interfaces import (
+ ISimulationRunner, ISimulationLoader, IResultStore, IReasoningAgent,
+ IEventListener, IExecutionStrategy, IModelAdapter, IResourceManager,
+ SimulationRequest, SimulationResult, SimulationStatus
+)
+
+
+# ===== FACTORY PATTERN =====
+
+class ComponentType(Enum):
+ """Types of components that can be created by the factory."""
+ SIMULATION_RUNNER = "simulation_runner"
+ RESULT_STORE = "result_store"
+ REASONING_AGENT = "reasoning_agent"
+ MODEL_LOADER = "model_loader"
+ EXECUTION_STRATEGY = "execution_strategy"
+
+
+class SimulationFactory:
+ """Factory for creating simulation-related components."""
+
+ def __init__(self):
+ self._creators: Dict[ComponentType, Callable] = {}
+ self._instances: Dict[str, Any] = {}
+
+ def register_creator(self, component_type: ComponentType, creator: Callable):
+ """Register a creator function for a component type."""
+ self._creators[component_type] = creator
+
+ def create(self, component_type: ComponentType, **kwargs) -> Any:
+ """Create a component of the specified type."""
+ if component_type not in self._creators:
+ raise ValueError(f"No creator registered for {component_type}")
+
+ creator = self._creators[component_type]
+ return creator(**kwargs)
+
+ def create_singleton(self, component_type: ComponentType, instance_id: str, **kwargs) -> Any:
+ """Create or retrieve a singleton instance."""
+ if instance_id in self._instances:
+ return self._instances[instance_id]
+
+ instance = self.create(component_type, **kwargs)
+ self._instances[instance_id] = instance
+ return instance
+
+ def get_registered_types(self) -> List[ComponentType]:
+ """Get list of registered component types."""
+ return list(self._creators.keys())
+
+
+# ===== STRATEGY PATTERN =====
+
+class LocalExecutionStrategy:
+ """Strategy for executing simulations locally."""
+
+ def __init__(self, timeout: float = 30.0):
+ self.timeout = timeout
+ self.logger = logging.getLogger("LocalExecutionStrategy")
+ self.logger.setLevel(logging.INFO)
+
+ def execute(self, request: SimulationRequest) -> SimulationResult:
+ """Execute simulation locally."""
+ self.logger.info(f"[LOCAL_EXECUTION] Starting local execution for model {request.model_id}")
+ self.logger.info(f"[LOCAL_EXECUTION] Parameters: {request.parameters}")
+ self.logger.info(f"[LOCAL_EXECUTION] Timeout: {self.timeout}s")
+
+ start_time = time.time()
+
+ try:
+ # Import here to avoid circular dependencies
+ from execute.run.simulation_runner import SimulationRunner
+ from db import get_simulation_path
+
+ self.logger.info(f"[LOCAL_EXECUTION] Getting simulation path for model {request.model_id}")
+ script_path = Path(get_simulation_path(request.model_id))
+ self.logger.info(f"[LOCAL_EXECUTION] Script path: {script_path}")
+
+ self.logger.info(f"[LOCAL_EXECUTION] Creating SimulationRunner")
+ runner = SimulationRunner()
+
+ self.logger.info(f"[LOCAL_EXECUTION] Running simulation with runner")
+ result = runner.run(script_path, request.parameters)
+
+ execution_time = time.time() - start_time
+ self.logger.info(f"[LOCAL_EXECUTION] Simulation completed in {execution_time:.3f}s")
+
+ success = result.get("_ok", False)
+ self.logger.info(f"[LOCAL_EXECUTION] Success status: {success}")
+
+ # Log result preview
+ if success:
+ self.logger.info(f"[LOCAL_EXECUTION] Creating successful SimulationResult")
+ self._log_final_result_preview(result)
+ else:
+ self.logger.warning(f"[LOCAL_EXECUTION] Creating failed SimulationResult")
+
+ return SimulationResult(
+ status=SimulationStatus.COMPLETED if success else SimulationStatus.FAILED,
+ parameters=request.parameters,
+ outputs={k: v for k, v in result.items() if not k.startswith("_")},
+ execution_time=execution_time,
+ stdout=result.get("_stdout", ""),
+ stderr=result.get("_stderr", ""),
+ error_message=result.get("_error_msg") if not success else None
+ )
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ self.logger.error(f"[LOCAL_EXECUTION] Execution failed after {execution_time:.3f}s: {str(e)}")
+ self.logger.error(f"[LOCAL_EXECUTION] Error type: {type(e).__name__}")
+
+ return SimulationResult(
+ status=SimulationStatus.FAILED,
+ parameters=request.parameters,
+ outputs={},
+ execution_time=execution_time,
+ error_message=str(e)
+ )
+
+ def can_handle(self, request: SimulationRequest) -> bool:
+ """Check if this strategy can handle the request."""
+ return True # Local execution can handle any request
+
+ def get_priority(self) -> int:
+ """Get priority (lower = higher priority)."""
+ return 10
+
+ def _log_final_result_preview(self, result: Dict[str, Any]) -> None:
+ """Log a preview of the final simulation results."""
+ self.logger.info(f"[LOCAL_EXECUTION] === FINAL RESULT SUMMARY ===")
+
+ # Show key metrics
+ if 'success' in result:
+ self.logger.info(f"[LOCAL_EXECUTION] Success: {result['success']}")
+
+ if 'solver_message' in result:
+ self.logger.info(f"[LOCAL_EXECUTION] Solver: {result['solver_message']}")
+
+ # Show data sizes
+ for key in ['t', 'x', 'y']:
+ if key in result and isinstance(result[key], (list, tuple)):
+ self.logger.info(f"[LOCAL_EXECUTION] {key.upper()} data points: {len(result[key])}")
+
+ # Show grid info if available
+ for key in ['x_grid', 'y_grid', 'u_grid', 'v_grid']:
+ if key in result and isinstance(result[key], (list, tuple)):
+ if len(result[key]) > 0 and isinstance(result[key][0], (list, tuple)):
+ self.logger.info(f"[LOCAL_EXECUTION] {key.upper()} grid: {len(result[key])}x{len(result[key][0])}")
+
+ # Show key parameters
+ for key in ['mu', 'z0', 'eval_time', 't_iteration', 'grid_points', 'mgrid_size']:
+ if key in result:
+ self.logger.info(f"[LOCAL_EXECUTION] {key}: {result[key]}")
+
+ self.logger.info(f"[LOCAL_EXECUTION] === END FINAL RESULT SUMMARY ===")
+
+
+class RemoteExecutionStrategy:
+ """Strategy for executing simulations remotely (placeholder)."""
+
+ def __init__(self, endpoint: str):
+ self.endpoint = endpoint
+
+ def execute(self, request: SimulationRequest) -> SimulationResult:
+ """Execute simulation remotely."""
+ # Placeholder implementation
+ raise NotImplementedError("Remote execution not yet implemented")
+
+ def can_handle(self, request: SimulationRequest) -> bool:
+ """Check if this strategy can handle the request."""
+ return False # Not implemented yet
+
+ def get_priority(self) -> int:
+ """Get priority."""
+ return 5 # Higher priority than local if available
+
+
+class ExecutionStrategyManager:
+ """Manages different execution strategies."""
+
+ def __init__(self):
+ self.strategies: List[IExecutionStrategy] = []
+
+ def add_strategy(self, strategy: IExecutionStrategy):
+ """Add an execution strategy."""
+ self.strategies.append(strategy)
+ # Sort by priority (lower number = higher priority)
+ self.strategies.sort(key=lambda s: s.get_priority())
+
+ def execute(self, request: SimulationRequest) -> SimulationResult:
+ """Execute using the best available strategy."""
+ for strategy in self.strategies:
+ if strategy.can_handle(request):
+ return strategy.execute(request)
+
+ raise RuntimeError("No execution strategy available for this request")
+
+
+# ===== OBSERVER PATTERN =====
+
+class SimulationEvent:
+ """Event data for simulation notifications."""
+
+ def __init__(self, event_type: str, data: Dict[str, Any]):
+ self.event_type = event_type
+ self.data = data
+ self.timestamp = time.time()
+
+
+class SimulationSubject:
+ """Subject that notifies observers of simulation events."""
+
+ def __init__(self):
+ self._observers: List[IEventListener] = []
+
+ def attach(self, observer: IEventListener):
+ """Attach an observer."""
+ if observer not in self._observers:
+ self._observers.append(observer)
+
+ def detach(self, observer: IEventListener):
+ """Detach an observer."""
+ if observer in self._observers:
+ self._observers.remove(observer)
+
+ def notify_started(self, request: SimulationRequest):
+ """Notify all observers that a simulation started."""
+ for observer in self._observers:
+ try:
+ observer.on_simulation_started(request)
+ except Exception as e:
+ print(f"Observer notification failed: {e}")
+
+ def notify_completed(self, result: SimulationResult):
+ """Notify all observers that a simulation completed."""
+ for observer in self._observers:
+ try:
+ observer.on_simulation_completed(result)
+ except Exception as e:
+ print(f"Observer notification failed: {e}")
+
+ def notify_failed(self, request: SimulationRequest, error: Exception):
+ """Notify all observers that a simulation failed."""
+ for observer in self._observers:
+ try:
+ observer.on_simulation_failed(request, error)
+ except Exception as e:
+ print(f"Observer notification failed: {e}")
+
+
+class LoggingObserver:
+ """Observer that logs simulation events."""
+
+ def __init__(self, log_file: Optional[Path] = None):
+ self.log_file = log_file
+
+ def on_simulation_started(self, request: SimulationRequest):
+ """Log simulation start."""
+ message = f"Simulation started: {request.model_id} with params {request.parameters}"
+ self._log(message)
+
+ def on_simulation_completed(self, result: SimulationResult):
+ """Log simulation completion."""
+ message = f"Simulation completed: {result.status.value} in {result.execution_time:.2f}s"
+ self._log(message)
+
+ def on_simulation_failed(self, request: SimulationRequest, error: Exception):
+ """Log simulation failure."""
+ message = f"Simulation failed: {request.model_id} - {str(error)}"
+ self._log(message)
+
+ def _log(self, message: str):
+ """Write log message."""
+ timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+ full_message = f"[{timestamp}] {message}"
+
+ if self.log_file:
+ with open(self.log_file, 'a') as f:
+ f.write(full_message + "\n")
+ else:
+ print(full_message)
+
+
+# ===== COMMAND PATTERN =====
+
+class Command(ABC):
+ """Abstract command interface."""
+
+ @abstractmethod
+ def execute(self) -> Any:
+ """Execute the command."""
+ pass
+
+ @abstractmethod
+ def undo(self) -> Any:
+ """Undo the command."""
+ pass
+
+
+class RunSimulationCommand(Command):
+ """Command to run a simulation."""
+
+ def __init__(self, runner: ISimulationRunner, request: SimulationRequest):
+ self.runner = runner
+ self.request = request
+ self.result: Optional[SimulationResult] = None
+
+ def execute(self) -> SimulationResult:
+ """Execute the simulation."""
+ self.result = self.runner.run(self.request)
+ return self.result
+
+ def undo(self) -> None:
+ """Undo not applicable for simulation execution."""
+ pass
+
+
+class StoreModelCommand(Command):
+ """Command to store a simulation model."""
+
+ def __init__(self, model_name: str, metadata: Dict[str, Any], script_content: str):
+ self.model_name = model_name
+ self.metadata = metadata
+ self.script_content = script_content
+ self.model_id: Optional[str] = None
+
+ def execute(self) -> str:
+ """Store the model."""
+ from db import store_simulation_script
+ import tempfile
+
+ # Create temporary script file
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(self.script_content)
+ temp_path = f.name
+
+ try:
+ self.model_id = store_simulation_script(
+ model_name=self.model_name,
+ metadata=self.metadata,
+ script_path=temp_path
+ )
+ return self.model_id
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+ def undo(self) -> None:
+ """Delete the stored model."""
+ if self.model_id:
+ # Implementation would delete the model from database
+ pass
+
+
+class CommandInvoker:
+ """Invoker that executes commands and maintains history."""
+
+ def __init__(self):
+ self.history: List[Command] = []
+
+ def execute_command(self, command: Command) -> Any:
+ """Execute a command and add to history."""
+ result = command.execute()
+ self.history.append(command)
+ return result
+
+ def undo_last(self) -> None:
+ """Undo the last command."""
+ if self.history:
+ command = self.history.pop()
+ command.undo()
+
+
+# ===== BUILDER PATTERN =====
+
+class SimulationConfigBuilder:
+ """Builder for creating complex simulation configurations."""
+
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ """Reset the builder state."""
+ self._config = {
+ 'model_id': None,
+ 'parameters': {},
+ 'execution_options': {},
+ 'validation_rules': [],
+ 'observers': [],
+ 'strategies': []
+ }
+ return self
+
+ def set_model(self, model_id: str):
+ """Set the simulation model."""
+ self._config['model_id'] = model_id
+ return self
+
+ def add_parameter(self, name: str, value: Any):
+ """Add a simulation parameter."""
+ self._config['parameters'][name] = value
+ return self
+
+ def add_parameters(self, parameters: Dict[str, Any]):
+ """Add multiple simulation parameters."""
+ self._config['parameters'].update(parameters)
+ return self
+
+ def set_execution_option(self, name: str, value: Any):
+ """Set an execution option."""
+ self._config['execution_options'][name] = value
+ return self
+
+ def set_timeout(self, timeout: float):
+ """Set execution timeout."""
+ self._config['execution_options']['timeout'] = timeout
+ return self
+
+ def set_priority(self, priority: int):
+ """Set execution priority."""
+ self._config['execution_options']['priority'] = priority
+ return self
+
+ def add_observer(self, observer: IEventListener):
+ """Add an event observer."""
+ self._config['observers'].append(observer)
+ return self
+
+ def add_strategy(self, strategy: IExecutionStrategy):
+ """Add an execution strategy."""
+ self._config['strategies'].append(strategy)
+ return self
+
+ def build(self) -> Dict[str, Any]:
+ """Build the final configuration."""
+ if not self._config['model_id']:
+ raise ValueError("Model ID is required")
+
+ config = self._config.copy()
+ self.reset()
+ return config
+
+
+# ===== SINGLETON PATTERN =====
+
+class SingletonMeta(type):
+ """Metaclass for creating singleton instances."""
+
+ _instances = {}
+ _lock = threading.Lock()
+
+ def __call__(cls, *args, **kwargs):
+ if cls not in cls._instances:
+ with cls._lock:
+ if cls not in cls._instances:
+ cls._instances[cls] = super().__call__(*args, **kwargs)
+ return cls._instances[cls]
+
+
+class ResourceManager(metaclass=SingletonMeta):
+ """Singleton resource manager for shared resources."""
+
+ def __init__(self):
+ if hasattr(self, '_initialized'):
+ return
+
+ self._resources: Dict[str, Any] = {}
+ self._locks: Dict[str, threading.Lock] = {}
+ self._initialized = True
+
+ def get_resource(self, resource_id: str, factory: Callable = None) -> Any:
+ """Get or create a resource."""
+ if resource_id not in self._resources:
+ if factory is None:
+ raise ValueError(f"Resource {resource_id} not found and no factory provided")
+
+ if resource_id not in self._locks:
+ self._locks[resource_id] = threading.Lock()
+
+ with self._locks[resource_id]:
+ if resource_id not in self._resources:
+ self._resources[resource_id] = factory()
+
+ return self._resources[resource_id]
+
+ def set_resource(self, resource_id: str, resource: Any):
+ """Set a resource."""
+ self._resources[resource_id] = resource
+
+ def release_resource(self, resource_id: str):
+ """Release a resource."""
+ if resource_id in self._resources:
+ resource = self._resources.pop(resource_id)
+ if hasattr(resource, 'cleanup'):
+ resource.cleanup()
+
+ def cleanup_all(self):
+ """Clean up all resources."""
+ for resource_id in list(self._resources.keys()):
+ self.release_resource(resource_id)
+
+
+# ===== ADAPTER PATTERN =====
+
+class GitHubScriptAdapter:
+ """Adapter for importing scripts from GitHub."""
+
+ def __init__(self):
+ self.supported_formats = ["github_url", "raw_github_url"]
+
+ def can_adapt(self, source_format: str, target_format: str) -> bool:
+ """Check if adapter can handle the conversion."""
+ return (source_format in self.supported_formats and
+ target_format == "simexr_script")
+
+ def adapt(self, source: str, source_format: str, target_format: str) -> str:
+ """Convert GitHub URL to SimExR script format."""
+ if not self.can_adapt(source_format, target_format):
+ raise ValueError(f"Cannot adapt from {source_format} to {target_format}")
+
+ if source_format == "github_url":
+ # Convert GitHub URL to raw URL
+ raw_url = self._github_url_to_raw(source)
+ else:
+ raw_url = source
+
+ # Download the script content
+ import requests
+ response = requests.get(raw_url)
+ response.raise_for_status()
+
+ script_content = response.text
+
+ # Adapt to SimExR format (ensure it has a simulate function)
+ if "def simulate(" not in script_content:
+ script_content = self._wrap_as_simulate_function(script_content)
+
+ return script_content
+
+ def _github_url_to_raw(self, github_url: str) -> str:
+ """Convert GitHub URL to raw content URL."""
+ if "github.com" in github_url and "/blob/" in github_url:
+ return github_url.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/")
+ return github_url
+
+ def _wrap_as_simulate_function(self, script_content: str) -> str:
+ """Wrap script content as a simulate function if needed."""
+ # This is a simple wrapper - could be more sophisticated
+ return f"""
+def simulate(**params):
+ '''Auto-generated simulate function wrapper.'''
+ # Original script content:
+{script_content}
+
+ # Return some default output
+ return {{"status": "completed", "params": params}}
+"""
+
+
+# ===== FACADE PATTERN =====
+
+class SimulationFacade:
+ """Simplified interface for complex simulation operations."""
+
+ def __init__(self):
+ self.factory = SimulationFactory()
+ self.strategy_manager = ExecutionStrategyManager()
+ self.subject = SimulationSubject()
+ self.command_invoker = CommandInvoker()
+ self.resource_manager = ResourceManager()
+
+ # Register default strategies
+ self.strategy_manager.add_strategy(LocalExecutionStrategy())
+
+ # Add default logging observer
+ log_observer = LoggingObserver()
+ self.subject.attach(log_observer)
+
+ def run_simulation(self, model_id: str, parameters: Dict[str, Any], **options) -> SimulationResult:
+ """Run a simulation with simplified interface."""
+ request = SimulationRequest(
+ model_id=model_id,
+ parameters=parameters,
+ execution_options=options
+ )
+
+ self.subject.notify_started(request)
+
+ try:
+ result = self.strategy_manager.execute(request)
+ self.subject.notify_completed(result)
+ return result
+ except Exception as e:
+ self.subject.notify_failed(request, e)
+ raise
+
+ def import_from_github(self, github_url: str, model_name: str, metadata: Dict[str, Any] = None) -> str:
+ """Import a simulation model from GitHub."""
+ adapter = GitHubScriptAdapter()
+
+ # Adapt the GitHub script
+ script_content = adapter.adapt(github_url, "github_url", "simexr_script")
+
+ # Store the model
+ command = StoreModelCommand(model_name, metadata or {}, script_content)
+ return self.command_invoker.execute_command(command)
+
+ def create_batch_configuration(self) -> SimulationConfigBuilder:
+ """Create a builder for batch simulation configuration."""
+ return SimulationConfigBuilder()
+
+ def add_observer(self, observer: IEventListener):
+ """Add an event observer."""
+ self.subject.attach(observer)
+
+ def cleanup(self):
+ """Clean up all resources."""
+ self.resource_manager.cleanup_all()
+
+
+# ===== DEPENDENCY INJECTION CONTAINER =====
+
+class DIContainer:
+ """Dependency injection container."""
+
+ def __init__(self):
+ self._services: Dict[str, Any] = {}
+ self._factories: Dict[str, Callable] = {}
+ self._singletons: Dict[str, Any] = {}
+
+ def register_instance(self, service_name: str, instance: Any):
+ """Register a service instance."""
+ self._services[service_name] = instance
+
+ def register_factory(self, service_name: str, factory: Callable):
+ """Register a factory function for a service."""
+ self._factories[service_name] = factory
+
+ def register_singleton(self, service_name: str, factory: Callable):
+ """Register a singleton service."""
+ self._factories[service_name] = factory
+ # Mark as singleton by adding to singletons dict with None value
+ if service_name not in self._singletons:
+ self._singletons[service_name] = None
+
+ def get(self, service_name: str) -> Any:
+ """Get a service instance."""
+ # Check for direct instance
+ if service_name in self._services:
+ return self._services[service_name]
+
+ # Check for singleton
+ if service_name in self._singletons:
+ if self._singletons[service_name] is None:
+ self._singletons[service_name] = self._factories[service_name]()
+ return self._singletons[service_name]
+
+ # Check for factory
+ if service_name in self._factories:
+ return self._factories[service_name]()
+
+ raise ValueError(f"Service {service_name} not registered")
+
+ def has(self, service_name: str) -> bool:
+ """Check if a service is registered."""
+ return (service_name in self._services or
+ service_name in self._factories or
+ service_name in self._singletons)
diff --git a/modules/research-framework/simexr_mod/core/services.py b/modules/research-framework/simexr_mod/core/services.py
new file mode 100644
index 0000000..e3bfd81
--- /dev/null
+++ b/modules/research-framework/simexr_mod/core/services.py
@@ -0,0 +1,691 @@
+"""
+Service layer implementations for SimExR.
+
+This module provides high-level service classes that orchestrate
+various components and implement business logic.
+"""
+
+from typing import Dict, Any, List, Optional, Union
+from pathlib import Path
+from dataclasses import dataclass
+import time
+import json
+import logging
+
+from .interfaces import (
+ ISimulationRunner, ISimulationLoader, IResultStore, IReasoningAgent,
+ SimulationRequest, SimulationResult, SimulationStatus
+)
+from .patterns import (
+ SimulationFacade, SimulationFactory, ExecutionStrategyManager,
+ SimulationSubject, CommandInvoker, ResourceManager, DIContainer,
+ LocalExecutionStrategy, LoggingObserver, GitHubScriptAdapter
+)
+
+
+@dataclass
+class ServiceConfiguration:
+ """Configuration for services."""
+ database_path: str = "mcp.db"
+ models_directory: str = "systems/models"
+ results_directory: str = "results_media"
+ default_timeout: float = 30.0
+ max_batch_size: int = 1000
+ enable_logging: bool = True
+ log_file: Optional[str] = None
+
+
+class SimulationService:
+ """High-level simulation service."""
+
+ def __init__(self, config: ServiceConfiguration = None):
+ self.config = config or ServiceConfiguration()
+ self.facade = SimulationFacade()
+ self.logger = self._setup_logging()
+
+ # Initialize components
+ self._initialize_components()
+
+ def _setup_logging(self) -> logging.Logger:
+ """Set up logging for the service."""
+ logger = logging.getLogger("SimulationService")
+ logger.setLevel(logging.INFO if self.config.enable_logging else logging.WARNING)
+
+ if not logger.handlers:
+ handler = logging.StreamHandler()
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ )
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ return logger
+
+ def _initialize_components(self):
+ """Initialize service components."""
+ if self.config.enable_logging:
+ log_file = Path(self.config.log_file) if self.config.log_file else None
+ observer = LoggingObserver(log_file)
+ self.facade.add_observer(observer)
+
+ def run_single_simulation(
+ self,
+ model_id: str,
+ parameters: Dict[str, Any],
+ timeout: Optional[float] = None
+ ) -> SimulationResult:
+ """Run a single simulation."""
+ self.logger.info(f"Running simulation for model {model_id}")
+
+ try:
+ result = self.facade.run_simulation(
+ model_id=model_id,
+ parameters=parameters,
+ timeout=timeout or self.config.default_timeout
+ )
+
+ self.logger.info(f"Simulation completed with status: {result.status.value}")
+
+ # Save results to database if successful
+ if result.status == SimulationStatus.COMPLETED:
+ try:
+ from db import store_simulation_results
+
+ # Convert SimulationResult to the format expected by store_simulation_results
+ result_row = {
+ **result.parameters, # Include all parameters
+ **result.outputs, # Include all outputs
+ '_ok': True, # Mark as successful
+ '_execution_time': result.execution_time
+ }
+
+ # Extract parameter keys from the parameters dict
+ param_keys = list(result.parameters.keys())
+
+ # Store the result
+ store_simulation_results(model_id, [result_row], param_keys)
+ self.logger.info(f"Results saved to database for model {model_id}")
+
+ except Exception as save_error:
+ self.logger.warning(f"Failed to save results to database: {save_error}")
+
+ return result
+
+ except Exception as e:
+ self.logger.error(f"Simulation failed: {str(e)}")
+ raise
+
+ def run_batch_simulations(
+ self,
+ model_id: str,
+ parameter_grid: List[Dict[str, Any]],
+ max_workers: int = 4
+ ) -> List[SimulationResult]:
+ """Run multiple simulations in batch."""
+ if len(parameter_grid) > self.config.max_batch_size:
+ raise ValueError(f"Batch size {len(parameter_grid)} exceeds maximum {self.config.max_batch_size}")
+
+ self.logger.info(f"Running batch of {len(parameter_grid)} simulations for model {model_id}")
+
+ # Import tqdm for progress bar
+ try:
+ from tqdm import tqdm
+ use_tqdm = True
+ self.logger.info("Using tqdm for progress tracking")
+ except ImportError:
+ use_tqdm = False
+ self.logger.warning("tqdm not available, running without progress bar")
+
+ results = []
+ iterator = tqdm(parameter_grid, desc=f"Running {model_id} simulations") if use_tqdm else enumerate(parameter_grid)
+
+ for i, parameters in (enumerate(iterator) if use_tqdm else iterator):
+ if use_tqdm:
+ # tqdm already provides the index
+ pass
+ else:
+ # Manual enumeration
+ i = i
+
+ self.logger.info(f"Running simulation {i+1}/{len(parameter_grid)}")
+ self.logger.debug(f"Parameters: {parameters}")
+
+ try:
+ result = self.run_single_simulation(model_id, parameters)
+ results.append(result)
+ if use_tqdm:
+ iterator.set_postfix({"status": "success"})
+ except Exception as e:
+ self.logger.error(f"Simulation {i+1} failed: {str(e)}")
+ # Create failed result
+ failed_result = SimulationResult(
+ status=SimulationStatus.FAILED,
+ parameters=parameters,
+ outputs={},
+ execution_time=0.0,
+ error_message=str(e)
+ )
+ results.append(failed_result)
+ if use_tqdm:
+ iterator.set_postfix({"status": "failed"})
+
+ successful = sum(1 for r in results if r.status == SimulationStatus.COMPLETED)
+ self.logger.info(f"Batch completed: {successful}/{len(results)} successful")
+
+ # Save successful results to database
+ if successful > 0:
+ try:
+ from db import store_simulation_results
+
+ # Convert successful SimulationResults to the format expected by store_simulation_results
+ successful_rows = []
+ param_keys = None
+
+ for result in results:
+ if result.status == SimulationStatus.COMPLETED:
+ result_row = {
+ **result.parameters, # Include all parameters
+ **result.outputs, # Include all outputs
+ '_ok': True, # Mark as successful
+ '_execution_time': result.execution_time
+ }
+ successful_rows.append(result_row)
+
+ # Use parameter keys from the first successful result
+ if param_keys is None:
+ param_keys = list(result.parameters.keys())
+
+ if successful_rows and param_keys:
+ store_simulation_results(model_id, successful_rows, param_keys)
+ self.logger.info(f"Saved {len(successful_rows)} results to database for model {model_id}")
+
+ except Exception as save_error:
+ self.logger.warning(f"Failed to save batch results to database: {save_error}")
+
+ return results
+
+ def import_model_from_github(
+ self,
+ github_url: str,
+ model_name: str,
+ description: str = "",
+ parameters: Dict[str, str] = None
+ ) -> str:
+ """Import a simulation model from GitHub."""
+ self.logger.info(f"Importing model from GitHub: {github_url}")
+
+ metadata = {
+ "description": description,
+ "source": "github",
+ "source_url": github_url,
+ "parameters": parameters or {},
+ "imported_at": time.time()
+ }
+
+ try:
+ model_id = self.facade.import_from_github(github_url, model_name, metadata)
+ self.logger.info(f"Successfully imported model with ID: {model_id}")
+ return model_id
+ except Exception as e:
+ self.logger.error(f"Failed to import model: {str(e)}")
+ raise
+
+ def get_model_info(self, model_id: str) -> Dict[str, Any]:
+ """Get information about a simulation model."""
+ try:
+ from db import get_simulation_path
+ script_path = get_simulation_path(model_id)
+
+ # Get metadata from database
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ models = db.simulation_repository.list({"id": model_id})
+ if not models:
+ raise ValueError(f"Model {model_id} not found")
+
+ model_info = models[0]
+ model_info["script_path"] = script_path
+
+ return model_info
+
+ except Exception as e:
+ self.logger.error(f"Failed to get model info: {str(e)}")
+ raise
+
+ def list_models(self) -> List[Dict[str, Any]]:
+ """List all available models."""
+ try:
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ return db.simulation_repository.list()
+
+ except Exception as e:
+ self.logger.error(f"Failed to list models: {str(e)}")
+ raise
+
+ def cleanup(self):
+ """Clean up service resources."""
+ self.facade.cleanup()
+
+
+class ReasoningService:
+ """High-level reasoning service."""
+
+ def __init__(self, config: ServiceConfiguration = None):
+ self.config = config or ServiceConfiguration()
+ self.logger = self._setup_logging()
+
+ def _setup_logging(self) -> logging.Logger:
+ """Set up logging for the service."""
+ logger = logging.getLogger("ReasoningService")
+ logger.setLevel(logging.INFO if self.config.enable_logging else logging.WARNING)
+
+ if not logger.handlers:
+ handler = logging.StreamHandler()
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ )
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ return logger
+
+ def ask_question(
+ self,
+ model_id: str,
+ question: str,
+ max_steps: int = 20
+ ) -> Dict[str, Any]:
+ """Ask a question about a simulation model."""
+ self.logger.info(f"Processing reasoning request for model {model_id}")
+
+ try:
+ from reasoning import ReasoningAgent
+ from db.config.database import DatabaseConfig
+
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ agent = ReasoningAgent(
+ model_id=model_id,
+ db_config=db_config,
+ max_steps=max_steps
+ )
+
+ result = agent.ask(question)
+
+ self.logger.info("Reasoning request completed successfully")
+ return {
+ "answer": result.answer,
+ "model_id": model_id,
+ "question": question,
+ "history": result.history,
+ "code_map": result.code_map,
+ "images": result.images
+ }
+
+ except Exception as e:
+ self.logger.error(f"Reasoning request failed: {str(e)}")
+ raise
+
+ def get_conversation_history(
+ self,
+ model_id: str,
+ limit: int = 50,
+ offset: int = 0
+ ) -> List[Dict[str, Any]]:
+ """Get conversation history for a model."""
+ try:
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ with db.config.get_sqlite_connection() as conn:
+ rows = conn.execute("""
+ SELECT id, model_id, question, answer, images, ts
+ FROM reasoning_agent
+ WHERE model_id = ?
+ ORDER BY ts DESC
+ LIMIT ? OFFSET ?
+ """, (model_id, limit, offset)).fetchall()
+
+ history = []
+ for row in rows:
+ history.append({
+ "id": row["id"],
+ "model_id": row["model_id"],
+ "question": row["question"],
+ "answer": row["answer"],
+ "images": row["images"],
+ "timestamp": row["ts"]
+ })
+
+ return history
+
+ except Exception as e:
+ self.logger.error(f"Failed to get conversation history: {str(e)}")
+ raise
+
+ def get_reasoning_statistics(self) -> Dict[str, Any]:
+ """Get statistics about reasoning usage."""
+ try:
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ with db.config.get_sqlite_connection() as conn:
+ # Overall stats
+ overall = conn.execute("""
+ SELECT
+ COUNT(*) as total_conversations,
+ COUNT(DISTINCT model_id) as unique_models,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ """).fetchone()
+
+ # Per-model stats
+ per_model = conn.execute("""
+ SELECT
+ model_id,
+ COUNT(*) as conversation_count,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ GROUP BY model_id
+ ORDER BY conversation_count DESC
+ """).fetchall()
+
+ return {
+ "overall": {
+ "total_conversations": overall["total_conversations"] if overall else 0,
+ "unique_models": overall["unique_models"] if overall else 0,
+ "first_conversation": overall["first_conversation"] if overall else None,
+ "last_conversation": overall["last_conversation"] if overall else None
+ },
+ "per_model": [
+ {
+ "model_id": row["model_id"],
+ "conversation_count": row["conversation_count"],
+ "first_conversation": row["first_conversation"],
+ "last_conversation": row["last_conversation"]
+ }
+ for row in per_model
+ ]
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to get reasoning statistics: {str(e)}")
+ raise
+
+
+class DataService:
+ """High-level data management service."""
+
+ def __init__(self, config: ServiceConfiguration = None):
+ self.config = config or ServiceConfiguration()
+ self.logger = self._setup_logging()
+
+ def _setup_logging(self) -> logging.Logger:
+ """Set up logging for the service."""
+ logger = logging.getLogger("DataService")
+ logger.setLevel(logging.INFO if self.config.enable_logging else logging.WARNING)
+
+ if not logger.handlers:
+ handler = logging.StreamHandler()
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ )
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ return logger
+
+ def get_simulation_results(
+ self,
+ model_id: Optional[str] = None,
+ limit: int = 100,
+ offset: int = 0
+ ) -> Dict[str, Any]:
+ """Get simulation results with pagination."""
+ try:
+ from db import Database, DatabaseConfig
+ import numpy as np
+ import pandas as pd
+
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ if model_id:
+ df = db.results_service.load_results(model_id=model_id)
+ else:
+ df = db.results_service.load_results()
+
+ # Apply pagination
+ total_count = len(df)
+ df_page = df.iloc[offset:offset + limit]
+
+ # Clean NaN values for JSON serialization
+ def clean_nan_values(obj):
+ """Recursively replace NaN values with None for JSON serialization."""
+ if isinstance(obj, dict):
+ return {k: clean_nan_values(v) for k, v in obj.items()}
+ elif isinstance(obj, list):
+ return [clean_nan_values(item) for item in obj]
+ elif isinstance(obj, (np.floating, float)) and np.isnan(obj):
+ return None
+ elif isinstance(obj, np.integer):
+ return int(obj)
+ elif isinstance(obj, np.floating):
+ return float(obj)
+ elif isinstance(obj, np.ndarray):
+ return clean_nan_values(obj.tolist())
+ else:
+ return obj
+
+ # Convert to records and clean NaN values
+ results = df_page.to_dict('records')
+ cleaned_results = clean_nan_values(results)
+
+ # Log preview of results
+ self.logger.info(f"=== RESULTS PREVIEW (First 5 rows) ===")
+ self.logger.info(f"Total count: {total_count}, Limit: {limit}, Offset: {offset}")
+ for i, result in enumerate(cleaned_results[:5]):
+ self.logger.info(f"Row {i+1}: {list(result.keys())}")
+ # Show key values for first few rows
+ if i < 3: # Show more details for first 3 rows
+ for key, value in list(result.items())[:10]: # First 10 keys
+ if isinstance(value, (list, tuple)) and len(value) > 5:
+ self.logger.info(f" {key}: {type(value).__name__} with {len(value)} items (first 3: {value[:3]})")
+ elif isinstance(value, dict):
+ self.logger.info(f" {key}: dict with {len(value)} keys")
+ else:
+ self.logger.info(f" {key}: {value}")
+ self.logger.info(f"=== END RESULTS PREVIEW ===")
+
+ return {
+ "total_count": total_count,
+ "limit": limit,
+ "offset": offset,
+ "results": cleaned_results
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to get simulation results: {str(e)}")
+ raise
+
+ def store_model(
+ self,
+ model_name: str,
+ script_content: str,
+ metadata: Dict[str, Any] = None
+ ) -> str:
+ """Store a new simulation model."""
+ try:
+ from db import store_simulation_script
+ import tempfile
+
+ # Create temporary script file
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+ f.write(script_content)
+ temp_path = f.name
+
+ try:
+ model_id = store_simulation_script(
+ model_name=model_name,
+ metadata=metadata or {},
+ script_path=temp_path
+ )
+
+ self.logger.info(f"Stored model {model_name} with ID: {model_id}")
+ return model_id
+
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+ except Exception as e:
+ self.logger.error(f"Failed to store model: {str(e)}")
+ raise
+
+ def delete_model(self, model_id: str) -> Dict[str, int]:
+ """Delete a model and all associated data."""
+ try:
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ with db.config.get_sqlite_connection() as conn:
+ # Delete associated data
+ results_deleted = conn.execute(
+ "DELETE FROM results WHERE model_id = ?",
+ (model_id,)
+ ).rowcount
+
+ reasoning_deleted = conn.execute(
+ "DELETE FROM reasoning_agent WHERE model_id = ?",
+ (model_id,)
+ ).rowcount
+
+ # Delete model
+ model_deleted = conn.execute(
+ "DELETE FROM simulations WHERE id = ?",
+ (model_id,)
+ ).rowcount
+
+ if model_deleted == 0:
+ raise ValueError(f"Model {model_id} not found")
+
+ self.logger.info(f"Deleted model {model_id} and associated data")
+
+ return {
+ "models_deleted": model_deleted,
+ "results_deleted": results_deleted,
+ "conversations_deleted": reasoning_deleted
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to delete model: {str(e)}")
+ raise
+
+ def get_database_statistics(self) -> Dict[str, Any]:
+ """Get comprehensive database statistics."""
+ try:
+ from db import Database, DatabaseConfig
+ db_config = DatabaseConfig(database_path=self.config.database_path)
+ db = Database(db_config)
+
+ with db.config.get_sqlite_connection() as conn:
+ # Model statistics
+ models_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_models,
+ MIN(created_at) as first_model,
+ MAX(created_at) as last_model
+ FROM simulations
+ """).fetchone()
+
+ # Results statistics
+ results_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_results,
+ COUNT(DISTINCT model_id) as models_with_results,
+ MIN(ts) as first_result,
+ MAX(ts) as last_result
+ FROM results
+ """).fetchone()
+
+ # Reasoning statistics
+ reasoning_stats = conn.execute("""
+ SELECT
+ COUNT(*) as total_conversations,
+ COUNT(DISTINCT model_id) as models_with_conversations,
+ MIN(ts) as first_conversation,
+ MAX(ts) as last_conversation
+ FROM reasoning_agent
+ """).fetchone()
+
+ # Database size
+ size_stats = conn.execute("PRAGMA page_count").fetchone()
+ page_size = conn.execute("PRAGMA page_size").fetchone()
+
+ db_size_bytes = (size_stats[0] if size_stats else 0) * (page_size[0] if page_size else 0)
+ db_size_mb = round(db_size_bytes / (1024 * 1024), 2)
+
+ return {
+ "database": {
+ "path": self.config.database_path,
+ "size_mb": db_size_mb
+ },
+ "models": {
+ "total": models_stats["total_models"] if models_stats else 0,
+ "first_created": models_stats["first_model"] if models_stats else None,
+ "last_created": models_stats["last_model"] if models_stats else None
+ },
+ "results": {
+ "total": results_stats["total_results"] if results_stats else 0,
+ "models_with_results": results_stats["models_with_results"] if results_stats else 0,
+ "first_result": results_stats["first_result"] if results_stats else None,
+ "last_result": results_stats["last_result"] if results_stats else None
+ },
+ "reasoning": {
+ "total_conversations": reasoning_stats["total_conversations"] if reasoning_stats else 0,
+ "models_with_conversations": reasoning_stats["models_with_conversations"] if reasoning_stats else 0,
+ "first_conversation": reasoning_stats["first_conversation"] if reasoning_stats else None,
+ "last_conversation": reasoning_stats["last_conversation"] if reasoning_stats else None
+ }
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to get database statistics: {str(e)}")
+ raise
+
+ def create_backup(self) -> Dict[str, Any]:
+ """Create a database backup."""
+ try:
+ import shutil
+ from datetime import datetime
+
+ # Create backup filename with timestamp
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ backup_path = f"{self.config.database_path}.backup_{timestamp}"
+
+ # Copy database file
+ shutil.copy2(self.config.database_path, backup_path)
+
+ # Get backup file size
+ backup_size = Path(backup_path).stat().st_size
+ backup_size_mb = round(backup_size / (1024 * 1024), 2)
+
+ self.logger.info(f"Created database backup: {backup_path}")
+
+ return {
+ "backup_path": backup_path,
+ "backup_size_mb": backup_size_mb,
+ "timestamp": timestamp
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to create backup: {str(e)}")
+ raise
diff --git a/modules/research-framework/simexr_mod/db/__init__.py b/modules/research-framework/simexr_mod/db/__init__.py
new file mode 100644
index 0000000..a8c43e4
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/__init__.py
@@ -0,0 +1,174 @@
+# db/__init__.py
+import sqlite3
+from pathlib import Path
+from typing import Optional
+
+from .config.database import DatabaseConfig
+from .services.store import StorageService
+from .services.results import ResultsService
+from .repositories.simulation import SimulationRepository
+
+
+class Database:
+ def __init__(self, config: Optional[DatabaseConfig] = None):
+ self.config = config or DatabaseConfig()
+ self.setup_database()
+
+ # Initialize repositories and services
+ self.simulation_repository = SimulationRepository(self.config)
+ self.storage_service = StorageService(self.simulation_repository)
+ self.results_service = ResultsService(self.simulation_repository)
+
+ def _conn(self, db_path: str | Path = None) -> sqlite3.Connection:
+ if db_path is None:
+ db_path = self.config.database_path
+ conn = sqlite3.connect(str(db_path))
+ conn.row_factory = sqlite3.Row
+ return conn
+
+ def setup_database(self):
+ """Initialize database tables"""
+ with self.config.get_sqlite_connection() as conn:
+ conn.execute("""
+ CREATE TABLE IF NOT EXISTS simulations
+ (
+ id
+ TEXT
+ PRIMARY
+ KEY,
+ name
+ TEXT
+ NOT
+ NULL,
+ metadata
+ TEXT,
+ script_path
+ TEXT
+ NOT
+ NULL,
+ created_at
+ TIMESTAMP
+ DEFAULT
+ CURRENT_TIMESTAMP
+ );
+ """)
+
+ conn.execute("""
+ CREATE TABLE IF NOT EXISTS simulation_results
+ (
+ id
+ INTEGER
+ PRIMARY
+ KEY
+ AUTOINCREMENT,
+ model_id
+ TEXT
+ NOT
+ NULL,
+ params
+ TEXT
+ NOT
+ NULL,
+ results
+ TEXT
+ NOT
+ NULL,
+ created_at
+ TIMESTAMP
+ DEFAULT
+ CURRENT_TIMESTAMP,
+ FOREIGN
+ KEY
+ (
+ model_id
+ ) REFERENCES simulations
+ (
+ id
+ )
+ );
+ """)
+
+ conn.execute("""
+ CREATE TABLE IF NOT EXISTS reasoning_agent
+ (
+ id
+ INTEGER
+ PRIMARY
+ KEY
+ AUTOINCREMENT,
+ model_id
+ TEXT
+ NOT
+ NULL,
+ question
+ TEXT
+ NOT
+ NULL,
+ answer
+ TEXT
+ NOT
+ NULL,
+ images
+ TEXT,
+ ts
+ TIMESTAMP
+ DEFAULT
+ CURRENT_TIMESTAMP,
+ FOREIGN
+ KEY
+ (
+ model_id
+ ) REFERENCES simulations
+ (
+ id
+ )
+ );
+ """)
+
+ @classmethod
+ def create_default(cls) -> 'Database':
+ """Create a database instance with default configuration"""
+ config = DatabaseConfig(
+ dialect="sqlite",
+ database_path=str(Path(__file__).parent.parent / "mcp.db")
+ )
+ return cls(config)
+
+
+# Create a default database instance
+default_db = Database.create_default()
+
+# Export functions for backward compatibility
+def store_simulation_script(model_name: str, metadata: dict, script_path: str) -> str:
+ """Store a simulation script and return model ID."""
+ return default_db.storage_service.store_simulation_script(model_name, metadata, script_path)
+
+def get_simulation_path(model_id: str, db_path: str = None) -> str:
+ """Get the path to a simulation script."""
+ return default_db.simulation_repository.get_simulation_path(model_id)
+
+def store_simulation_results(model_id: str, rows: list, param_keys: list, db_path: str = None) -> None:
+ """Store simulation results."""
+ default_db.simulation_repository.store_simulation_results(model_id, rows, param_keys)
+
+def store_report(model_id: str, question: str, answer: str, images: list) -> None:
+ """Store a reasoning report."""
+ from .services.reasoning import ReasoningService
+ service = ReasoningService()
+ service.store_report(model_id, question, answer, images)
+
+__all__ = [
+ # Classes
+ 'Database',
+ 'DatabaseConfig',
+ 'StorageService',
+ 'ResultsService',
+ 'SimulationRepository',
+ 'default_db',
+
+ # Compatibility functions
+ 'store_simulation_script',
+ 'get_simulation_path',
+ 'store_simulation_results',
+ 'store_report'
+]
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/base.py b/modules/research-framework/simexr_mod/db/base.py
new file mode 100644
index 0000000..c17fc4f
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/base.py
@@ -0,0 +1,117 @@
+"""
+Base classes for the db module to provide common interfaces and inheritance.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Protocol
+from pathlib import Path
+
+
+class DatabaseConfigProtocol(Protocol):
+ """Protocol for database configuration implementations."""
+
+ database_path: str
+
+ def get_sqlite_connection(self):
+ """Get a SQLite connection context manager."""
+ ...
+
+
+class RepositoryProtocol(Protocol):
+ """Protocol for repository implementations."""
+
+ def get(self, id: Any) -> Optional[Any]:
+ """Get an entity by ID."""
+ ...
+
+ def list(self, filters: Dict[str, Any] = None) -> List[Any]:
+ """List entities with optional filters."""
+ ...
+
+
+class BaseService(ABC):
+ """Base class for all service implementations."""
+
+ def __init__(self, repository=None):
+ self.repository = repository
+
+ @abstractmethod
+ def _validate_inputs(self, **kwargs) -> bool:
+ """Validate service method inputs."""
+ pass
+
+
+class BaseStorageService(BaseService):
+ """Base class for storage service implementations."""
+
+ @abstractmethod
+ def store(self, data: Dict[str, Any]) -> str:
+ """Store data and return identifier."""
+ pass
+
+ @abstractmethod
+ def retrieve(self, identifier: str) -> Dict[str, Any]:
+ """Retrieve data by identifier."""
+ pass
+
+
+class BaseResultsService(BaseService):
+ """Base class for results service implementations."""
+
+ @abstractmethod
+ def load_results(self, **kwargs) -> Any:
+ """Load results with filtering options."""
+ pass
+
+ @abstractmethod
+ def store_results(self, results: List[Dict[str, Any]], **kwargs) -> None:
+ """Store results data."""
+ pass
+
+
+class BaseRepository(ABC):
+ """Base class for all repository implementations."""
+
+ def __init__(self, db_config=None):
+ self.db_config = db_config
+
+ @abstractmethod
+ def get(self, id: Any) -> Optional[Any]:
+ """Get an entity by ID."""
+ pass
+
+ @abstractmethod
+ def list(self, filters: Dict[str, Any] = None) -> List[Any]:
+ """List entities with optional filters."""
+ pass
+
+ def _validate_id(self, id: Any) -> bool:
+ """Validate entity ID format."""
+ return id is not None and str(id).strip()
+
+
+class BaseModel(ABC):
+ """Base class for all model implementations."""
+
+ @abstractmethod
+ def to_dict(self) -> Dict[str, Any]:
+ """Convert model to dictionary representation."""
+ pass
+
+ @classmethod
+ @abstractmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'BaseModel':
+ """Create model instance from dictionary."""
+ pass
+
+
+class DatabaseManagerProtocol(Protocol):
+ """Protocol for database manager implementations."""
+
+ def setup_database(self) -> None:
+ """Initialize database schema."""
+ ...
+
+ def get_connection(self) -> Any:
+ """Get database connection."""
+ ...
diff --git a/modules/research-framework/simexr_mod/db/config/__init__.py b/modules/research-framework/simexr_mod/db/config/__init__.py
new file mode 100644
index 0000000..0db438d
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/config/__init__.py
@@ -0,0 +1,5 @@
+"""Configuration module for database connections."""
+
+from .database import DatabaseConfig
+
+__all__ = ["DatabaseConfig"]
diff --git a/modules/research-framework/simexr_mod/db/config/database.py b/modules/research-framework/simexr_mod/db/config/database.py
new file mode 100644
index 0000000..a5b073a
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/config/database.py
@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+from typing import Optional
+from contextlib import contextmanager
+import sqlite3
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.engine import Engine
+
+@dataclass
+class DatabaseConfig:
+ dialect: str = "sqlite"
+ database_path: str = "mcp.db"
+ echo: bool = False
+ host: Optional[str] = None
+ port: Optional[int] = None
+ username: Optional[str] = None
+ password: Optional[str] = None
+ _engine: Optional[Engine] = None
+ _session_factory: Optional[sessionmaker] = None
+
+ @property
+ def connection_string(self) -> str:
+ if self.dialect == "sqlite":
+ return f"sqlite:///{self.database_path}"
+ elif self.dialect == "postgresql":
+ return f"postgresql://{self.username}:{self.password}@{self.host}:{self.port}/{self.database_path}"
+ raise ValueError(f"Unsupported dialect: {self.dialect}")
+
+ def get_engine(self) -> Engine:
+ if self._engine is None:
+ self._engine = create_engine(self.connection_string, echo=self.echo)
+ return self._engine
+
+ def get_session_factory(self) -> sessionmaker:
+ if self._session_factory is None:
+ self._session_factory = sessionmaker(bind=self.get_engine())
+ return self._session_factory
+
+ @contextmanager
+ def get_session(self) -> Session:
+ session = self.get_session_factory()()
+ try:
+ yield session
+ session.commit()
+ except Exception:
+ session.rollback()
+ raise
+ finally:
+ session.close()
+
+ @contextmanager
+ def get_sqlite_connection(self) -> sqlite3.Connection:
+ """Get a SQLite connection with row factory set to dict-like rows"""
+ conn = sqlite3.connect(str(self.database_path))
+ conn.row_factory = sqlite3.Row
+ try:
+ yield conn
+ conn.commit()
+ except Exception:
+ conn.rollback()
+ raise
+ finally:
+ conn.close()
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/models/__init__.py b/modules/research-framework/simexr_mod/db/models/__init__.py
new file mode 100644
index 0000000..d0982b7
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/models/__init__.py
@@ -0,0 +1,6 @@
+"""Model classes for database entities."""
+
+from .base import BaseModel
+from .simulation import SimulationResult
+
+__all__ = ["BaseModel", "SimulationResult"]
diff --git a/modules/research-framework/simexr_mod/db/models/base.py b/modules/research-framework/simexr_mod/db/models/base.py
new file mode 100644
index 0000000..95f72f4
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/models/base.py
@@ -0,0 +1,34 @@
+# db/models/base.py
+
+from datetime import datetime
+from typing import Dict, Any
+from ..base import BaseModel as AbstractBaseModel
+
+
+class BaseModel(AbstractBaseModel):
+ """Base model class with common fields and methods."""
+
+ def __init__(self):
+ self.created_at = datetime.utcnow()
+ self.updated_at = datetime.utcnow()
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Convert model to dictionary representation."""
+ return {
+ "created_at": self.created_at.isoformat() if self.created_at else None,
+ "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'BaseModel':
+ """Create model instance from dictionary."""
+ instance = cls()
+ if "created_at" in data and data["created_at"]:
+ instance.created_at = datetime.fromisoformat(data["created_at"].replace('Z', '+00:00'))
+ if "updated_at" in data and data["updated_at"]:
+ instance.updated_at = datetime.fromisoformat(data["updated_at"].replace('Z', '+00:00'))
+ return instance
+
+ def update_timestamp(self):
+ """Update the updated_at timestamp."""
+ self.updated_at = datetime.utcnow()
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/models/simulation.py b/modules/research-framework/simexr_mod/db/models/simulation.py
new file mode 100644
index 0000000..9e6c21d
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/models/simulation.py
@@ -0,0 +1,43 @@
+# db/models/simulation.py
+
+from datetime import datetime
+from typing import Dict, Any, List, Optional
+from .base import BaseModel
+
+class SimulationResult(BaseModel):
+ def __init__(self, model_id: str, params: Dict[str, Any], results: Dict[str, Any]):
+ self.model_id = model_id
+ self.params = params
+ self.results = results
+ self.created_at = datetime.utcnow()
+ self.success = True
+ self.error_message: Optional[str] = None
+
+ def to_dict(self) -> Dict[str, Any]:
+ base_dict = super().to_dict()
+ base_dict.update({
+ "model_id": self.model_id,
+ "params": self.params,
+ "results": self.results,
+ "success": self.success,
+ "error_message": self.error_message
+ })
+ return base_dict
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'SimulationResult':
+ """Create SimulationResult from dictionary."""
+ instance = cls(
+ model_id=data["model_id"],
+ params=data.get("params", {}),
+ results=data.get("results", {})
+ )
+ instance.success = data.get("success", True)
+ instance.error_message = data.get("error_message")
+
+ # Handle timestamp parsing
+ if "created_at" in data and data["created_at"]:
+ if isinstance(data["created_at"], str):
+ instance.created_at = datetime.fromisoformat(data["created_at"].replace('Z', '+00:00'))
+
+ return instance
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/repositories/__init__.py b/modules/research-framework/simexr_mod/db/repositories/__init__.py
new file mode 100644
index 0000000..e935b54
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/repositories/__init__.py
@@ -0,0 +1,7 @@
+"""Repository classes for data access."""
+
+from .base import BaseRepository
+from .reasoning import ReasoningRepository
+from .simulation import SimulationRepository
+
+__all__ = ["BaseRepository", "ReasoningRepository", "SimulationRepository"]
diff --git a/modules/research-framework/simexr_mod/db/repositories/base.py b/modules/research-framework/simexr_mod/db/repositories/base.py
new file mode 100644
index 0000000..60ae0a6
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/repositories/base.py
@@ -0,0 +1,43 @@
+# db/repositories/base.py
+
+from typing import Any, Dict, List, Optional
+from sqlalchemy.orm import Session
+from ..base import BaseRepository as AbstractBaseRepository
+from ..models.base import BaseModel
+
+
+class BaseRepository(AbstractBaseRepository):
+ """SQLAlchemy-based repository implementation."""
+
+ def __init__(self, session: Session = None, db_config=None):
+ super().__init__(db_config=db_config)
+ self.session = session
+
+ def get(self, id: Any) -> Optional[BaseModel]:
+ """Get entity by ID - to be implemented by subclasses."""
+ if not self._validate_id(id):
+ return None
+ raise NotImplementedError("Subclasses must implement get method")
+
+ def list(self, filters: Dict[str, Any] = None) -> List[BaseModel]:
+ """List entities with filters - to be implemented by subclasses."""
+ raise NotImplementedError("Subclasses must implement list method")
+
+ def create(self, model: BaseModel) -> BaseModel:
+ """Create new entity - to be implemented by subclasses."""
+ if not isinstance(model, BaseModel):
+ raise ValueError("Model must be an instance of BaseModel")
+ raise NotImplementedError("Subclasses must implement create method")
+
+ def update(self, model: BaseModel) -> BaseModel:
+ """Update existing entity - to be implemented by subclasses."""
+ if not isinstance(model, BaseModel):
+ raise ValueError("Model must be an instance of BaseModel")
+ model.update_timestamp()
+ raise NotImplementedError("Subclasses must implement update method")
+
+ def delete(self, id: Any) -> None:
+ """Delete entity by ID - to be implemented by subclasses."""
+ if not self._validate_id(id):
+ raise ValueError("Invalid ID provided")
+ raise NotImplementedError("Subclasses must implement delete method")
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/repositories/reasoning.py b/modules/research-framework/simexr_mod/db/repositories/reasoning.py
new file mode 100644
index 0000000..fd117e9
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/repositories/reasoning.py
@@ -0,0 +1,24 @@
+
+from pathlib import Path
+from typing import List
+import json
+from ..config.database import DatabaseConfig
+
+class ReasoningRepository:
+ def __init__(self, db_config: DatabaseConfig = None):
+ self.db_config = db_config or DatabaseConfig()
+
+ def store_report(self, model_id: str, question: str, answer: str, image_paths: List[str]) -> None:
+ """
+ Insert a reasoning report into the `reasoning_agent` table.
+ """
+ with self.db_config.get_sqlite_connection() as conn:
+ conn.execute("""
+ INSERT INTO reasoning_agent (model_id, question, answer, images)
+ VALUES (?, ?, ?, ?)
+ """, (
+ model_id,
+ question,
+ answer,
+ json.dumps(image_paths, ensure_ascii=False),
+ ))
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/repositories/simulation.py b/modules/research-framework/simexr_mod/db/repositories/simulation.py
new file mode 100644
index 0000000..f62df8a
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/repositories/simulation.py
@@ -0,0 +1,114 @@
+from pathlib import Path
+from typing import List, Dict, Optional, Any
+import json
+from ..config.database import DatabaseConfig
+from ..base import BaseRepository as AbstractBaseRepository
+
+
+class SimulationRepository(AbstractBaseRepository):
+ def __init__(self, db_config: DatabaseConfig = None):
+ config = db_config or DatabaseConfig()
+ super().__init__(db_config=config)
+ # For backward compatibility
+ self.db_config = config
+
+ def get_simulation_path(self, model_id: str) -> str:
+ """
+ Return the absolute path to `simulate.py` for the given model_id.
+
+ Args:
+ model_id: The ID of the model to find
+
+ Returns:
+ str: Path to the simulation script
+
+ Raises:
+ KeyError: If the model_id is unknown
+ """
+ with self.db_config.get_sqlite_connection() as conn:
+ row = conn.execute(
+ "SELECT script_path FROM simulations WHERE id = ?",
+ (model_id,)
+ ).fetchone()
+
+ if row is None:
+ raise KeyError(f"model_id '{model_id}' not found in DB {self.db_config.database_path}")
+
+ return row["script_path"]
+
+ def store_simulation_results(self, model_id: str, rows: List[dict], param_keys: List[str]) -> None:
+ """
+ Store simulation results in the results table.
+
+ Args:
+ model_id: The ID of the model
+ rows: List of result dictionaries from simulation runs
+ param_keys: List of parameter names used in the simulation
+ """
+ with self.db_config.get_sqlite_connection() as conn:
+ for row in rows:
+ # Split data into params and outputs
+ params = self._extract_parameters(row, param_keys)
+ outputs = self._extract_results(row, param_keys)
+
+ conn.execute("""
+ INSERT INTO results (model_id, params, outputs, ts)
+ VALUES (?, ?, ?, CURRENT_TIMESTAMP)
+ """, (
+ model_id,
+ params,
+ outputs
+ ))
+
+ @staticmethod
+ def _extract_parameters(row: dict, param_keys: List[str]) -> str:
+ """Extract and serialize parameters from result row"""
+ params = {k: row[k] for k in param_keys if k in row}
+ # Include any special fields that start with underscore
+ params.update({
+ k: v for k, v in row.items()
+ if k.startswith('_') and k not in ('_ok', '_error_msg', '_error_type')
+ })
+ return json.dumps(params)
+
+ @staticmethod
+ def _extract_results(row: dict, param_keys: List[str]) -> str:
+ """Extract and serialize results, excluding parameters and special fields"""
+ results = {
+ k: v for k, v in row.items()
+ if not k.startswith('_') and k not in param_keys
+ }
+ # Include error information if present
+ if not row.get('_ok', True):
+ results['error'] = {
+ 'type': row.get('_error_type', ''),
+ 'message': row.get('_error_msg', '')
+ }
+ return json.dumps(results)
+
+ # Implement abstract methods from BaseRepository
+ def get(self, id: Any) -> Optional[Any]:
+ """Get simulation by ID."""
+ try:
+ return self.get_simulation_path(str(id))
+ except KeyError:
+ return None
+
+ def list(self, filters: Dict[str, Any] = None) -> List[Any]:
+ """List simulations with optional filters."""
+ with self.db_config.get_sqlite_connection() as conn:
+ query = "SELECT id, name, metadata, script_path FROM simulations"
+ params = []
+
+ if filters:
+ where_clauses = []
+ for key, value in filters.items():
+ if key in ['id', 'name']:
+ where_clauses.append(f"{key} = ?")
+ params.append(value)
+
+ if where_clauses:
+ query += " WHERE " + " AND ".join(where_clauses)
+
+ rows = conn.execute(query, params).fetchall()
+ return [dict(row) for row in rows]
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/services/__init__.py b/modules/research-framework/simexr_mod/db/services/__init__.py
new file mode 100644
index 0000000..0015caf
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/services/__init__.py
@@ -0,0 +1,7 @@
+"""Service classes for business logic."""
+
+from .reasoning import ReasoningService
+from .results import ResultsService
+from .store import StorageService
+
+__all__ = ["ReasoningService", "ResultsService", "StorageService"]
diff --git a/modules/research-framework/simexr_mod/db/services/reasoning.py b/modules/research-framework/simexr_mod/db/services/reasoning.py
new file mode 100644
index 0000000..9f78fd3
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/services/reasoning.py
@@ -0,0 +1,29 @@
+from typing import List
+from pathlib import Path
+import logging
+
+from db.repositories.reasoning import ReasoningRepository
+
+log = logging.getLogger(__name__)
+
+
+class ReasoningService:
+ def __init__(self, repository: ReasoningRepository = None):
+ self.repository = repository or ReasoningRepository()
+
+ def store_report(self, model_id: str, question: str, answer: str, image_paths: List[str]) -> None:
+ """
+ Store a reasoning report with associated images.
+
+ Args:
+ model_id: The ID of the model
+ question: The question asked
+ answer: The answer provided
+ image_paths: List of paths to associated images
+ """
+ try:
+ self.repository.store_report(model_id, question, answer, image_paths)
+ log.info("Stored reasoning report for model %s", model_id)
+ except Exception as e:
+ log.error("Failed to store reasoning report for model %s: %s", model_id, str(e))
+ raise
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/services/results.py b/modules/research-framework/simexr_mod/db/services/results.py
new file mode 100644
index 0000000..9c33d5a
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/services/results.py
@@ -0,0 +1,147 @@
+import datetime
+import json
+import sqlite3
+from pathlib import Path
+from typing import Dict, Any, List
+import pandas as pd
+
+from ..repositories.simulation import SimulationRepository
+from ..utils.json_utils import _safe_parse
+from ..utils.transform_utils import _explode_row
+from ..base import BaseResultsService
+
+# Import sanitize_metadata with fallback
+try:
+ from core.script_utils import sanitize_metadata
+except ImportError:
+ def sanitize_metadata(data, media_dir, media_paths, prefix=""):
+ return data
+
+
+class ResultsService(BaseResultsService):
+ def __init__(self, simulation_repo: SimulationRepository):
+ super().__init__(repository=simulation_repo)
+ self.simulation_repo = simulation_repo
+
+ def load_results(self,
+ db_path: str | Path = "mcp.db",
+ model_id: str | None = None,
+ ) -> pd.DataFrame:
+ """
+ Load the `results` table, parse JSON cols, and EXPLODE any list/array fields
+ (from params or outputs) into rowwise records.
+
+ Returned columns:
+ model_id, ts, step, <param fields>, <output fields>
+
+ Notes:
+ - If multiple array fields have different lengths in a run, scalars are broadcast
+ and shorter arrays are padded with None to match the longest length.
+ - If a run has no arrays at all, it's kept as a single row with step=0.
+ """
+ con = sqlite3.connect(str(db_path))
+ try:
+ raw_df = pd.read_sql("SELECT model_id, params, outputs, ts FROM results", con)
+ finally:
+ con.close()
+
+ if model_id is not None:
+ raw_df = raw_df[raw_df["model_id"] == model_id].reset_index(drop=True)
+
+ # Parse JSON
+ raw_df["params"] = raw_df["params"].apply(_safe_parse)
+ raw_df["outputs"] = raw_df["outputs"].apply(_safe_parse)
+
+ # Drop unparseable rows
+ raw_df = raw_df[raw_df["params"].apply(bool) & raw_df["outputs"].apply(bool)].reset_index(drop=True)
+
+ # Explode each row based on list-like fields
+ exploded_frames: list[pd.DataFrame] = []
+ for _, r in raw_df.iterrows():
+ exploded = _explode_row(
+ model_id=r["model_id"],
+ ts=r["ts"],
+ params=r["params"],
+ outputs=r["outputs"],
+ )
+ exploded_frames.append(exploded)
+
+ if not exploded_frames:
+ return pd.DataFrame(columns=["model_id", "ts", "step"])
+
+ final = pd.concat(exploded_frames, ignore_index=True)
+
+ # Optional: stable column ordering → id/timestamps first, then others (params before outputs if you want)
+ # Already merged; if you need specific ordering, you can sort keys or provide a custom order here.
+
+ return final
+
+ def store_simulation_results(
+ self,
+ model_id: str,
+ rows: List[Dict[str, Any]],
+ param_keys: List[str] | None = None,
+ db_path: str | Path = None,
+ ) -> None:
+ """
+ Persist experiment result rows with sanitized outputs and saved media.
+
+ DB schema:
+ - id: autoincrement
+ - model_id: FK
+ - ts: timestamp
+ - params: input params (JSON)
+ - outputs: returned result (JSON)
+ - media_paths: separate media (e.g. figures, animations)
+ """
+
+ if db_path is None:
+ db_path = self.simulation_repo.db_config.database_path
+
+ if param_keys is None and rows:
+ param_keys = [k for k in rows[0].keys() if k != "error"]
+
+ media_root = Path("results_media") / model_id
+ media_root.mkdir(parents=True, exist_ok=True)
+
+ with self.simulation_repo.db_config.get_sqlite_connection() as c:
+ ts_now = datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z"
+
+ for idx, row in enumerate(rows):
+ params = {k: row[k] for k in param_keys if k in row}
+ outputs = {k: v for k, v in row.items() if k not in params}
+
+ media_paths: List[str] = []
+ sanitized_outputs = sanitize_metadata(outputs, media_root, media_paths, prefix=f"row{idx}")
+
+ c.execute(
+ "INSERT INTO results (model_id, ts, params, outputs, media_paths) VALUES (?,?,?,?,?)",
+ (
+ model_id,
+ ts_now,
+ json.dumps(params, ensure_ascii=False),
+ json.dumps(sanitized_outputs, ensure_ascii=False),
+ json.dumps(media_paths, ensure_ascii=False),
+ ),
+ )
+
+ # Implement abstract methods from BaseResultsService
+ def load_results_base(self, **kwargs):
+ """Load results with filtering options (base implementation)."""
+ return super().load_results(
+ db_path=kwargs.get("db_path", "mcp.db"),
+ model_id=kwargs.get("model_id")
+ )
+
+ def store_results(self, results: List[Dict[str, Any]], **kwargs) -> None:
+ """Store results data."""
+ self.store_simulation_results(
+ model_id=kwargs["model_id"],
+ rows=results,
+ param_keys=kwargs.get("param_keys"),
+ db_path=kwargs.get("db_path")
+ )
+
+ def _validate_inputs(self, **kwargs) -> bool:
+ """Validate inputs for results operations."""
+ return "model_id" in kwargs and kwargs["model_id"]
diff --git a/modules/research-framework/simexr_mod/db/services/store.py b/modules/research-framework/simexr_mod/db/services/store.py
new file mode 100644
index 0000000..352ae82
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/services/store.py
@@ -0,0 +1,127 @@
+# db/services/store.py
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Dict, Any, List, Union
+
+from ..config.database import DatabaseConfig
+from ..repositories.simulation import SimulationRepository
+from ..base import BaseStorageService
+
+class StorageService(BaseStorageService):
+ def __init__(self, simulation_repo: SimulationRepository):
+ super().__init__(repository=simulation_repo)
+ self.simulation_repo = simulation_repo
+
+ def store_simulation_script(
+ self,
+ model_name: str,
+ metadata: Dict[str, Any],
+ script_path: Union[str, Path],
+ db_path: Union[str, Path] = None,
+ ) -> str:
+ """
+ Store a simulation script entry if it doesn't already exist.
+ Returns a unique model_id derived from model name + script content.
+ """
+ if db_path is None:
+ db_path = self.simulation_repo.db_config.database_path
+
+ script_path = str(script_path)
+ from ..utils.hash_utils import generate_model_id
+ model_id = generate_model_id(model_name, script_path)
+
+ with self.simulation_repo.db_config.get_sqlite_connection() as c:
+ # Ensure table exists
+ c.execute(
+ """CREATE TABLE IF NOT EXISTS simulations
+ (
+ id
+ TEXT
+ PRIMARY
+ KEY,
+ name
+ TEXT,
+ metadata
+ TEXT,
+ script_path
+ TEXT,
+ media_paths
+ TEXT
+ )"""
+ )
+
+ # Check if the exact model_id already exists
+ existing = c.execute(
+ "SELECT id FROM simulations WHERE id = ?", (model_id,)
+ ).fetchone()
+
+ if existing:
+ print(f"[✓] Simulation already exists: {model_id}")
+ return model_id
+
+ # Insert new unique entry
+ c.execute(
+ """INSERT INTO simulations (id, name, metadata, script_path)
+ VALUES (?, ?, ?, ?)""",
+ (model_id, model_name, json.dumps(metadata), script_path),
+ )
+ print(f"[+] Stored new simulation: {model_id}")
+
+ return model_id
+
+ def get_model_metadata(self, model_id: str, db_path: str | Path = None) -> dict:
+ if db_path is None:
+ db_path = self.simulation_repo.db_config.database_path
+ with self.simulation_repo.db_config.get_sqlite_connection() as c:
+ row = c.execute(
+ "SELECT metadata FROM simulations WHERE id = ?", (model_id,)
+ )
+ row = row.fetchone()
+ if row is None:
+ raise ValueError(f"No metadata found for model_id={model_id}")
+ return json.loads(row["metadata"])
+
+ def get_simulation_script_code(self, model_id: str, db_path: str = None) -> str:
+ """
+ Fetch the saved path for this model_id, read that file,
+ dedent it, and return the actual Python code as a string.
+ """
+ if db_path is None:
+ db_path = self.simulation_repo.db_config.database_path
+
+ import textwrap
+ with self.simulation_repo.db_config.get_sqlite_connection() as conn:
+ row = conn.execute("SELECT script_path FROM simulations WHERE id = ?", (model_id,)).fetchone()
+
+ if not row:
+ raise ValueError(f"No script found for model_id={model_id!r}")
+
+ script_path = row[0]
+ code = Path(script_path).read_text(encoding="utf-8")
+ return textwrap.dedent(code)
+
+ # Implement abstract methods from BaseStorageService
+ def store(self, data: Dict[str, Any]) -> str:
+ """Store simulation data and return model ID."""
+ return self.store_simulation_script(
+ model_name=data["name"],
+ metadata=data.get("metadata", {}),
+ script_path=data["script_path"]
+ )
+
+ def retrieve(self, identifier: str) -> Dict[str, Any]:
+ """Retrieve simulation data by model ID."""
+ metadata = self.get_model_metadata(identifier)
+ script_code = self.get_simulation_script_code(identifier)
+ return {
+ "id": identifier,
+ "metadata": metadata,
+ "script_code": script_code
+ }
+
+ def _validate_inputs(self, **kwargs) -> bool:
+ """Validate inputs for storage operations."""
+ required_fields = ["model_name", "script_path"]
+ return all(field in kwargs and kwargs[field] for field in required_fields)
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/utils/__init__.py b/modules/research-framework/simexr_mod/db/utils/__init__.py
new file mode 100644
index 0000000..e45f605
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/utils/__init__.py
@@ -0,0 +1,7 @@
+"""Utility modules for database operations."""
+
+from .hash_utils import generate_model_id
+from .json_utils import _safe_parse
+from .transform_utils import _explode_row, _is_listy, _to_list
+
+__all__ = ["generate_model_id", "_safe_parse", "_explode_row", "_is_listy", "_to_list"]
diff --git a/modules/research-framework/simexr_mod/db/utils/hash_utils.py b/modules/research-framework/simexr_mod/db/utils/hash_utils.py
new file mode 100644
index 0000000..ea9b3c1
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/utils/hash_utils.py
@@ -0,0 +1,24 @@
+import hashlib
+from pathlib import Path
+from typing import Union
+
+HASH_LENGTH = 12
+
+def generate_model_id(model_name: str, model_script_path: Union[str, Path]) -> str:
+ """
+ Generate a unique model identifier by combining the model name with a content hash.
+
+ Args:
+ model_name: Name of the machine learning model
+ model_script_path: Path to the model's script file
+
+ Returns:
+ str: Combined identifier in format 'model_name_contenthash'
+ """
+ def calculate_content_hash(file_content: str) -> str:
+ return hashlib.sha1(file_content.encode()).hexdigest()[:HASH_LENGTH]
+
+ script_content = Path(model_script_path).read_text()
+ content_hash = calculate_content_hash(script_content)
+
+ return f"{model_name}_{content_hash}"
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/db/utils/json_utils.py b/modules/research-framework/simexr_mod/db/utils/json_utils.py
new file mode 100644
index 0000000..a986e05
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/utils/json_utils.py
@@ -0,0 +1,78 @@
+import datetime
+import json
+from pathlib import Path
+from typing import List, Dict, Any
+import pandas as pd
+import numpy as np
+
+
+def _safe_parse(x: Any) -> dict:
+ if isinstance(x, dict):
+ return x
+ if x is None or (isinstance(x, float) and pd.isna(x)):
+ return {}
+ try:
+ return json.loads(x)
+ except Exception:
+ return {}
+
+def _is_listy(v: Any) -> bool:
+ # Treat numpy arrays, lists, tuples, Series as list-like
+ return isinstance(v, (list, tuple, np.ndarray, pd.Series))
+
+
+def _to_list(v: Any) -> list:
+ if isinstance(v, np.ndarray):
+ return v.tolist()
+ if isinstance(v, pd.Series):
+ return v.to_list()
+ if isinstance(v, (list, tuple)):
+ return list(v)
+ # scalar → list of one
+ return [v]
+
+# def store_simulation_results(
+# model_id: str,
+# rows: List[Dict[str, Any]],
+# param_keys: List[str] | None = None,
+# db_path: str | Path = DB_DEFAULT,
+# ) -> None:
+# """
+# Persist experiment result rows with sanitized outputs and saved media.
+#
+# DB schema:
+# - id: autoincrement
+# - model_id: FK
+# - ts: timestamp
+# - params: input params (JSON)
+# - outputs: returned result (JSON)
+# - media_paths: separate media (e.g. figures, animations)
+# """
+#
+# if param_keys is None and rows:
+# param_keys = [k for k in rows[0].keys() if k != "error"]
+#
+# media_root = Path("results_media") / model_id
+# media_root.mkdir(parents=True, exist_ok=True)
+#
+# with _conn(db_path) as c:
+#
+# ts_now = datetime.utcnow().isoformat(timespec="seconds") + "Z"
+#
+# for idx, row in enumerate(rows):
+# params = {k: row[k] for k in param_keys if k in row}
+# outputs = {k: v for k, v in row.items() if k not in params}
+#
+# media_paths: List[str] = []
+# sanitized_outputs = sanitize_metadata(outputs, media_root, media_paths, prefix=f"row{idx}")
+#
+# c.execute(
+# "INSERT INTO results (model_id, ts, params, outputs, media_paths) VALUES (?,?,?,?,?)",
+# (
+# model_id,
+# ts_now,
+# json.dumps(params, ensure_ascii=False),
+# json.dumps(sanitized_outputs, ensure_ascii=False),
+# json.dumps(media_paths, ensure_ascii=False),
+# ),
+# )
diff --git a/modules/research-framework/simexr_mod/db/utils/transform_utils.py b/modules/research-framework/simexr_mod/db/utils/transform_utils.py
new file mode 100644
index 0000000..95a6dbc
--- /dev/null
+++ b/modules/research-framework/simexr_mod/db/utils/transform_utils.py
@@ -0,0 +1,87 @@
+from typing import Any
+
+import pandas as pd
+
+
+def _is_listy(v: Any) -> bool:
+ """Check if value is list-like (list, tuple, numpy array, pandas Series)."""
+ import numpy as np
+ return isinstance(v, (list, tuple, np.ndarray, pd.Series))
+
+
+def _to_list(v: Any) -> list:
+ """Convert value to list format."""
+ import numpy as np
+ if isinstance(v, np.ndarray):
+ return v.tolist()
+ if isinstance(v, pd.Series):
+ return v.to_list()
+ if isinstance(v, (list, tuple)):
+ return list(v)
+ # scalar → list of one
+ return [v]
+
+
+def _explode_row(model_id: str, ts: Any, params: dict, outputs: dict) -> pd.DataFrame:
+ """
+ Explode a single row where some fields in params/outputs may be arrays.
+ Strategy:
+ - Collect all keys from params + outputs
+ - Determine the per-key sequence lengths (only for list-like values)
+ - If no list-like values exist → return a single-row dataframe
+ - Otherwise, define max_len = max(list lengths)
+ - For each key:
+ * if list-like: pad/truncate to max_len (pads with None)
+ * if scalar: repeat the scalar max_len times
+ - Return a dataframe with max_len rows, adding a 'step' index (0..max_len-1)
+ """
+ # Flatten key space
+ all_keys = list(dict.fromkeys([*params.keys(), *outputs.keys()]))
+
+ # Compute lengths for list-like values
+ lengths = []
+ for k in all_keys:
+ v = params.get(k, outputs.get(k, None)) # prefer params; either is fine for length check
+ if _is_listy(v):
+ lengths.append(len(_to_list(v)))
+
+ if not lengths:
+ # No arrays: single-row record
+ row = {"model_id": model_id, "ts": ts, "step": 0}
+ # Merge params & outputs; params take precedence on key collisions
+ merged = {**outputs, **params}
+ row.update(merged)
+ return pd.DataFrame([row])
+
+ max_len = max(lengths)
+
+ def _series_for(k: str) -> list:
+ # prefer params[k] over outputs[k] only for value source when both present
+ if k in params:
+ v = params[k]
+ else:
+ v = outputs.get(k, None)
+
+ if _is_listy(v):
+ lst = _to_list(v)
+ # pad to max_len
+ if len(lst) < max_len:
+ lst = lst + [None] * (max_len - len(lst))
+ elif len(lst) > max_len:
+ lst = lst[:max_len]
+ return lst
+ else:
+ # scalar → repeat
+ return [v] * max_len
+
+ data = {
+ "model_id": [model_id] * max_len,
+ "ts": [ts] * max_len,
+ "step": list(range(max_len)),
+ }
+ for k in all_keys:
+ data[k] = _series_for(k)
+
+ return pd.DataFrame(data)
+
+
diff --git a/modules/research-framework/simexr_mod/execute/__init__.py b/modules/research-framework/simexr_mod/execute/__init__.py
new file mode 100644
index 0000000..e906e38
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/__init__.py
@@ -0,0 +1,84 @@
+"""
+Execute module for SimExR - handles simulation execution, testing, and related operations.
+
+This module provides classes for:
+- Loading and executing simulation scripts
+- Batch running simulations
+- Testing and refining simulation code
+- Managing dependencies and formatting
+- Logging and persistence
+"""
+
+# Import core classes for external use
+from .base import (
+ BaseRunner,
+ BaseManager,
+ BaseTester,
+ BaseFormatter,
+ BaseRepository,
+ SimulateLoaderProtocol,
+ LoggerProtocol
+)
+
+from .loader.simulate_loader import SimulateLoader
+from .loader.transform_code import ExternalScriptImporter
+
+from .logging.run_logger import RunLogger
+
+from .model.smoke_test_result import SmokeTestResult
+
+from .persistence.save_script import ScriptRepository
+
+from .run.batch_runner import BatchRunner
+from .run.simulation_runner import SimulationRunner
+
+# FixAgent import moved to lazy loading to avoid langchain dependency issues
+from .test.simulation_refiner import SimulationRefiner
+from .test.smoke_tester import SmokeTester
+
+from .utils.black_formatter import BlackFormatter
+from .utils.error_context import ErrorContext
+from .utils.json_utlils import json_convert
+from .utils.model_utils import make_variant_name
+from .utils.python_utils import CodeUtils
+from .utils.requirements_manager import RequirementManager
+
+__all__ = [
+ # Base classes
+ "BaseRunner",
+ "BaseManager",
+ "BaseTester",
+ "BaseFormatter",
+ "BaseRepository",
+ "SimulateLoaderProtocol",
+ "LoggerProtocol",
+
+ # Import classes
+ "SimulateLoader",
+ "ExternalScriptImporter",
+
+ # Logging
+ "RunLogger",
+
+ # Models
+ "SmokeTestResult",
+
+ # Persistence
+ "ScriptRepository",
+
+ # Runners
+ "BatchRunner",
+ "SimulationRunner",
+
+ # Test classes (FixAgent available via lazy import)
+ "SimulationRefiner",
+ "SmokeTester",
+
+ # Utilities
+ "BlackFormatter",
+ "ErrorContext",
+ "json_convert",
+ "make_variant_name",
+ "CodeUtils",
+ "RequirementManager"
+]
diff --git a/modules/research-framework/simexr_mod/execute/base.py b/modules/research-framework/simexr_mod/execute/base.py
new file mode 100644
index 0000000..8b182f9
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/base.py
@@ -0,0 +1,80 @@
+"""
+Base classes for the execute module to provide common interfaces and inheritance.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Protocol
+
+
+class SimulateLoaderProtocol(Protocol):
+ """Protocol for simulate loader implementations."""
+
+ def import_simulate(self, script_path: Path, iu=None) -> Any:
+ """Import the simulate function from a script."""
+ ...
+
+
+class LoggerProtocol(Protocol):
+ """Protocol for logger implementations."""
+
+ @staticmethod
+ def get_logger(script_path: Path) -> Any:
+ """Get a logger for the given script path."""
+ ...
+
+ @staticmethod
+ def append_jsonl(script_path: Path, record: dict, filename: str = "runs.jsonl") -> None:
+ """Append a JSONL record to the log file."""
+ ...
+
+
+@dataclass
+class BaseRunner(ABC):
+ """Base class for all runner implementations."""
+
+ @abstractmethod
+ def run(self, *args, **kwargs) -> Any:
+ """Run the execution task."""
+ pass
+
+
+@dataclass
+class BaseManager(ABC):
+ """Base class for all manager implementations."""
+
+ @abstractmethod
+ def extract(self, content: str) -> List[str]:
+ """Extract items from content."""
+ pass
+
+
+@dataclass
+class BaseTester(ABC):
+ """Base class for all tester implementations."""
+
+ @abstractmethod
+ def test(self, script_path: Path) -> Any:
+ """Test the script at the given path."""
+ pass
+
+
+@dataclass
+class BaseFormatter(ABC):
+ """Base class for all formatter implementations."""
+
+ @abstractmethod
+ def format(self, content: str) -> str:
+ """Format the given content."""
+ pass
+
+
+@dataclass
+class BaseRepository(ABC):
+ """Base class for all repository implementations."""
+
+ @abstractmethod
+ def save_and_register(self, metadata: Dict[str, Any], code: str) -> Any:
+ """Save and register the content."""
+ pass
diff --git a/modules/research-framework/simexr_mod/execute/loader/__init__.py b/modules/research-framework/simexr_mod/execute/loader/__init__.py
new file mode 100644
index 0000000..5fe9031
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/loader/__init__.py
@@ -0,0 +1,6 @@
+"""Import module for loading and transforming external scripts."""
+
+from .simulate_loader import SimulateLoader
+from .transform_code import ExternalScriptImporter
+
+__all__ = ["SimulateLoader", "ExternalScriptImporter"]
diff --git a/modules/research-framework/simexr_mod/execute/loader/simulate_loader.py b/modules/research-framework/simexr_mod/execute/loader/simulate_loader.py
new file mode 100644
index 0000000..0900d74
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/loader/simulate_loader.py
@@ -0,0 +1,55 @@
+import uuid
+import importlib.util
+import logging
+from pathlib import Path
+from typing import Callable, Any
+
+
+class SimulateLoader:
+ """Dynamically import simulate() from a file path under a random module name."""
+
+ def __init__(self):
+ self._importlib_util = importlib.util
+ self.logger = logging.getLogger("SimulateLoader")
+ self.logger.setLevel(logging.INFO)
+
+ def import_simulate(self, script_path: Path, iu=None) -> Callable[..., Any]:
+ """
+ Import the simulate function from a script file.
+
+ Args:
+ script_path: Path to the Python script containing simulate function
+ iu: importlib.util module (for dependency injection/testing)
+
+ Returns:
+ The simulate function from the module
+
+ Raises:
+ ImportError: If module cannot be loaded
+ AssertionError: If simulate function is missing
+ """
+ self.logger.info(f"[SIMULATE_LOADER] Starting import of simulate function from {script_path}")
+
+ import_util = iu if iu is not None else self._importlib_util
+ name = f"simulate_{uuid.uuid4().hex}"
+ self.logger.info(f"[SIMULATE_LOADER] Generated module name: {name}")
+
+ self.logger.info(f"[SIMULATE_LOADER] Creating spec from file location")
+ spec = import_util.spec_from_file_location(name, script_path)
+ if spec is None or spec.loader is None:
+ self.logger.error(f"[SIMULATE_LOADER] Cannot load {script_path}")
+ raise ImportError(f"Cannot load {script_path}")
+
+ self.logger.info(f"[SIMULATE_LOADER] Creating module from spec")
+ mod = import_util.module_from_spec(spec)
+
+ self.logger.info(f"[SIMULATE_LOADER] Executing module")
+ spec.loader.exec_module(mod) # type: ignore[attr-defined]
+
+ self.logger.info(f"[SIMULATE_LOADER] Checking for simulate function")
+ if not hasattr(mod, "simulate"):
+ self.logger.error(f"[SIMULATE_LOADER] simulate() function missing from module")
+ raise AssertionError("simulate() missing")
+
+ self.logger.info(f"[SIMULATE_LOADER] Successfully imported simulate function")
+ return mod.simulate
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/loader/transform_code.py b/modules/research-framework/simexr_mod/execute/loader/transform_code.py
new file mode 100644
index 0000000..2b125c1
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/loader/transform_code.py
@@ -0,0 +1,83 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Tuple, Dict, Any
+import os
+
+# Note: These imports may need adjustment based on actual project structure
+
+from code.refactor.llm_refactor import refactor_to_single_entry
+from code.utils.github_utils import fetch_notebook_from_github
+from code.utils.notebook_utils import notebook_to_script
+
+
+@dataclass
+class ExternalScriptImporter:
+ """
+ Full pipeline for an external notebook or script:
+ 1) If it's a notebook, convert to .py
+ 2) Refactor to single-entry `simulate(**params)`
+ 3) Extract metadata (from refactorer)
+ 4) Iterative smoke-tests & auto-correction
+ 5) Return (model_id, metadata)
+ """
+ models_root: Path = Path("external_models")
+
+ def __post_init__(self):
+ """Initialize dependencies after dataclass creation."""
+ self.models_root.mkdir(parents=True, exist_ok=True)
+
+ def import_and_refactor(
+ self,
+ source_url: str,
+ model_name: str,
+ dest_dir: str,
+ max_smoke_iters: int = 3,
+ llm_model: str = "gpt-5-mini",
+ ) -> Tuple[str, Dict[str, Any]]:
+ """
+ Import and refactor external script/notebook.
+
+ Args:
+ source_url: URL to the source file
+ model_name: Name for the model
+ dest_dir: Destination directory
+ max_smoke_iters: Maximum smoke test iterations
+ llm_model: LLM model to use for refactoring
+
+ Returns:
+ Tuple of (model_id, metadata)
+ """
+ print(f"[TRANSFORM_CODE] Starting import_and_refactor for {source_url}")
+ print(f"[TRANSFORM_CODE] Fetching notebook from GitHub...")
+ nb_path = fetch_notebook_from_github(source_url, dest_dir=dest_dir)
+ print(f"[TRANSFORM_CODE] Notebook fetched: {nb_path}")
+
+ print(f"[TRANSFORM_CODE] Converting notebook to script...")
+ py_path = notebook_to_script(nb_path, output_dir=str(self.models_root))
+ print(f"[TRANSFORM_CODE] Script created: {py_path}")
+
+ # Refactor into single entrypoint + extract metadata
+ print(f"[TRANSFORM_CODE] Calling refactor_to_single_entry...")
+ script_path, metadata = refactor_to_single_entry(Path(py_path))
+ print(f"[TRANSFORM_CODE] Refactoring completed. Script path: {script_path}")
+
+ # Optionally set/override user-facing name for slugging
+ metadata = dict(metadata or {})
+ metadata.setdefault("model_name", model_name)
+ print(f"[TRANSFORM_CODE] Metadata: {metadata}")
+
+ # Iterative smoke-test + correction loop
+ try:
+ # Try new structure first (lazy import to avoid circular dependencies)
+ from execute.test.simulation_refiner import SimulationRefiner
+ refiner = SimulationRefiner(
+ script_path=script_path,
+ model_name=model_name,
+ max_iterations=max_smoke_iters
+ )
+ model_id = refiner.refine()
+ except (NameError, ImportError, ModuleNotFoundError):
+ # Fallback - just use the script path as model_id
+ import hashlib
+ model_id = hashlib.md5(f"{model_name}_{script_path}".encode()).hexdigest()[:12]
+ return model_id, metadata
diff --git a/modules/research-framework/simexr_mod/execute/logging/__init__.py b/modules/research-framework/simexr_mod/execute/logging/__init__.py
new file mode 100644
index 0000000..029abd5
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/logging/__init__.py
@@ -0,0 +1,5 @@
+"""Logging module for execution tracking."""
+
+from .run_logger import RunLogger
+
+__all__ = ["RunLogger"]
diff --git a/modules/research-framework/simexr_mod/execute/logging/run_logger.py b/modules/research-framework/simexr_mod/execute/logging/run_logger.py
new file mode 100644
index 0000000..259db21
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/logging/run_logger.py
@@ -0,0 +1,51 @@
+import json
+import logging
+import logging.handlers
+import sys
+from pathlib import Path
+
+
+class RunLogger:
+ """
+ Logger utilities per model directory. Writes to models/<model>/logs/runner.log
+ and supports JSONL append.
+ """
+ @staticmethod
+ def _ensure_log_dir(script_path: Path) -> Path:
+ log_dir = script_path.parent / "logs"
+ log_dir.mkdir(parents=True, exist_ok=True)
+ return log_dir
+
+ @staticmethod
+ def get_logger(script_path: Path) -> logging.Logger:
+ log_dir = RunLogger._ensure_log_dir(script_path)
+ log_file = log_dir / "runner.log"
+ key = f"runner::{script_path.parent.resolve()}"
+ logger = logging.getLogger(key)
+ if logger.handlers:
+ return logger
+
+ logger.setLevel(logging.DEBUG)
+
+ fh = logging.handlers.RotatingFileHandler(
+ log_file, maxBytes=2_000_000, backupCount=5, encoding="utf-8"
+ )
+ fh.setLevel(logging.DEBUG)
+ fmt = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
+ fh.setFormatter(fmt)
+ logger.addHandler(fh)
+
+ ch = logging.StreamHandler(sys.stdout)
+ ch.setLevel(logging.INFO)
+ ch.setFormatter(fmt)
+ logger.addHandler(ch)
+
+ logger.debug("run logger initialized")
+ return logger
+
+ @staticmethod
+ def append_jsonl(script_path: Path, record: dict, filename: str = "runs.jsonl") -> None:
+ p = RunLogger._ensure_log_dir(script_path) / filename
+ p.parent.mkdir(parents=True, exist_ok=True)
+ with p.open("a", encoding="utf-8") as f:
+ f.write(json.dumps(record, ensure_ascii=False) + "\n")
diff --git a/modules/research-framework/simexr_mod/execute/model/__init__.py b/modules/research-framework/simexr_mod/execute/model/__init__.py
new file mode 100644
index 0000000..a756920
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/model/__init__.py
@@ -0,0 +1,5 @@
+"""Model classes for execution results."""
+
+from .smoke_test_result import SmokeTestResult
+
+__all__ = ["SmokeTestResult"]
diff --git a/modules/research-framework/simexr_mod/execute/model/smoke_test_result.py b/modules/research-framework/simexr_mod/execute/model/smoke_test_result.py
new file mode 100644
index 0000000..08c42e0
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/model/smoke_test_result.py
@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class SmokeTestResult:
+ ok: bool
+ log: str # "OK" or traceback/message
diff --git a/modules/research-framework/simexr_mod/execute/persistence/__init__.py b/modules/research-framework/simexr_mod/execute/persistence/__init__.py
new file mode 100644
index 0000000..ff12de3
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/persistence/__init__.py
@@ -0,0 +1,5 @@
+"""Persistence module for saving and loading simulation scripts."""
+
+from .save_script import ScriptRepository
+
+__all__ = ["ScriptRepository"]
diff --git a/modules/research-framework/simexr_mod/execute/persistence/save_script.py b/modules/research-framework/simexr_mod/execute/persistence/save_script.py
new file mode 100644
index 0000000..682b4fb
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/persistence/save_script.py
@@ -0,0 +1,34 @@
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Any, Tuple
+
+from slugify import slugify
+
+from execute.base import BaseRepository
+
+# Import database function - adjust path as needed
+from db import store_simulation_script
+
+
+
+@dataclass
+class ScriptRepository(BaseRepository):
+ """Persists metadata & simulate.py; registers the script in DB."""
+ root: Path = Path("models")
+
+ def save_and_register(self, metadata: Dict[str, Any], code: str) -> Tuple[str, Path]:
+ model_slug = slugify(metadata.get("model_name", "unnamed_model"))
+ model_dir = self.root / model_slug
+ model_dir.mkdir(parents=True, exist_ok=True)
+
+ (model_dir / "metadata.json").write_text(json.dumps(metadata, indent=2))
+ (model_dir / "simulate.py").write_text(code)
+
+ model_id = store_simulation_script(
+ model_name=model_slug,
+ metadata=metadata,
+ script_path=str(model_dir / "simulate.py"),
+ )
+ print(f"[✓] stored model_id = {model_id} dir = {model_dir}")
+ return model_id, model_dir
diff --git a/modules/research-framework/simexr_mod/execute/run/__init__.py b/modules/research-framework/simexr_mod/execute/run/__init__.py
new file mode 100644
index 0000000..ff3cde3
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/run/__init__.py
@@ -0,0 +1,6 @@
+"""Execution runners for single and batch simulations."""
+
+from .batch_runner import BatchRunner
+from .simulation_runner import SimulationRunner
+
+__all__ = ["BatchRunner", "SimulationRunner"]
diff --git a/modules/research-framework/simexr_mod/execute/run/batch_runner.py b/modules/research-framework/simexr_mod/execute/run/batch_runner.py
new file mode 100644
index 0000000..df59a9b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/run/batch_runner.py
@@ -0,0 +1,89 @@
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Dict, Any
+
+import pandas as pd
+from tqdm import tqdm
+
+from execute.base import BaseRunner
+from execute.logging.run_logger import RunLogger
+from execute.run.simulation_runner import SimulationRunner
+from execute.utils.requirements_manager import RequirementManager
+from db import get_simulation_path, store_simulation_results
+
+
+
+@dataclass
+class BatchRunner(BaseRunner):
+ """
+ Batch executor: ensures dependencies, adjusts sys.path, runs grid,
+ writes CSV, and persists to DB.
+ """
+ reqs: RequirementManager = field(default_factory=RequirementManager)
+ single_runner: SimulationRunner = field(default_factory=SimulationRunner)
+
+ def run(
+ self,
+ model_id: str,
+ param_grid: List[Dict[str, Any]],
+ output_csv: str = "results.csv",
+ db_path: str = "mcp.db",
+ ) -> None:
+ script_path = Path(get_simulation_path(model_id, db_path=db_path))
+ model_dir = script_path.parent
+ lib_dir = model_dir / "lib"
+
+ logger = RunLogger.get_logger(script_path)
+ logger.info(f"[BATCH] start | model_id={model_id} | script={script_path}")
+ logger.info(f"[BATCH] grid_size={len(param_grid)} | output_csv={output_csv} | db={db_path}")
+
+ # Install deps once (best-effort)
+ try:
+ reqs = self.reqs.extract(script_path.read_text())
+ logger.info(f"[BATCH] requirements: {reqs or 'none'}")
+ self.reqs.install(reqs, lib_dir)
+ logger.info("[BATCH] requirements ready")
+ except Exception as e:
+ logger.exception(f"[BATCH] dependency setup failed: {e}")
+ # continue; simulate() may still work if deps are already in env
+
+ # Import path setup
+ if str(lib_dir) not in sys.path:
+ sys.path.insert(0, str(lib_dir))
+ if str(model_dir) not in sys.path:
+ sys.path.insert(0, str(model_dir))
+
+ rows: List[Dict[str, Any]] = []
+ for i, p in enumerate(tqdm(param_grid, desc=f"Running {model_id}"), start=1):
+ logger.info(f"[BATCH] run {i}/{len(param_grid)}")
+ row = self.single_runner.run(script_path, p)
+ rows.append(row)
+ if not row.get("_ok", False):
+ logger.warning(f"[BATCH] run {i} failed: {row.get('_error_type')} | {row.get('_error_msg')}")
+
+ # Persist CSV
+ try:
+ df = pd.DataFrame(rows)
+ out = Path(output_csv)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ df.to_csv(out, index=False)
+ logger.info(f"[BATCH] wrote CSV | rows={len(df)} | path={out}")
+ print(f"{len(df)} rows → {out}") # retain original print
+ except Exception as e:
+ logger.exception(f"[BATCH] failed to write CSV: {e}")
+
+ # Persist to DB (best-effort)
+ try:
+ store_simulation_results(
+ model_id=model_id,
+ rows=rows,
+ param_keys=list(param_grid[0].keys()) if param_grid else [],
+ db_path=db_path,
+ )
+ logger.info(f"[BATCH] stored {len(rows)} rows in DB {db_path}")
+ print(f"Stored {len(rows)} rows in DB {db_path}") # retain original print
+ except Exception as e:
+ logger.exception(f"[BATCH] DB persistence failed: {e}")
+
+ logger.info("[BATCH] done")
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/run/simulation_runner.py b/modules/research-framework/simexr_mod/execute/run/simulation_runner.py
new file mode 100644
index 0000000..e88d97e
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/run/simulation_runner.py
@@ -0,0 +1,170 @@
+import datetime
+import io
+import traceback
+import logging
+from contextlib import redirect_stdout, redirect_stderr
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Any
+
+from execute.base import BaseRunner
+from execute.loader.simulate_loader import SimulateLoader
+from execute.logging.run_logger import RunLogger
+from execute.utils.json_utlils import json_convert
+
+
+@dataclass
+class SimulationRunner(BaseRunner):
+ """
+ Execute simulate(**params) capturing stdout/stderr and logging in detail.
+ Returns a row that merges params + results + diagnostic fields.
+ """
+ loader: SimulateLoader = field(default_factory=SimulateLoader)
+ logger_factory: RunLogger = field(default_factory=RunLogger)
+
+ def __post_init__(self):
+ """Initialize logging."""
+ self.logger = logging.getLogger("SimulationRunner")
+ self.logger.setLevel(logging.INFO)
+
+ def run(self, script_path: Path, params: Dict[str, Any]) -> Dict[str, Any]:
+ """Run a single simulation with the given parameters."""
+ self.logger.info(f"[SIMULATION_RUNNER] Starting simulation execution")
+ self.logger.info(f"[SIMULATION_RUNNER] Script path: {script_path}")
+ self.logger.info(f"[SIMULATION_RUNNER] Parameters: {params}")
+
+ logger = self.logger_factory.get_logger(script_path)
+ run_id = self._generate_run_id()
+
+ logger.info(f"[RUN] start simulate | script={script_path.name} | run_id={run_id}")
+ logger.debug(f"[RUN] params={params}")
+
+ start_ts = datetime.datetime.utcnow()
+ cap_out, cap_err = io.StringIO(), io.StringIO()
+
+ try:
+ self.logger.info(f"[SIMULATION_RUNNER] Executing simulation...")
+ result = self._execute_simulation(script_path, params, cap_out, cap_err)
+ duration = (datetime.datetime.utcnow() - start_ts).total_seconds()
+
+ self.logger.info(f"[SIMULATION_RUNNER] Simulation completed successfully in {duration:.3f}s")
+ self.logger.info(f"[SIMULATION_RUNNER] Result keys: {list(result.keys())}")
+
+ # Log preview of results (first 5 rows for time series data)
+ self._log_result_preview(result)
+
+ row = self._create_success_row(params, result, duration, cap_out, cap_err, run_id, script_path)
+ self._log_success(logger, duration, result, row)
+
+ except Exception as e:
+ duration = (datetime.datetime.utcnow() - start_ts).total_seconds()
+ self.logger.error(f"[SIMULATION_RUNNER] Simulation failed after {duration:.3f}s: {str(e)}")
+ self.logger.error(f"[SIMULATION_RUNNER] Error type: {type(e).__name__}")
+
+ row = self._create_error_row(params, e, duration, cap_out, cap_err, run_id, script_path)
+ self._log_error(logger, e, duration)
+
+ self.logger.info(f"[SIMULATION_RUNNER] Appending results to log file")
+ self.logger_factory.append_jsonl(script_path, row)
+ self.logger.info(f"[SIMULATION_RUNNER] Simulation execution completed")
+ return row
+
+ def _generate_run_id(self) -> str:
+ """Generate a unique run ID."""
+ return datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z"
+
+ def _execute_simulation(self, script_path: Path, params: Dict[str, Any],
+ cap_out: io.StringIO, cap_err: io.StringIO) -> Dict[str, Any]:
+ """Execute the simulation and return results."""
+ self.logger.info(f"[SIMULATION_RUNNER] Importing simulate function from {script_path}")
+ simulate = self.loader.import_simulate(script_path)
+ self.logger.info(f"[SIMULATION_RUNNER] Successfully imported simulate function")
+
+ self.logger.info(f"[SIMULATION_RUNNER] Calling simulate(**params) with captured stdout/stderr")
+ with redirect_stdout(cap_out), redirect_stderr(cap_err):
+ res = simulate(**params)
+ self.logger.info(f"[SIMULATION_RUNNER] simulate() function completed")
+
+ if not isinstance(res, dict):
+ self.logger.error(f"[SIMULATION_RUNNER] simulate() returned {type(res)}, expected dict")
+ raise TypeError(f"simulate() must return dict, got {type(res)}")
+
+ self.logger.info(f"[SIMULATION_RUNNER] Result validation passed, returning {len(res)} keys")
+ return res
+
+ def _create_success_row(self, params: Dict[str, Any], result: Dict[str, Any],
+ duration: float, cap_out: io.StringIO, cap_err: io.StringIO,
+ run_id: str, script_path: Path) -> Dict[str, Any]:
+ """Create a success result row."""
+ row = {
+ **params,
+ **result,
+ "_ok": True,
+ "_duration_s": duration,
+ "_stdout": cap_out.getvalue(),
+ "_stderr": cap_err.getvalue(),
+ "_error_type": "",
+ "_error_msg": "",
+ "_traceback": "",
+ "_run_id": run_id,
+ "_script": str(script_path),
+ }
+ return json_convert(row)
+
+ def _create_error_row(self, params: Dict[str, Any], error: Exception,
+ duration: float, cap_out: io.StringIO, cap_err: io.StringIO,
+ run_id: str, script_path: Path) -> Dict[str, Any]:
+ """Create an error result row."""
+ return {
+ **params,
+ "_ok": False,
+ "_duration_s": duration,
+ "_stdout": cap_out.getvalue(),
+ "_stderr": cap_err.getvalue(),
+ "_error_type": type(error).__name__,
+ "_error_msg": str(error),
+ "_traceback": traceback.format_exc(),
+ "_run_id": run_id,
+ "_script": str(script_path),
+ }
+
+ def _log_success(self, logger, duration: float, result: Dict[str, Any], row: Dict[str, Any]) -> None:
+ """Log successful execution."""
+ logger.info(f"[RUN] ok | duration={duration:.3f}s | keys={list(result.keys())[:8]}")
+ if row["_stderr"]:
+ logger.warning(f"[RUN] stderr non-empty ({len(row['_stderr'])} chars)")
+
+ def _log_result_preview(self, result: Dict[str, Any]) -> None:
+ """Log a preview of the simulation results (first 5 rows)."""
+ self.logger.info(f"[SIMULATION_RUNNER] === RESULT PREVIEW (First 5 rows) ===")
+
+ # Show time series data if available
+ if 't' in result and isinstance(result['t'], (list, tuple)) and len(result['t']) > 0:
+ t_data = result['t'][:5]
+ self.logger.info(f"[SIMULATION_RUNNER] Time (t): {t_data}")
+
+ if 'x' in result and isinstance(result['x'], (list, tuple)) and len(result['x']) > 0:
+ x_data = result['x'][:5]
+ self.logger.info(f"[SIMULATION_RUNNER] X trajectory: {x_data}")
+
+ if 'y' in result and isinstance(result['y'], (list, tuple)) and len(result['y']) > 0:
+ y_data = result['y'][:5]
+ self.logger.info(f"[SIMULATION_RUNNER] Y trajectory: {y_data}")
+
+ # Show key scalar results
+ scalar_keys = ['success', 'mu', 'z0', 'eval_time', 't_iteration', 'grid_points', 'mgrid_size']
+ for key in scalar_keys:
+ if key in result:
+ self.logger.info(f"[SIMULATION_RUNNER] {key}: {result[key]}")
+
+ # Show solver message if available
+ if 'solver_message' in result:
+ self.logger.info(f"[SIMULATION_RUNNER] Solver message: {result['solver_message']}")
+
+ self.logger.info(f"[SIMULATION_RUNNER] === END RESULT PREVIEW ===")
+
+ def _log_error(self, logger, error: Exception, duration: float) -> None:
+ """Log error execution."""
+ logger.error(f"[RUN] fail | duration={duration:.3f}s | {type(error).__name__}: {error}")
+ if isinstance(error, (TypeError, ValueError)):
+ logger.error("[RUN] hint: check integer-only sizes (e.g., N, array shapes) and dtype coercion.")
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/test/__init__.py b/modules/research-framework/simexr_mod/execute/test/__init__.py
new file mode 100644
index 0000000..e55a6ec
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/test/__init__.py
@@ -0,0 +1,8 @@
+"""Testing and refinement modules for simulation code."""
+
+# FixAgent import made lazy to avoid langchain_openai dependency at startup
+# from .fix_agent import FixAgent
+from .simulation_refiner import SimulationRefiner
+from .smoke_tester import SmokeTester
+
+__all__ = ["SimulationRefiner", "SmokeTester"] # FixAgent removed to avoid circular imports
diff --git a/modules/research-framework/simexr_mod/execute/test/fix_agent.py b/modules/research-framework/simexr_mod/execute/test/fix_agent.py
new file mode 100644
index 0000000..52a0870
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/test/fix_agent.py
@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from langchain.agents import initialize_agent, AgentType
+from langchain_core.messages import HumanMessage
+from langchain_core.tools import Tool
+from langchain_openai import ChatOpenAI
+
+from execute.utils.python_utils import CodeUtils
+from reasoning.tools.python_exec import PythonExecTool
+from utils.config import settings
+
+
+@dataclass
+class FixAgent:
+ """
+ Wraps a LangChain agent that can execute Python and propose code fixes.
+ You can swap this out for another backend without touching SimulationRefiner.
+ """
+ llm_name: str = "gpt-4.1"
+ temperature: float = 0.0
+ openai_api_key: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ self._llm = ChatOpenAI(
+ model_name=self.llm_name,
+ temperature=self.temperature,
+ openai_api_key=self.openai_api_key or settings.openai_api_key,
+ )
+
+ # Python execution tool; keep signature identical to your usage
+ self._py_tool = PythonExecTool()
+ self._run_tool = Tool(
+ name="python_exec",
+ func=lambda code: self._py_tool.run_python(code, df=None), # df=None for smoketests
+ description="Executes Python code and returns {ok, stdout, stderr, images}.",
+ )
+
+ self._agent = initialize_agent(
+ tools=[self._run_tool],
+ llm=self._llm,
+ agent=AgentType.OPENAI_FUNCTIONS,
+ verbose=False,
+ )
+
+ def propose_fix(self, error_log: str, current_src: str) -> str:
+ """
+ Given a failing traceback and current source, returns corrected Python code.
+ """
+ prompt = (
+ f"The following code failed during runtime with this error:\n\n"
+ f"```\n{error_log.strip()}\n```\n\n"
+ "Please correct the function. Return ONLY valid Python code (no markdown, no explanations):\n\n"
+ f"{current_src.strip()}"
+ )
+ response = self._agent.run([HumanMessage(content=prompt)]).strip()
+ return CodeUtils.extract_python_code(response)
diff --git a/modules/research-framework/simexr_mod/execute/test/simulation_refiner.py b/modules/research-framework/simexr_mod/execute/test/simulation_refiner.py
new file mode 100644
index 0000000..48f6460
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/test/simulation_refiner.py
@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from execute.test.smoke_tester import SmokeTester
+
+# Import database function - adjust path as needed
+from db import store_simulation_script
+
+if TYPE_CHECKING:
+ from execute.test.fix_agent import FixAgent
+
+
+@dataclass
+class SimulationRefiner:
+ """
+ Iteratively smoke-tests a simulate.py and uses an agent to repair it.
+ Writes intermediate .iter{i}.py files; returns model_id when passing.
+ """
+ script_path: Path
+ model_name: str
+ max_iterations: int = 3
+ smoke_tester: SmokeTester = field(default_factory=SmokeTester)
+ agent: "FixAgent" = field(default=None) # Lazy loaded to avoid langchain_openai dependency
+
+ def refine(self) -> str:
+ for i in range(1, self.max_iterations + 1):
+ res = self.smoke_tester.test(self.script_path)
+ if res.ok:
+ print(f"[✓] simulate.py passed smoke test on iteration {i}")
+ final_model_id = store_simulation_script(
+ model_name=self.model_name,
+ metadata={}, # keep parity with your original
+ script_path=str(self.script_path),
+ )
+ return final_model_id
+
+ print(f"[!] simulate.py failed on iteration {i}:\n{res.log.strip()}")
+ current_src = self.script_path.read_text()
+
+ # Lazy load FixAgent only when needed
+ if self.agent is None:
+ try:
+ from execute.test.fix_agent import FixAgent
+ self.agent = FixAgent()
+ except ImportError as e:
+ print(f"Warning: Cannot load FixAgent: {e}")
+ # Return a model_id anyway (fallback)
+ import hashlib
+ fallback_id = hashlib.md5(f"{self.model_name}_{self.script_path}".encode()).hexdigest()[:12]
+ return fallback_id
+
+ corrected_code = self.agent.propose_fix(res.log, current_src)
+
+ # Save intermediate & replace current
+ iter_path = self.script_path.with_name(f"{self.script_path.stem}.iter{i}.py")
+ iter_path.write_text(corrected_code)
+ self.script_path.write_text(corrected_code)
+
+ raise RuntimeError("simulate.py still failing after all correction attempts.")
+
diff --git a/modules/research-framework/simexr_mod/execute/test/smoke_tester.py b/modules/research-framework/simexr_mod/execute/test/smoke_tester.py
new file mode 100644
index 0000000..c9657a8
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/test/smoke_tester.py
@@ -0,0 +1,45 @@
+import importlib.util
+import inspect
+import traceback
+from dataclasses import dataclass
+from pathlib import Path
+
+from execute.base import BaseTester
+from execute.model.smoke_test_result import SmokeTestResult
+
+
+@dataclass
+class SmokeTester(BaseTester):
+ """
+ Imports a script module and calls `simulate(**dummy)` where
+ each keyword-like parameter is set to 0.0 (preserves your behavior).
+ """
+ timeout_s: int = 30 # currently informational; extend to enforce timeouts if needed
+
+ def test(self, script_path: Path) -> SmokeTestResult:
+ # 1) import under a fresh module name
+ spec = importlib.util.spec_from_file_location("smoketest_mod", str(script_path))
+ mod = importlib.util.module_from_spec(spec)
+ assert spec and spec.loader, "Invalid module spec"
+ spec.loader.exec_module(mod) # type: ignore[attr-defined]
+
+ # 2) find simulate(...)
+ if not hasattr(mod, "simulate"):
+ return SmokeTestResult(False, "No `simulate` function defined")
+
+ sig = inspect.signature(mod.simulate)
+ # 3) dummy args: every KW-ish param → 0.0
+ dummy = {
+ name: 0.0
+ for name, param in sig.parameters.items()
+ if param.kind in (param.POSITIONAL_OR_KEYWORD, param.KEYWORD_ONLY)
+ }
+
+ # 4) call simulate
+ try:
+ out = mod.simulate(**dummy)
+ if not isinstance(out, dict):
+ return SmokeTestResult(False, f"`simulate` returned {type(out)}, not dict")
+ return SmokeTestResult(True, "OK")
+ except Exception:
+ return SmokeTestResult(False, traceback.format_exc())
diff --git a/modules/research-framework/simexr_mod/execute/utils/__init__.py b/modules/research-framework/simexr_mod/execute/utils/__init__.py
new file mode 100644
index 0000000..901d3b5
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/__init__.py
@@ -0,0 +1,17 @@
+"""Utility modules for execution support."""
+
+from .black_formatter import BlackFormatter
+from .error_context import ErrorContext
+from .json_utlils import json_convert
+from .model_utils import make_variant_name
+from .python_utils import CodeUtils
+from .requirements_manager import RequirementManager
+
+__all__ = [
+ "BlackFormatter",
+ "ErrorContext",
+ "json_convert",
+ "make_variant_name",
+ "CodeUtils",
+ "RequirementManager"
+]
diff --git a/modules/research-framework/simexr_mod/execute/utils/black_formatter.py b/modules/research-framework/simexr_mod/execute/utils/black_formatter.py
new file mode 100644
index 0000000..35f6561
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/black_formatter.py
@@ -0,0 +1,24 @@
+import subprocess
+from dataclasses import dataclass
+from typing import Sequence
+
+from execute.base import BaseFormatter
+
+
+@dataclass
+class BlackFormatter(BaseFormatter):
+ """Formats Python code via black; falls back to original code on failure."""
+ black_cmd: Sequence[str] = ("black", "-q", "-")
+
+ def format(self, code: str) -> str:
+ try:
+ res = subprocess.run(
+ list(self.black_cmd),
+ input=code,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ return res.stdout
+ except Exception:
+ return code
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/utils/error_context.py b/modules/research-framework/simexr_mod/execute/utils/error_context.py
new file mode 100644
index 0000000..1edfaf5
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/error_context.py
@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ErrorContext:
+ """Creates concise error context for LLM retries."""
+
+ def syntax(self, code: str, err: SyntaxError, around: int = 2) -> str:
+ lines = code.splitlines()
+ lineno = err.lineno or 0
+ first = max(1, lineno - around)
+ last = min(len(lines), lineno + around)
+ snippet = "\n".join(
+ f"{'→' if i == lineno else ' '} {i:>4}: {lines[i-1]}"
+ for i in range(first, last + 1)
+ )
+ return (
+ f"SyntaxError `{err.msg}` at line {err.lineno}\n"
+ f"Context:\n{snippet}\n\n"
+ "Please correct the code and return only the updated Python."
+ )
+
+ def runtime(self, trace: str, limit: int = 25) -> str:
+ tb_tail = "\n".join(trace.splitlines()[-limit:]) or "<no traceback captured>"
+ last = trace.strip().splitlines()[-1] if trace else "<no output>"
+ return (
+ f"RuntimeError `{last}`\n"
+ f"Traceback (last {limit} lines):\n{tb_tail}\n\n"
+ "Please fix the code and return only the updated Python."
+ )
diff --git a/modules/research-framework/simexr_mod/execute/utils/json_utlils.py b/modules/research-framework/simexr_mod/execute/utils/json_utlils.py
new file mode 100644
index 0000000..230f820
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/json_utlils.py
@@ -0,0 +1,30 @@
+import json
+from typing import Any
+
+import numpy as np
+
+
+def json_convert(val: Any) -> Any:
+ import datetime as dt
+
+ if isinstance(val, (np.generic,)):
+ return val.item()
+ if isinstance(val, np.ndarray):
+ return [json_convert(v) for v in val.tolist()]
+ if isinstance(val, set):
+ return [json_convert(v) for v in val]
+ if isinstance(val, (dt.datetime, dt.date, dt.time)):
+ return val.isoformat()
+ if isinstance(val, (bytes, bytearray)):
+ return val.decode("utf-8", errors="replace")
+ if isinstance(val, complex):
+ return {"real": float(val.real), "imag": float(val.imag)}
+ if isinstance(val, dict):
+ return {k: json_convert(v) for k, v in val.items()}
+ if isinstance(val, (list, tuple)):
+ return [json_convert(v) for v in val]
+ try:
+ json.dumps(val)
+ return val
+ except Exception:
+ return str(val)
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/utils/model_utils.py b/modules/research-framework/simexr_mod/execute/utils/model_utils.py
new file mode 100644
index 0000000..2147343
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/model_utils.py
@@ -0,0 +1,42 @@
+import re
+import hashlib
+from pathlib import Path
+
+# Import database function - adjust path as needed
+from db import get_simulation_path
+
+def _sha256(s: str) -> str:
+ return hashlib.sha256(s.encode("utf-8")).hexdigest()
+
+def make_variant_name(model_id: str, new_script: str, hash_len: int = 12) -> tuple[str, str, str]:
+ """
+ Create (variant_name, variant_path, variant_model_id) using a '<prefix>_<hash>' model id.
+
+ Examples:
+ model_id='lorenz_attractor_ea73a2d691d3'
+ -> prefix='lorenz_attractor'
+ model_id='lorenz_attractor_ea73a2d691d3::anything'
+ -> prefix='lorenz_attractor'
+
+ We compute new_hash = sha256(new_script)[:hash_len] and return:
+ variant_model_id = f'{prefix}_{new_hash}'
+ variant_name = f'{variant_model_id}.py'
+ variant_path = Path(get_simulation_path(prefix)).with_name(variant_name)
+ """
+ # Strip any '::suffix' if present
+ base = model_id.split("::", 1)[0]
+
+ # Extract prefix by removing a trailing _<hexhash> (6..64 hex chars) if present
+ m = re.match(r"^(?P<prefix>.+?)_(?P<hash>[0-9a-fA-F]{6,64})$", base)
+ prefix = m.group("prefix") if m else base
+
+ # Compute new short hash from script content
+ new_hash = _sha256(new_script)[:hash_len]
+
+ # Compose ids/paths
+ variant_model_id = f"{prefix}_{new_hash}"
+ variant_name = f"{variant_model_id}.py"
+ # Save alongside the base model's script
+ variant_path = Path(get_simulation_path(model_id)).with_name(variant_name)
+
+ return variant_name, str(variant_path), variant_model_id
diff --git a/modules/research-framework/simexr_mod/execute/utils/python_utils.py b/modules/research-framework/simexr_mod/execute/utils/python_utils.py
new file mode 100644
index 0000000..0f6fd65
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/python_utils.py
@@ -0,0 +1,32 @@
+import re
+import textwrap
+
+
+class CodeUtils:
+ """Small static helpers for code text processing."""
+
+ @staticmethod
+ def extract_python_code(response: str) -> str:
+ """
+ Given an LLM response that may contain explanation + fenced code,
+ extract just the Python code (same logic you had, packaged).
+ """
+ m = re.search(r"```(?:python)?\s*([\s\S]+?)```", response, re.IGNORECASE)
+ if m:
+ return m.group(1).strip()
+
+ idx = response.find("def simulate")
+ if idx != -1:
+ return response[idx:].strip()
+
+ return response.strip()
+
+ @staticmethod
+ def dedent_if_needed(code: str) -> str:
+ """
+ If the first non-blank line is indented, dedent & strip leading whitespace.
+ """
+ first = next((l for l in code.splitlines() if l.strip()), "")
+ if first.startswith((" ", "\t")):
+ return textwrap.dedent(code).lstrip()
+ return code
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/execute/utils/requirements_manager.py b/modules/research-framework/simexr_mod/execute/utils/requirements_manager.py
new file mode 100644
index 0000000..385ade0
--- /dev/null
+++ b/modules/research-framework/simexr_mod/execute/utils/requirements_manager.py
@@ -0,0 +1,65 @@
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Iterable
+
+from execute.base import BaseManager
+
+
+@dataclass
+class RequirementManager(BaseManager):
+ """Extracts and installs Python requirements referenced by the generated code."""
+ enable_install: bool = True
+
+ _IGNORE: frozenset = frozenset({"__future__", "typing"})
+
+ def extract(self, script: str) -> List[str]:
+ """
+ - Parse `REQUIREMENTS = ["pkg1", "pkg2"]`
+ - Fallback: scan `import X` / `from X import ...`
+ """
+ pkgs: List[str] = []
+
+ m = re.search(r"REQUIREMENTS\s*=\s*\[(.*?)\]", script, re.S)
+ if m:
+ pkgs.extend(re.findall(r"[\"']([^\"']+)[\"']", m.group(1)))
+
+ for line in script.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ m1 = re.match(r"import\s+([\w_]+)", line)
+ m2 = re.match(r"from\s+([\w_]+)", line)
+ name = m1.group(1) if m1 else (m2.group(1) if m2 else None)
+ if name and name not in self._IGNORE:
+ pkgs.append(name)
+
+ return sorted(set(pkgs))
+
+ def install(self, pkgs: Iterable[str], target_dir: Path = None) -> None:
+ """Install packages, optionally to a target directory."""
+ if not self.enable_install:
+ return
+ for pkg in pkgs:
+ try:
+ __import__(pkg)
+ except ModuleNotFoundError:
+ print(f"📦 Installing '{pkg}' …")
+ try:
+ cmd = [sys.executable, "-m", "pip", "install", "--upgrade", "--no-cache-dir", pkg]
+ if target_dir:
+ cmd.extend(["--target", str(target_dir)])
+ subprocess.run(
+ cmd,
+ check=True,
+ stdout=subprocess.DEVNULL,
+ )
+ except subprocess.CalledProcessError as e:
+ print(f"⚠️ pip install failed for '{pkg}': {e}")
+
+ def ensure_installed(self, pkgs: Iterable[str]) -> None:
+ """Legacy method for backwards compatibility."""
+ self.install(pkgs)
+
diff --git a/modules/research-framework/simexr_mod/llm/local_llm.py b/modules/research-framework/simexr_mod/llm/local_llm.py
new file mode 100644
index 0000000..38f7e53
--- /dev/null
+++ b/modules/research-framework/simexr_mod/llm/local_llm.py
@@ -0,0 +1,52 @@
+import subprocess
+
+class LocalLLM:
+ """
+ Thin wrapper around Ollama (or LM Studio) to run a local model.
+
+ Parameters
+ ----------
+ model : str
+ Ollama model tag, e.g. "deepseek-coder:6.7b-instruct".
+ """
+
+ def __init__(self, model: str = "codellama:7b-instruct"):
+ self.model = model
+
+ # ------------- PUBLIC API ------------------------------------------------
+ def generate(
+ self,
+ prompt: str,
+ system_prompt: str = "",
+ temperature: float = 0.0,
+ num_tokens: int | None = None, # optional n-token limit
+ ) -> str:
+ """
+ Call the local LLM with a prompt.
+
+ Uses the Ollama chat command:
+ /set parameter temperature <value>
+
+ Notes
+ -----
+ * Works even on older Ollama builds that don’t support --temp.
+ * You can still change top-p, top-k, etc. the same way.
+ """
+ # prepend the /set command, then optional system prompt
+ header_lines = [f"/set parameter temperature {temperature}"]
+ if system_prompt.strip():
+ header_lines.append(system_prompt.strip())
+ header = "\n\n".join(header_lines)
+
+ full_prompt = f"{header}\n\n{prompt.strip()}"
+
+ cmd = ["ollama", "run", self.model, full_prompt]
+ if num_tokens is not None:
+ cmd += ["-n", str(num_tokens), "--no-cache"]
+
+ result = subprocess.run(cmd, capture_output=True, text=True)
+ if result.returncode != 0:
+ raise RuntimeError(f"Ollama stderr:\n{result.stderr}")
+ if not result.stdout.strip():
+ raise RuntimeError("Ollama returned an empty response.")
+ return result.stdout.strip()
diff --git a/modules/research-framework/simexr_mod/llm/prompt_templates.py b/modules/research-framework/simexr_mod/llm/prompt_templates.py
new file mode 100644
index 0000000..1894ef3
--- /dev/null
+++ b/modules/research-framework/simexr_mod/llm/prompt_templates.py
@@ -0,0 +1,192 @@
+parser_prompt = """You are a precise JSON generator.
+
+Given a natural‑language query describing a physical system and experimental goal,
+extract the following structured metadata and return only valid ASCII JSON:
+
+{
+ "model_name": str,
+ "equations": str,
+ "initial_conditions": [str, ...], # must be a JSON ARRAY, not an object
+ "parameters": { str: str, ... },
+ "vary_variable": { str: list | str, ... }, # each value is EITHER a JSON list OR a plain string
+ "objective": str
+}
+
+Output rules
+------------
+• Return **ONLY** the raw JSON object—no Markdown, comments, or code fences.
+• Use **ONLY** double quotes (") and valid JSON (no trailing commas).
+• Use plain ASCII: letters, digits, standard punctuation.
+ – Write Greek letters as names: theta, omega, pi, etc.
+ – Use * for multiplication, / for division, ' for derivatives.
+
+**Do NOT**
+ • wrap lists or tuples inside quotes
+ • put units or superscripts inside numeric strings
+ • insert unescaped line‑breaks (newline or carriage‑return) inside any string value.
+ Every "value" must be a single physical line or use \\n escapes.
+
+Independent‑variable rule
+-------------------------
+1. If the query specifies a range, grid, list, or sweep for an independent
+ variable (e.g. “simulate for t from 0 to 50” or “x in [0,1]”), put that
+ variable in "vary_variable" with its values.
+2. If no varying quantity is explicitly given but the query is a time‑domain
+ simulation, default to `"t": []` (empty list means implicit time grid).
+3. Allowed formats for each value:
+ • JSON **list** → `[0, 0.1, 0.2]` or `["0.5", "1.0"]`
+ • JSON **3‑item list** → `[0, 50, 0.01]` (start, end, step)
+ • Simple **string range** → `"0-50"`
+ • Empty list → `[]` (unknown / default)
+
+Other fields
+------------
+• "parameters": constant parameter values or expressions as **strings with no units**.
+• "initial_conditions": each entry like `"theta(0)=1.5708"` or `"x'(0)=0"`.
+• Make a sensible guess if a field is missing.
+
+"""
+
+codegen_prompt_template = r"""
+You are a Python code-generator for **single-run physical simulations**.
+
+Given the structured metadata below, emit *only* executable Python code
+up to the sentinel. Any text after the sentinel is ignored.
+
+──────────────────── FORMAT RULES ────────────────────
+- Output pure Python (no Markdown / HTML / tags / artefacts).
+- Use # comments for explanations; no standalone prose.
+- ASCII outside string literals; never leave an unterminated string.
+- **Do NOT include the metadata object in the code.**
+
+────────────────── REQUIRED STRUCTURE ─────────────────
+1. Dependency header – one line:
+ REQUIREMENTS = ["numpy", "scipy", "matplotlib"]
+
+2. Imports – standard aliases:
+ import json, sys
+ import numpy as np
+ from scipy.integrate import odeint
+ import matplotlib.pyplot as plt
+
+3. Reproducibility (at top level, before simulate):
+ # ─── Reproducibility ───────────────────────────────
+ np.random.seed(0)
+
+4. Function `simulate(**params)`:
+ - If initial condition values are missing from params, make intelligent guess for the values.
+ - **Check data types of incoming params**:
+ • If a numeric value arrives as a string or Python scalar, cast to `np.float64`
+ • If a list arrives, cast to a NumPy array of `np.float64`
+ • Guarantee arrays are at least 1‑D (e.g. via `np.atleast_1d`), if they are not make necessary operations.
+ - Internally use NumPy arrays and NumPy scalar types for all numeric work.
+ - Pin ODE tolerances for deterministic integration:
+ sol = odeint(..., atol=1e-9, rtol=1e-9)
+ - (If an analytic solution exists, compute it and return it.)
+ - **Before returning**, convert:
+ • any NumPy arrays → Python lists
+ • any NumPy scalar types → Python built‑in (`float`/`int`)
+ - Return a dict of Python‑internal scalars and/or small lists.
+ - Make sure the return dict has keys that are just param names and output param names
+ with corresponding experimental values as lists.
+ - Add an assert to ensure the return is a dict.
+
+5. CLI runner (executable script):
+ if __name__ == "__main__":
+ import argparse, json
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--params", required=True,
+ help="JSON string with simulation parameters")
+ args = ap.parse_args()
+ result = simulate(**json.loads(args.params))
+ print(json.dumps(result))
+
+──────────────────── METADATA (reference only) ─────────
+{metadata_json}
+
+### END_OF_CODE
+"""
+
+
+
+repair_prompt_template = """
+You previously wrote Python code for a physics simulation, but it failed.
+
+---
+METADATA (read-only)
+{metadata_json}
+
+---
+BUGGY CODE
+{buggy_code}
+
+
+---
+OBSERVATION
+{error_log}
+
+
+---
+TASK
+Think step-by-step how to fix the problem, then output the *complete, corrected* code file.
+Remember:
+* keep the same public API (simulate(**params))
+* follow all the formatting rules from earlier (no markdown, no triple-quotes, etc.)
+* output **only** the python source.
+"""
+
+analysis_prompt = """You are a data-analysis agent that has access to a helper tool called
+python_exec. A pandas DataFrame named `df` (already loaded in memory)
+holds the experimental results.
+
+◆ OUTPUT FORMAT
+Return **one** JSON object, nothing else:
+
+{
+ "thoughts": "<explain what you are going to do>",
+ "code": "<python to run, or null>",
+ "answer": "<final answer, or null>"
+}
+
+◆ PROTOCOL
+step 1 FIRST reply must include Python in "code" and leave "answer": null.
+ Write plain Python – no ``` fences.
+
+step 2 The orchestrator executes the code with python_exec and sends you
+ a role=tool message containing the JSON result.
+
+step 3 Seeing that tool message, reply again with
+ • updated "thoughts"
+ • "code": null
+ • the finished "answer".
+
+◆ RULES
+- The whole reply must be valid JSON — no trailing commas, no extra text.
+- Do **not** guess the answer before you see the tool result.
+- Keep code short; only import what you need (pandas, numpy, etc.).
+"""
+
+SYSTEM_PROMPT_TEMPLATE = """\
+You are a scientific reasoning assistant.
+
+Below is the simulation model code I ran (you may refer to it as needed for your analysis):
+
+{SIM_CODE}
+
+You also have a pandas DataFrame `df` containing all experiment results, with columns:
+ {SCHEMA}
+
+Simulation metadata parameters (name → description):
+ {PARAMS}
+
+When you need to analyse or plot the data, call the tool exactly as JSON:
+ {{
+ "tool": "python_exec_on_df",
+ "args": {{ "code": "<your python code here>" }}
+ }}
+
+When you are finished, respond exactly with one JSON object:
+ {{ "answer": "<your diagnostic report and conclusion>" }}
+
+– Only one JSON object per message, with either `"tool"` or `"answer"`.
+"""
diff --git a/modules/research-framework/simexr_mod/pages/param_annotations.py b/modules/research-framework/simexr_mod/pages/param_annotations.py
new file mode 100644
index 0000000..171fbc1
--- /dev/null
+++ b/modules/research-framework/simexr_mod/pages/param_annotations.py
@@ -0,0 +1,453 @@
+# pages/param_annotations.py
+# SimExR: Parameter Annotations & Model Management
+
+import streamlit as st
+import json
+import requests
+import pandas as pd
+from typing import Dict, List, Any
+import time
+
+# API Configuration
+API_BASE_URL = "http://127.0.0.1:8000"
+
+def make_api_request(method: str, endpoint: str, data: Dict = None, params: Dict = None) -> Dict:
+ """Make an API request and return the response."""
+ url = f"{API_BASE_URL}{endpoint}"
+ headers = {"Content-Type": "application/json"}
+
+ try:
+ if method.upper() == "GET":
+ response = requests.get(url, headers=headers, params=params)
+ elif method.upper() == "POST":
+ if params:
+ response = requests.post(url, headers=headers, params=params)
+ else:
+ response = requests.post(url, headers=headers, json=data)
+ elif method.upper() == "DELETE":
+ response = requests.delete(url, headers=headers)
+ else:
+ raise ValueError(f"Unsupported method: {method}")
+
+ response.raise_for_status()
+ return response.json()
+
+ except requests.exceptions.RequestException as e:
+ st.error(f"API request failed: {e}")
+ return {"error": str(e)}
+
+def search_models(query: str, limit: int = 10) -> List[Dict]:
+ """Search for models using the fuzzy search API with caching."""
+ cache_key = f"{query}_{limit}"
+
+ # Check cache first
+ if cache_key in st.session_state.cached_search_results:
+ return st.session_state.cached_search_results[cache_key]
+
+ # Fetch from API
+ result = make_api_request("GET", f"/simulation/models/search?name={query}&limit={limit}")
+ models = result.get("models", []) if "error" not in result else []
+
+ # Cache the results
+ st.session_state.cached_search_results[cache_key] = models
+ return models
+
+def get_model_info(model_id: str) -> Dict:
+ """Get detailed information about a model with caching."""
+ # Check cache first
+ if model_id in st.session_state.cached_model_info:
+ return st.session_state.cached_model_info[model_id]
+
+ # Fetch from API
+ result = make_api_request("GET", f"/simulation/models/{model_id}")
+ model_info = result.get("model", {}) if "error" not in result else {}
+
+ # Cache the results
+ if model_info:
+ st.session_state.cached_model_info[model_id] = model_info
+
+ return model_info
+
+def get_model_script(model_id: str) -> str:
+ """Get the refactored script for a model."""
+ result = make_api_request("GET", f"/simulation/models/{model_id}/script")
+ return result.get("script", "") if "error" not in result else ""
+
+def save_model_script(model_id: str, script: str) -> Dict:
+ """Save the modified script for a model."""
+ data = {"script": script}
+ result = make_api_request("POST", f"/simulation/models/{model_id}/script", data)
+ return result
+
+def extract_parameters_from_script(script_content: str) -> Dict:
+ """Extract parameters from script content using simple AST analysis."""
+ import ast
+ import re
+
+ params = {}
+
+ try:
+ # Parse the script
+ tree = ast.parse(script_content)
+
+ # Look for parameter definitions in the simulate function
+ for node in ast.walk(tree):
+ if isinstance(node, ast.FunctionDef) and node.name == 'simulate':
+ # Look for parameter handling in the function
+ for stmt in ast.walk(node):
+ if isinstance(stmt, ast.Assign):
+ for target in stmt.targets:
+ if isinstance(target, ast.Name):
+ param_name = target.id
+ # Skip common variable names
+ if param_name not in ['result', 'params', 'i', 'j', 'k', 'x', 'y', 't']:
+ # Try to extract default value
+ default_value = None
+ if isinstance(stmt.value, ast.Constant):
+ default_value = stmt.value.value
+ elif isinstance(stmt.value, ast.Num):
+ default_value = stmt.value.n
+ elif isinstance(stmt.value, ast.Str):
+ default_value = stmt.value.s
+ elif isinstance(stmt.value, ast.List):
+ default_value = [elt.value if isinstance(elt, ast.Constant) else str(elt) for elt in stmt.value.elts]
+
+ # Determine parameter type
+ param_type = 'string'
+ if isinstance(default_value, (int, float)):
+ param_type = 'number'
+ elif isinstance(default_value, bool):
+ param_type = 'boolean'
+ elif isinstance(default_value, list):
+ param_type = 'array'
+
+ params[param_name] = {
+ 'type': param_type,
+ 'default': default_value,
+ 'description': f'Parameter {param_name} extracted from script'
+ }
+
+ # Also look for params.get() calls to find parameters
+ param_pattern = r'params\.get\([\'"]([^\'"]+)[\'"]'
+ matches = re.findall(param_pattern, script_content)
+
+ for param_name in matches:
+ if param_name not in params:
+ params[param_name] = {
+ 'type': 'string',
+ 'default': '',
+ 'description': f'Parameter {param_name} found in params.get() call'
+ }
+
+ except Exception as e:
+ st.warning(f"Error parsing script: {e}")
+
+ return params
+
+def analyze_parameter_occurrences(script_content: str, parameters: Dict) -> Dict:
+ """Analyze script to find parameter occurrences and their context."""
+ import re
+
+ occurrences = {}
+
+ for param_name in parameters.keys():
+ param_occurrences = []
+ lines = script_content.split('\n')
+
+ for line_num, line in enumerate(lines, 1):
+ # Look for parameter usage in the line
+ if param_name in line:
+ # Get context (surrounding lines)
+ start_line = max(0, line_num - 2)
+ end_line = min(len(lines), line_num + 1)
+ context_lines = lines[start_line:end_line]
+ context = '\n'.join(context_lines)
+
+ # Determine usage type
+ usage_type = 'unknown'
+ if f'params.get("{param_name}"' in line or f"params.get('{param_name}'" in line:
+ usage_type = 'parameter_access'
+ elif f'{param_name} =' in line:
+ usage_type = 'assignment'
+ elif param_name in line and any(op in line for op in ['+', '-', '*', '/', '=']):
+ usage_type = 'calculation'
+ else:
+ usage_type = 'reference'
+
+ param_occurrences.append({
+ 'line': line_num,
+ 'context': line.strip(),
+ 'full_context': context,
+ 'usage_type': usage_type
+ })
+
+ occurrences[param_name] = param_occurrences
+
+ return occurrences
+
+# Initialize session state for parameter tracking
+if "parameter_changes" not in st.session_state:
+ st.session_state.parameter_changes = {}
+if "original_parameters" not in st.session_state:
+ st.session_state.original_parameters = {}
+if "current_script" not in st.session_state:
+ st.session_state.current_script = ""
+
+# Initialize session state for caching (needed for this page)
+if "cached_model_info" not in st.session_state:
+ st.session_state.cached_model_info = {}
+if "cached_model_results" not in st.session_state:
+ st.session_state.cached_model_results = {}
+if "cached_model_code" not in st.session_state:
+ st.session_state.cached_model_code = {}
+if "cached_search_results" not in st.session_state:
+ st.session_state.cached_search_results = {}
+if "selected_model_id" not in st.session_state:
+ st.session_state.selected_model_id = None
+
+st.title("📝 Parameter Annotations & Script Management")
+
+# Model Selection
+st.header("1. Select Model")
+
+search_query = st.text_input("Search models", placeholder="Enter model name...")
+
+if search_query:
+ models = search_models(search_query, limit=10)
+ if models:
+ model_options = {f"{m['name']} ({m['id']})": m['id'] for m in models}
+ selected_model = st.selectbox("Choose a model", list(model_options.keys()))
+
+ if selected_model:
+ model_id = model_options[selected_model]
+ st.session_state.selected_model_id = model_id
+
+ # Get model info
+ model_info = get_model_info(model_id)
+
+ if model_info:
+ st.success(f"✅ Selected model: {model_info.get('name', model_id)}")
+
+ # Display model metadata
+ with st.expander("📋 Model Information"):
+ st.json(model_info)
+
+ # Extract parameters from model info and script
+ extracted_params = model_info.get('parameters', {})
+
+ # Get script content for parameter extraction
+ script_result = make_api_request("GET", f"/simulation/models/{model_id}/script")
+ script_content = ""
+ if "error" not in script_result:
+ script_content = script_result.get("script", "")
+
+ # Extract parameters from script if not available in model info
+ if not extracted_params and script_content:
+ extracted_params = extract_parameters_from_script(script_content)
+
+ if extracted_params:
+ st.header("2. Parameter Management")
+
+ # Store original parameters if not already stored
+ if model_id not in st.session_state.original_parameters:
+ st.session_state.original_parameters[model_id] = extracted_params.copy()
+ st.session_state.parameter_changes[model_id] = {}
+
+ # Analyze script for parameter occurrences
+ param_occurrences = analyze_parameter_occurrences(script_content, extracted_params)
+
+ # Two-column layout: Parameters on left, Statistics on right
+ col1, col2 = st.columns([2, 1])
+
+ with col1:
+ st.markdown("### 📊 Parameters & Values")
+
+ # Parameter editing form
+ with st.form("parameter_form"):
+ updated_params = {}
+
+ for param_name, param_info in extracted_params.items():
+ param_type = param_info.get('type', 'string')
+ param_default = param_info.get('default', '')
+ occurrence_count = len(param_occurrences.get(param_name, []))
+
+ # Determine status and change indicator
+ is_changed = param_name in st.session_state.parameter_changes.get(model_id, {})
+ change_tag = " 🔄" if is_changed else ""
+
+ # Status indicator
+ if occurrence_count == 0:
+ status = "🔴"
+ elif occurrence_count == 1:
+ status = "🟡"
+ else:
+ status = "🟢"
+
+ st.markdown(f"**{param_name}** ({param_type}) {status}{change_tag}")
+
+ # Create appropriate input based on type
+ if param_type == 'number':
+ value = st.number_input(
+ f"Value for {param_name}",
+ value=float(param_default) if param_default else 0.0,
+ key=f"param_{model_id}_{param_name}"
+ )
+ elif param_type == 'boolean':
+ value = st.checkbox(
+ f"Value for {param_name}",
+ value=bool(param_default) if param_default else False,
+ key=f"param_{model_id}_{param_name}"
+ )
+ elif param_type == 'array':
+ value_str = st.text_input(
+ f"Value for {param_name} (JSON array)",
+ value=json.dumps(param_default) if param_default else "[]",
+ key=f"param_{model_id}_{param_name}"
+ )
+ try:
+ value = json.loads(value_str)
+ except json.JSONDecodeError:
+ value = param_default
+ else:
+ value = st.text_input(
+ f"Value for {param_name}",
+ value=str(param_default) if param_default else "",
+ key=f"param_{model_id}_{param_name}"
+ )
+
+ updated_params[param_name] = value
+
+ # Track changes
+ original_value = st.session_state.original_parameters[model_id].get(param_name, {}).get('default', '')
+ if value != original_value:
+ st.session_state.parameter_changes[model_id][param_name] = {
+ 'original': original_value,
+ 'current': value,
+ 'changed': True,
+ 'occurrences': occurrence_count
+ }
+ else:
+ if param_name in st.session_state.parameter_changes[model_id]:
+ del st.session_state.parameter_changes[model_id][param_name]
+
+ submit_params = st.form_submit_button("💾 Save Changes")
+
+ with col2:
+ st.markdown("### 📈 Statistics & Actions")
+
+ # Parameter statistics
+ total_params = len(extracted_params)
+ active_params = sum(1 for param in extracted_params.keys() if param_occurrences.get(param))
+ unused_params = total_params - active_params
+ changed_params = len(st.session_state.parameter_changes.get(model_id, {}))
+
+ st.metric("Total Parameters", total_params)
+ st.metric("Active Parameters", active_params)
+ st.metric("Changed Parameters", changed_params)
+
+ # Quick actions
+ st.markdown("### ⚡ Quick Actions")
+
+ if st.button("🔄 Reset All"):
+ st.session_state.parameter_changes[model_id] = {}
+ st.rerun()
+
+ if st.button("📊 Export"):
+ param_data = {
+ "model_id": model_id,
+ "parameters": updated_params,
+ "changes": st.session_state.parameter_changes.get(model_id, {}),
+ "occurrences": param_occurrences
+ }
+ st.download_button(
+ label="📄 Download",
+ data=json.dumps(param_data, indent=2),
+ file_name=f"{model_id}_parameters.json",
+ mime="application/json"
+ )
+
+ # Show change summary
+ if changed_params > 0:
+ st.success(f"🔄 {changed_params} parameters modified")
+ with st.expander("📝 View Changes"):
+ for param_name, change_info in st.session_state.parameter_changes[model_id].items():
+ st.write(f"**{param_name}:** {change_info['original']} → {change_info['current']}")
+ else:
+ st.info("✅ No changes detected")
+
+ # Script Management
+ st.header("3. Script Management")
+
+ # Get current script
+ current_script = get_model_script(model_id)
+
+ if current_script:
+ st.subheader("📝 Refactored Script")
+
+ # Script editing
+ edited_script = st.text_area(
+ "Edit the refactored script:",
+ value=current_script,
+ height=400,
+ key=f"script_{model_id}"
+ )
+
+ col1, col2 = st.columns(2)
+
+ with col1:
+ if st.button("💾 Save Script Changes"):
+ result = save_model_script(model_id, edited_script)
+ if "error" not in result:
+ st.success("✅ Script saved successfully!")
+ else:
+ st.error(f"❌ Failed to save script: {result.get('error')}")
+
+ with col2:
+ if st.button("🔄 Reset Script"):
+ st.rerun()
+
+ # Script preview
+ with st.expander("👀 Script Preview"):
+ st.code(edited_script, language="python")
+
+ # Simulation with updated parameters
+ st.header("4. Quick Simulation")
+
+ if st.button("🚀 Run Simulation with Current Parameters"):
+ # Get the updated parameters from the form
+ updated_params = {}
+ for param_name in extracted_params.keys():
+ param_key = f"param_{model_id}_{param_name}"
+ if param_key in st.session_state:
+ updated_params[param_name] = st.session_state[param_key]
+
+ if updated_params:
+ with st.spinner("Running simulation with updated parameters..."):
+ data = {
+ "model_id": model_id,
+ "parameters": updated_params
+ }
+
+ result = make_api_request("POST", "/simulation/run", data)
+
+ if "error" not in result:
+ st.success("✅ Simulation completed successfully!")
+
+ with st.expander("📊 Simulation Results"):
+ st.json(result)
+
+ # Store results for other pages
+ st.session_state.simulation_results = result
+ else:
+ st.error(f"❌ Simulation failed: {result.get('error')}")
+ else:
+ st.warning("⚠️ No parameters available for simulation")
+
+ else:
+ st.error("❌ Failed to load model information")
+ else:
+ st.info("Please select a model to continue")
+ else:
+ st.warning("No models found matching your search.")
+else:
+ st.info("🔍 Enter a model name to search and get started with parameter annotations")
diff --git a/modules/research-framework/simexr_mod/reasoning/__init__.py b/modules/research-framework/simexr_mod/reasoning/__init__.py
new file mode 100644
index 0000000..c909fb1
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/__init__.py
@@ -0,0 +1,81 @@
+"""
+Reasoning module for SimExR - handles AI-powered analysis and reasoning operations.
+
+This module provides classes for:
+- Reasoning agents that can analyze data and answer questions
+- LLM client interfaces for different AI models
+- Tools for executing Python code and simulations
+- Utilities for managing conversation history and extracting results
+"""
+
+# Import core classes for external use
+from .base import (
+ BaseAgent,
+ BaseClient,
+ BaseSimulationTool,
+ BaseUtility,
+ LLMClientProtocol,
+ ReasoningToolProtocol,
+ HistoryManagerProtocol,
+ CodeExtractorProtocol
+)
+
+from .agent.loop import ReasoningAgent
+
+from .messages.llm_client import LLMClient
+from .messages.model import ModelMessage
+from .messages.openai_client import OpenAIChatClient
+
+from .model.reasoning_result import ReasoningResult
+
+from .tools.final_answer import FinalAnswerTool
+from .tools.python_exec import PythonExecTool
+from .tools.simulate_exec import SimulateTools
+
+from .utils.extract_code_map import extract_code_map
+from .utils.history import prune_history
+from .utils.json_utils import _safe_parse
+from .utils.load_results import load_results
+
+from .helpers.chat_utils import prune_history as prune_history_helper
+from .helpers.prompts import _default_system_prompt, _append_tool_message
+
+from .config.tools import _openai_tools_spec
+
+__all__ = [
+ # Base classes
+ "BaseAgent",
+ "BaseClient",
+ "BaseSimulationTool",
+ "BaseUtility",
+ "LLMClientProtocol",
+ "ReasoningToolProtocol",
+ "HistoryManagerProtocol",
+ "CodeExtractorProtocol",
+
+ # Agent classes
+ "ReasoningAgent",
+
+ # Message classes
+ "LLMClient",
+ "ModelMessage",
+ "OpenAIChatClient",
+
+ # Model classes
+ "ReasoningResult",
+
+ # Tool classes
+ "FinalAnswerTool",
+ "PythonExecTool",
+ "SimulateTools",
+
+ # Utility functions
+ "extract_code_map",
+ "prune_history",
+ "_safe_parse",
+ "load_results",
+ "prune_history_helper",
+ "_default_system_prompt",
+ "_append_tool_message",
+ "_openai_tools_spec"
+]
diff --git a/modules/research-framework/simexr_mod/reasoning/agent/__init__.py b/modules/research-framework/simexr_mod/reasoning/agent/__init__.py
new file mode 100644
index 0000000..87ba182
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/agent/__init__.py
@@ -0,0 +1,5 @@
+"""Agent module for reasoning operations."""
+
+from .loop import ReasoningAgent
+
+__all__ = ["ReasoningAgent"]
diff --git a/modules/research-framework/simexr_mod/reasoning/agent/loop.py b/modules/research-framework/simexr_mod/reasoning/agent/loop.py
new file mode 100644
index 0000000..7cd3d36
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/agent/loop.py
@@ -0,0 +1,266 @@
+import json
+import logging
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, List, Dict, Any, Optional
+
+from langchain_core.tools import BaseTool
+
+from reasoning.base import BaseAgent
+from reasoning.messages.llm_client import LLMClient
+from reasoning.messages.openai_client import OpenAIChatClient
+from reasoning.model.reasoning_result import ReasoningResult
+from reasoning.tools.final_answer import FinalAnswerTool
+from reasoning.tools.python_exec import PythonExecTool
+from reasoning.tools.simulate_exec import SimulateTools
+from reasoning.utils.load_results import load_results
+from reasoning.helpers.prompts import _default_system_prompt, _append_tool_message
+from reasoning.helpers.chat_utils import prune_history
+from reasoning.config.tools import _openai_tools_spec
+from db.config.database import DatabaseConfig
+
+LOG_FMT = "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
+logging.basicConfig(level=logging.INFO, format=LOG_FMT, stream=sys.stdout)
+log = logging.getLogger("agent_loop")
+
+@dataclass
+class ReasoningAgent(BaseAgent):
+ model_id: str
+ db_config: DatabaseConfig = field(default_factory=lambda: DatabaseConfig())
+ db_path: str = ""
+ llm: LLMClient = field(default_factory=lambda: OpenAIChatClient(model="gpt-5-mini", temperature=1.0))
+ max_steps: int = 20
+ temperature: float = 1.0
+
+ # Hooks / callbacks (override as needed)
+ system_prompt_builder: Callable[[List[str]], str] = field(default=None)
+ history_pruner: Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]] = field(default=None)
+ report_store: Callable[[str, str, str, List[str]], None] = field(default=None) # (model_id, question, answer, images)
+
+ # Internal tool instances (bound to model/db and df when loaded)
+ _tools: Dict[str, BaseTool] = field(init=False, default_factory=dict)
+
+ def __post_init__(self) -> None:
+ # Set up database path
+ self.db_path = self.db_config.database_path
+
+ # Set defaults for optional callbacks
+ if self.system_prompt_builder is None:
+ self.system_prompt_builder = _default_system_prompt
+ if self.history_pruner is None:
+ self.history_pruner = prune_history
+ if self.report_store is None:
+ # Lazy import to avoid circulars; replace with your store_report
+ from db import store_report as _store_report # type: ignore
+ self.report_store = _store_report
+
+ # ── Public entrypoint ────────────────────────────────────────────────────
+ def ask(self, question: str, stop_flag: Optional[Callable[[], bool]] = None) -> ReasoningResult:
+ """Main entry point for asking questions to the reasoning agent."""
+ log.info("=== Starting analysis for model_id=%s ===", self.model_id)
+
+ # Initialize session
+ df, schema = self._load_context()
+ self._build_tools(df)
+ history = self._initialize_conversation(question, schema)
+
+ # Initialize tracking
+ tracking_state = self._initialize_tracking()
+
+ # Main reasoning loop
+ return self._reasoning_loop(history, tracking_state, stop_flag, question)
+
+ def _load_context(self) -> tuple:
+ """Load data context and schema."""
+ df = load_results(db_path=self.db_path, model_id=self.model_id)
+ schema = list(df.columns)
+ return df, schema
+
+ def _build_tools(self, df: Any) -> None:
+ """Build tools bound to this session."""
+ self._tools = {
+ "python_exec": PythonExecTool(df=df),
+ "run_simulation_for_model": SimulateTools(db_config=self.db_config, default_model_id=self.model_id),
+ "run_batch_for_model": SimulateTools(db_config=self.db_config, default_model_id=self.model_id),
+ "final_answer": FinalAnswerTool(),
+ }
+
+ def _initialize_conversation(self, question: str, schema: List[str]) -> List[Dict[str, Any]]:
+ """Initialize conversation history."""
+ return [
+ {"role": "system", "content": self.system_prompt_builder(schema)},
+ {"role": "user", "content": question},
+ ]
+
+ def _initialize_tracking(self) -> Dict[str, Any]:
+ """Initialize tracking state for images and code."""
+ return {
+ "seen_imgs": {p.name for p in Path.cwd().glob("*.png")},
+ "all_images": [],
+ "code_map": {},
+ "step_idx": 0
+ }
+
+ def _reasoning_loop(self, history: List[Dict[str, Any]], tracking_state: Dict[str, Any],
+ stop_flag: Optional[Callable[[], bool]], question: str) -> ReasoningResult:
+ """Main reasoning loop."""
+ tools_spec = _openai_tools_spec()
+
+ for _ in range(self.max_steps):
+ if stop_flag and stop_flag():
+ return self._create_result(history, tracking_state, "(stopped)")
+
+ # Get LLM response
+ msg = self.llm.chat(messages=self.history_pruner(history), tools=tools_spec)
+ assistant_entry = self._create_assistant_entry(msg)
+ history.append(assistant_entry)
+
+ # Process tool calls if any
+ if assistant_entry.get("tool_calls"):
+ final_result = self._process_tool_calls(
+ history, assistant_entry["tool_calls"], tracking_state, question
+ )
+ if final_result: # final_answer was called
+ return final_result
+ continue
+
+ # Nudge if no tool call
+ self._nudge_for_tool_call(history)
+
+ # Loop exhausted
+ log.error("Agent loop exhausted without an answer")
+ return self._create_result(history, tracking_state, "(no answer)")
+
+ def _create_assistant_entry(self, msg: Dict[str, Any]) -> Dict[str, Any]:
+ """Create assistant entry from LLM message."""
+ assistant_entry = {"role": "assistant", "content": msg.get("content", "")}
+ if "tool_calls" in msg:
+ assistant_entry["tool_calls"] = msg["tool_calls"]
+ return assistant_entry
+
+ def _process_tool_calls(self, history: List[Dict[str, Any]], tool_calls: List[Dict[str, Any]],
+ tracking_state: Dict[str, Any], question: str) -> Optional[ReasoningResult]:
+ """Process all tool calls in the assistant message."""
+ for tc in tool_calls:
+ call_id = tc["id"]
+ fname = tc["function"]["name"]
+ raw_args = tc["function"]["arguments"] or "{}"
+
+ # Parse arguments
+ args = self._parse_tool_args(history, call_id, raw_args)
+ if args is None:
+ continue
+
+ # Handle code tracking for python_exec
+ if fname == "python_exec":
+ if not self._track_python_code(history, call_id, args, tracking_state):
+ continue
+
+ # Execute tool
+ result = self._execute_tool(history, call_id, fname, args)
+ if result is None:
+ continue
+
+ # Track images
+ self._track_images(result, tracking_state)
+
+ # Handle final answer specially
+ if fname == "final_answer":
+ return self._handle_final_answer(history, call_id, result, tracking_state, question)
+
+ # Append tool result and continue
+ self._append_tool_message(history, call_id, result)
+
+ return None # Continue loop
+
+ def _parse_tool_args(self, history: List[Dict[str, Any]], call_id: str, raw_args: str) -> Optional[Dict[str, Any]]:
+ """Parse tool arguments from JSON string."""
+ try:
+ return json.loads(raw_args)
+ except json.JSONDecodeError:
+ self._append_tool_message(history, call_id, {"ok": False, "stderr": "Malformed tool arguments"})
+ return None
+
+ def _track_python_code(self, history: List[Dict[str, Any]], call_id: str,
+ args: Dict[str, Any], tracking_state: Dict[str, Any]) -> bool:
+ """Track Python code for python_exec calls."""
+ code = args.get("code", "")
+ if not isinstance(code, str) or not code.strip():
+ self._append_tool_message(history, call_id, {"ok": False, "stderr": "No code provided"})
+ return False
+
+ tracking_state["code_map"][tracking_state["step_idx"]] = code
+ tracking_state["step_idx"] += 1
+ return True
+
+ def _execute_tool(self, history: List[Dict[str, Any]], call_id: str,
+ fname: str, args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """Execute a tool and return its result."""
+ tool = self._tools.get(fname)
+ if not tool:
+ self._append_tool_message(history, call_id, {"ok": False, "stderr": f"Unknown tool '{fname}'"})
+ return None
+
+ try:
+ return tool._run(**args) # type: ignore[arg-type]
+ except TypeError as e:
+ result = {"ok": False, "stderr": f"Bad arguments: {e}"}
+ except Exception as e:
+ result = {"ok": False, "stderr": f"{type(e).__name__}: {e}"}
+
+ return result
+
+ def _track_images(self, result: Any, tracking_state: Dict[str, Any]) -> None:
+ """Track new images from tool results."""
+ if not isinstance(result, dict):
+ return
+
+ for img in result.get("images", []):
+ if img not in tracking_state["seen_imgs"]:
+ tracking_state["seen_imgs"].add(img)
+ tracking_state["all_images"].append(img)
+
+ def _handle_final_answer(self, history: List[Dict[str, Any]], call_id: str,
+ result: Dict[str, Any], tracking_state: Dict[str, Any],
+ question: str) -> ReasoningResult:
+ """Handle final_answer tool call."""
+ fa = result if isinstance(result, dict) else {}
+ answer_text = fa.get("answer", "")
+ merged_images = list({*tracking_state["all_images"], *fa.get("images", [])})
+
+ # Persist report
+ try:
+ self.report_store(self.model_id, question, answer_text, merged_images)
+ except Exception as e:
+ log.warning("store_report failed: %s", e)
+
+ # Echo tool message and return
+ self._append_tool_message(history, call_id, fa)
+ return ReasoningResult(
+ history=history,
+ code_map=tracking_state["code_map"],
+ answer=answer_text,
+ images=merged_images
+ )
+
+ def _nudge_for_tool_call(self, history: List[Dict[str, Any]]) -> None:
+ """Nudge the model to use a tool call."""
+ history.append({
+ "role": "user",
+ "content": "Please respond with a tool call (python_exec / run_simulation_for_model / run_batch_for_model / final_answer)."
+ })
+
+ def _create_result(self, history: List[Dict[str, Any]], tracking_state: Dict[str, Any],
+ answer: str) -> ReasoningResult:
+ """Create a ReasoningResult from current state."""
+ return ReasoningResult(
+ history=history,
+ code_map=tracking_state["code_map"],
+ answer=answer,
+ images=tracking_state["all_images"]
+ )
+
+ def _append_tool_message(self, history: List[Dict[str, Any]], call_id: str, payload: Any) -> None:
+ """Helper method to append tool messages to history."""
+ _append_tool_message(history, call_id, payload)
diff --git a/modules/research-framework/simexr_mod/reasoning/base.py b/modules/research-framework/simexr_mod/reasoning/base.py
new file mode 100644
index 0000000..afc9166
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/base.py
@@ -0,0 +1,94 @@
+"""
+Base classes for the reasoning module to provide common interfaces and inheritance.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Protocol, Optional, Callable
+from pathlib import Path
+
+
+class LLMClientProtocol(Protocol):
+ """Protocol for LLM client implementations."""
+
+ def chat(self, messages: List[Dict[str, Any]], tools: List[Dict[str, Any]]) -> Dict[str, Any]:
+ """Send messages to LLM and get response."""
+ ...
+
+
+class ReasoningToolProtocol(Protocol):
+ """Protocol for reasoning tool implementations."""
+
+ def _run(self, **kwargs) -> Dict[str, Any]:
+ """Execute the tool with given arguments."""
+ ...
+
+
+@dataclass
+class BaseAgent(ABC):
+ """Base class for all reasoning agent implementations."""
+
+ model_id: str
+ max_steps: int = 20
+
+ @abstractmethod
+ def ask(self, question: str, stop_flag: Optional[Callable[[], bool]] = None) -> Any:
+ """Process a question and return reasoning result."""
+ pass
+
+
+@dataclass
+class BaseClient(ABC):
+ """Base class for all LLM client implementations."""
+
+ model: str
+ temperature: float = 0.0
+
+ @abstractmethod
+ def chat(self, messages: List[Dict[str, Any]], tools: List[Dict[str, Any]]) -> Dict[str, Any]:
+ """Send chat request to LLM."""
+ pass
+
+
+@dataclass
+class BaseSimulationTool(ABC):
+ """Base class for simulation tool implementations."""
+
+ db_path: str
+ default_model_id: Optional[str] = None
+
+ @abstractmethod
+ def run_simulation_for_model(self, params: Dict[str, Any], model_id: Optional[str] = None) -> Dict[str, Any]:
+ """Run a single simulation."""
+ pass
+
+ @abstractmethod
+ def run_batch_for_model(self, grid: List[Dict[str, Any]], model_id: Optional[str] = None) -> List[Dict[str, Any]]:
+ """Run a batch of simulations."""
+ pass
+
+
+@dataclass
+class BaseUtility(ABC):
+ """Base class for utility functions."""
+
+ @abstractmethod
+ def process(self, data: Any) -> Any:
+ """Process data according to utility function."""
+ pass
+
+
+class HistoryManagerProtocol(Protocol):
+ """Protocol for conversation history management."""
+
+ def prune_history(self, messages: List[Dict[str, Any]], max_bundles: int = 2) -> List[Dict[str, Any]]:
+ """Prune conversation history to manageable size."""
+ ...
+
+
+class CodeExtractorProtocol(Protocol):
+ """Protocol for code extraction from conversation history."""
+
+ def extract_code_map(self, history: List[Dict[str, Any]]) -> Dict[int, str]:
+ """Extract code snippets from conversation history."""
+ ...
diff --git a/modules/research-framework/simexr_mod/reasoning/config/__init__.py b/modules/research-framework/simexr_mod/reasoning/config/__init__.py
new file mode 100644
index 0000000..36b266c
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/config/__init__.py
@@ -0,0 +1,5 @@
+"""Configuration module for reasoning tools."""
+
+from .tools import _openai_tools_spec
+
+__all__ = ["_openai_tools_spec"]
diff --git a/modules/research-framework/simexr_mod/reasoning/config/tools.py b/modules/research-framework/simexr_mod/reasoning/config/tools.py
new file mode 100644
index 0000000..ff03177
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/config/tools.py
@@ -0,0 +1,76 @@
+from typing import List, Dict, Any
+
+
+def _openai_tools_spec() -> List[Dict[str, Any]]:
+ """
+ Static tool JSON schema advertised to the LLM. Keep names in sync with tool instances.
+ """
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "python_exec",
+ "description": "Execute Python against the in-memory DataFrame `df` and matplotlib.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "code": {"type": "string",
+ "description": "Complete Python snippet that uses `df` and calls plt.show()."},
+ },
+ "required": ["code"],
+ "additionalProperties": False,
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "run_simulation_for_model",
+ "description": "Run ONE simulation for the bound model_id and append results to DB.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "params": {"type": "object", "description": "Kwargs for simulate(**params)."},
+ },
+ "required": ["params"],
+ "additionalProperties": True,
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "run_batch_for_model",
+ "description": "Run a small batch (≤ 24) of parameter dicts for the bound model_id and append results.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "grid": {
+ "type": "array",
+ "items": {"type": "object"},
+ "description": "Array of parameter dicts.",
+ },
+ },
+ "required": ["grid"],
+ "additionalProperties": False,
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "final_answer",
+ "description": "Return the final answer and optional values/images to finish.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "answer": {"type": "string"},
+ "values": {"type": "array", "items": {"type": "number"}},
+ "images": {"type": "array", "items": {"type": "string"}},
+ },
+ "required": ["answer"],
+ "additionalProperties": False,
+ },
+ },
+ },
+ ]
diff --git a/modules/research-framework/simexr_mod/reasoning/helpers/__init__.py b/modules/research-framework/simexr_mod/reasoning/helpers/__init__.py
new file mode 100644
index 0000000..229ed3e
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/helpers/__init__.py
@@ -0,0 +1,6 @@
+"""Helper modules for reasoning operations."""
+
+from .chat_utils import prune_history
+from .prompts import _default_system_prompt, _append_tool_message
+
+__all__ = ["prune_history", "_default_system_prompt", "_append_tool_message"]
diff --git a/modules/research-framework/simexr_mod/reasoning/helpers/chat_utils.py b/modules/research-framework/simexr_mod/reasoning/helpers/chat_utils.py
new file mode 100644
index 0000000..804d6da
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/helpers/chat_utils.py
@@ -0,0 +1,27 @@
+from typing import List, Dict, Any
+
+def prune_history(msgs: List[Dict[str, Any]], max_assistant_bundles: int = 2) -> List[Dict[str, Any]]:
+ # Keep first system, first user, and last N assistant+tool bundles.
+ sys = next((m for m in msgs if m["role"] == "system"), None)
+ usr = next((m for m in msgs if m["role"] == "user"), None)
+ # collect assistant bundles
+ bundles = []
+ i = 0
+ while i < len(msgs):
+ m = msgs[i]
+ if m.get("role") == "assistant":
+ b = [m]
+ j = i + 1
+ while j < len(msgs) and msgs[j].get("role") == "tool":
+ b.append(msgs[j])
+ j += 1
+ bundles.append(b)
+ i = j
+ else:
+ i += 1
+ out = []
+ if sys: out.append(sys)
+ if usr: out.append(usr)
+ for b in bundles[-max_assistant_bundles:]:
+ out.extend(b)
+ return out
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/reasoning/helpers/prompts.py b/modules/research-framework/simexr_mod/reasoning/helpers/prompts.py
new file mode 100644
index 0000000..ca29996
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/helpers/prompts.py
@@ -0,0 +1,29 @@
+import json
+from typing import List, Dict, Any
+
+
+def _default_system_prompt(schema: List[str]) -> str:
+ return f"""
+ You are a scientific reasoning assistant.
+
+ Only communicate using tool calls (no free text in assistant messages).
+ At each step call exactly one tool:
+
+ - python_exec(code: string) → run Python on in-memory `df` (call plt.show() to emit plots) with schema {schema}.
+ - run_simulation_for_model(params: object) → run ONE simulation and append results to DB.
+ - run_batch_for_model(grid: object[]) → run a small list of param dicts and append results.
+ - final_answer(answer: string, values?: number[], images?: string[]) → when DONE, return the final result.
+
+ Rules:
+ - Always send a tool call; never write prose or raw JSON in assistant content.
+ - For python_exec: provide a complete snippet that uses `df`; call plt.show() per figure.
+ - Ensure integer-only sizes (e.g., N) are integers in params.
+ - Keep grids modest (≤ 24).
+ """.strip()
+
+def _append_tool_message(history: List[Dict[str, Any]], call_id: str, payload: Any) -> None:
+ history.append({
+ "role": "tool",
+ "tool_call_id": call_id,
+ "content": json.dumps(payload),
+ })
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/reasoning/messages/__init__.py b/modules/research-framework/simexr_mod/reasoning/messages/__init__.py
new file mode 100644
index 0000000..295898f
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/messages/__init__.py
@@ -0,0 +1,7 @@
+"""Message handling for LLM communication."""
+
+from .llm_client import LLMClient
+from .model import ModelMessage
+from .openai_client import OpenAIChatClient
+
+__all__ = ["LLMClient", "ModelMessage", "OpenAIChatClient"]
diff --git a/modules/research-framework/simexr_mod/reasoning/messages/llm_client.py b/modules/research-framework/simexr_mod/reasoning/messages/llm_client.py
new file mode 100644
index 0000000..5e36f6e
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/messages/llm_client.py
@@ -0,0 +1,7 @@
+from typing import Protocol, List, Dict, Any
+
+from reasoning.messages.model import ModelMessage
+
+
+class LLMClient(Protocol):
+ def chat(self, messages: List[ModelMessage], tools: List[Dict[str, Any]]) -> ModelMessage: ...
diff --git a/modules/research-framework/simexr_mod/reasoning/messages/model.py b/modules/research-framework/simexr_mod/reasoning/messages/model.py
new file mode 100644
index 0000000..e05afa8
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/messages/model.py
@@ -0,0 +1,8 @@
+from typing import TypedDict
+
+
+class ModelMessage(TypedDict, total=False):
+ role: str
+ content: str
+ tool_calls: list # raw passthrough of SDK fields
+
diff --git a/modules/research-framework/simexr_mod/reasoning/messages/openai_client.py b/modules/research-framework/simexr_mod/reasoning/messages/openai_client.py
new file mode 100644
index 0000000..54025ea
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/messages/openai_client.py
@@ -0,0 +1,36 @@
+from typing import List, Dict, Any
+
+from reasoning.base import BaseClient
+from reasoning.messages.model import ModelMessage
+
+
+class OpenAIChatClient(BaseClient):
+ """
+ Wrapper to keep your code insulated from SDK changes.
+ """
+ def __init__(self, model: str = "gpt-5-mini", temperature: float = 1.0):
+ super().__init__(model=model)
+ import openai # local import to avoid hard dependency elsewhere
+ self._openai = openai
+
+ def chat(self, messages: List[ModelMessage], tools: List[Dict[str, Any]]) -> ModelMessage:
+ resp = self._openai.chat.completions.create(
+ model=self.model,
+ messages=messages,
+ tools=tools
+ )
+ msg = resp.choices[0].message
+ out: ModelMessage = {"role": "assistant", "content": msg.content or ""}
+ if getattr(msg, "tool_calls", None):
+ out["tool_calls"] = [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {
+ "name": tc.function.name,
+ "arguments": tc.function.arguments or "{}",
+ },
+ }
+ for tc in msg.tool_calls
+ ]
+ return out
diff --git a/modules/research-framework/simexr_mod/reasoning/model/__init__.py b/modules/research-framework/simexr_mod/reasoning/model/__init__.py
new file mode 100644
index 0000000..fa158eb
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/model/__init__.py
@@ -0,0 +1,5 @@
+"""Model classes for reasoning results."""
+
+from .reasoning_result import ReasoningResult
+
+__all__ = ["ReasoningResult"]
diff --git a/modules/research-framework/simexr_mod/reasoning/model/reasoning_result.py b/modules/research-framework/simexr_mod/reasoning/model/reasoning_result.py
new file mode 100644
index 0000000..c574778
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/model/reasoning_result.py
@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import List, Dict, Any
+
+
+@dataclass
+class ReasoningResult:
+ history: List[Dict[str, Any]]
+ code_map: Dict[int, str]
+ answer: str
+ images: List[str]
diff --git a/modules/research-framework/simexr_mod/reasoning/tools/__init__.py b/modules/research-framework/simexr_mod/reasoning/tools/__init__.py
new file mode 100644
index 0000000..dffc54d
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/tools/__init__.py
@@ -0,0 +1,7 @@
+"""Tool modules for reasoning operations."""
+
+from .final_answer import FinalAnswerTool
+from .python_exec import PythonExecTool
+from .simulate_exec import SimulateTools
+
+__all__ = ["FinalAnswerTool", "PythonExecTool", "SimulateTools"]
diff --git a/modules/research-framework/simexr_mod/reasoning/tools/final_answer.py b/modules/research-framework/simexr_mod/reasoning/tools/final_answer.py
new file mode 100644
index 0000000..cab0da7
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/tools/final_answer.py
@@ -0,0 +1,58 @@
+from pydantic import Field
+from typing import Optional, List, Dict, Any
+
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+
+class FinalAnswerArgs(BaseModel):
+ """Schema for the final answer payload."""
+ answer: str = Field(..., description="Final explanation/answer.")
+ values: Optional[List[float]] = Field(
+ default=None,
+ description="Optional numeric values to surface with the answer.",
+ )
+ images: Optional[List[str]] = Field(
+ default=None,
+ description="Optional image filenames to attach (e.g., generated plots).",
+ )
+
+
+class FinalAnswerTool(BaseTool):
+ """
+ Pseudo-tool used by the model to TERMINATE the session.
+
+ Behavior:
+ - Simply echoes the structured payload back to the agent:
+ { "answer": str, "values": list[float], "images": list[str] }
+ - The agent loop is responsible for:
+ * merging with any previously captured images
+ * calling store_report(model_id, question, answer, images)
+ * returning the final ReasoningResult
+
+ Keep this tool side-effect free; it's a signal for the agent to stop.
+ """
+ name: str = "final_answer"
+ description: str = "Return the final answer and optional values/images to finish the task."
+ args_schema: type = FinalAnswerArgs
+
+ def _run(
+ self,
+ answer: str,
+ values: Optional[List[float]] = None,
+ images: Optional[List[str]] = None,
+ ) -> Dict[str, Any]:
+ # Echo payload; agent consumes it and performs persistence/return.
+ return {
+ "answer": answer,
+ "values": values or [],
+ "images": images or [],
+ }
+
+ async def _arun(
+ self,
+ answer: str,
+ values: Optional[List[float]] = None,
+ images: Optional[List[str]] = None,
+ ) -> Dict[str, Any]:
+ # Async mirror if your agent uses async tooling.
+ return self._run(answer=answer, values=values, images=images)
diff --git a/modules/research-framework/simexr_mod/reasoning/tools/python_exec.py b/modules/research-framework/simexr_mod/reasoning/tools/python_exec.py
new file mode 100644
index 0000000..1b690c5
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/tools/python_exec.py
@@ -0,0 +1,167 @@
+import contextlib
+import io
+import traceback
+import textwrap
+import uuid
+import os
+import time
+from pathlib import Path
+from typing import Dict, Optional, Union, Any, List, Literal
+
+import matplotlib
+from matplotlib import pyplot as plt
+
+# Temporary imports - these functions need to be implemented or imported correctly
+try:
+ from core.script_utils import _capture_show, _media_dir_for, sanitize_metadata
+except ImportError:
+ # Mock implementations for now
+ def _capture_show(images_list):
+ return plt.show
+
+ def _media_dir_for(model_id):
+ return Path(f"media/{model_id}" if model_id else "media")
+
+ def sanitize_metadata(data, media_dir, media_paths, prefix=""):
+ return data
+
+matplotlib.use("Agg") # headless matplotlib
+import pandas as pd, json
+
+from pandas import DataFrame
+
+
+# tools.py
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field, PrivateAttr
+
+
+ExecMode = Literal["analysis", "simulate"]
+
+class PythonExecArgs(BaseModel):
+ code: Optional[str] = Field(default=None, description="Python source to execute.")
+ mode: ExecMode = Field(default="analysis", description="'analysis' or 'simulate'")
+ params: Optional[Dict[str, Any]] = Field(default=None, description="Kwargs for simulate(**params) when mode='simulate'")
+ model_id: Optional[str] = Field(default=None, description="Folder key for media grouping")
+ timeout_s: float = Field(default=30.0, description="Per-run timeout for simulate mode (soft check)")
+
+class PythonExecTool(BaseTool):
+ """
+ Unified tool:
+ - analysis mode: executes arbitrary Python against optional `df`
+ returns {ok, stdout, stderr, images}
+ - simulate mode: executes code that defines `def simulate(**params)->dict`
+ returns {ok, stdout, stderr, images, outputs, media_paths}
+ Backwards compatible with your previous usage if you pass only code.
+ """
+ name: str = Field("python_exec")
+ description: str = (
+ "Execute Python. In 'analysis' mode, runs code against DataFrame `df` if provided. "
+ "In 'simulate' mode, runs code that defines simulate(**params)->dict and returns "
+ "JSON-safe outputs with media paths. Keys: ok, stdout, stderr, images, [outputs, media_paths]."
+ )
+ _df: Optional[DataFrame] = PrivateAttr(default=None)
+
+ def __init__(self, df: Optional[DataFrame] = None):
+ super().__init__() # ensure BaseModel init
+ self._df = df
+
+ # LangChain calls _run with a dict of args (function calling)
+ def _run(self, args: Dict[str, Any]) -> dict:
+ payload = PythonExecArgs(**args)
+ if not payload.code:
+ return {"ok": False, "stdout": "", "stderr": "No code provided.", "images": []}
+ if payload.mode == "simulate":
+ return self.run_simulation(
+ code=payload.code,
+ params=payload.params or {},
+ model_id=payload.model_id,
+ timeout_s=payload.timeout_s,
+ )
+ # default: analysis
+ return self.run_python(code=payload.code, df=self._df)
+
+ # -------- analysis mode (unchanged behavior) --------
+ def run_python(self, code: str, df: Optional[pd.DataFrame]) -> Dict[str, Any]:
+ before = {f for f in os.listdir() if f.lower().endswith(".png")}
+ images: List[str] = []
+ old_show = _capture_show(images)
+
+ stdout_buf, stderr_buf = io.StringIO(), io.StringIO()
+ ok = True
+ g = {"plt": plt, "pd": pd, "np": __import__("numpy")}
+ if df is not None:
+ g["df"] = df
+
+ try:
+ with contextlib.redirect_stdout(stdout_buf), contextlib.redirect_stderr(stderr_buf):
+ exec(textwrap.dedent(code), g)
+ except Exception:
+ stderr_buf.write(traceback.format_exc())
+ ok = False
+ finally:
+ plt.show = old_show
+
+ after = {f for f in os.listdir() if f.lower().endswith(".png")}
+ new_images = sorted(after - before)
+ # also include images saved via our plt.show hook
+ for p in images:
+ if p not in new_images:
+ new_images.append(p)
+
+ return {"ok": ok, "stdout": stdout_buf.getvalue(), "stderr": stderr_buf.getvalue(), "images": new_images}
+
+ # # -------- simulate mode --------
+ # def run_simulation(
+ # self,
+ # code: str,
+ # params: Dict[str, Any],
+ # model_id: Optional[str] = None,
+ # timeout_s: float = 30.0,
+ # ) -> Dict[str, Any]:
+ # media_dir = _media_dir_for(model_id)
+ #
+ # before = {f for f in os.listdir() if f.lower().endswith(".png")}
+ # images: List[str] = []
+ # old_show = _capture_show(images)
+ #
+ # stdout_buf, stderr_buf = io.StringIO(), io.StringIO()
+ # ok, ret, err = True, None, ""
+ # g = {"plt": plt, "np": __import__("numpy")} # simulation shouldn't need df/pd by default
+ #
+ # start = time.time()
+ # try:
+ # with contextlib.redirect_stdout(stdout_buf), contextlib.redirect_stderr(stderr_buf):
+ # exec(textwrap.dedent(code), g)
+ # sim = g.get("simulate")
+ # if not callable(sim):
+ # raise RuntimeError("No callable `simulate(**params)` found.")
+ # ret = sim(**params)
+ # except Exception:
+ # ok = False
+ # err = traceback.format_exc()
+ # finally:
+ # plt.show = old_show
+ #
+ # elapsed = time.time() - start
+ # if elapsed > timeout_s:
+ # ok = False
+ # err = (err + "\n" if err else "") + f"Timeout exceeded: {elapsed:.1f}s > {timeout_s:.1f}s"
+ #
+ # after = {f for f in os.listdir() if f.lower().endswith(".png")}
+ # disk_new = sorted(after - before)
+ # for f in disk_new:
+ # if f not in images:
+ # images.append(f)
+ #
+ # media_paths: List[str] = []
+ # outputs = sanitize_metadata(ret, media_dir, media_paths, prefix="ret")
+ #
+ # return {
+ # "ok": ok,
+ # "stdout": stdout_buf.getvalue(),
+ # "stderr": err or stderr_buf.getvalue(),
+ # "images": images,
+ # "outputs": outputs,
+ # "media_paths": media_paths,
+ # }
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/reasoning/tools/simulate_exec.py b/modules/research-framework/simexr_mod/reasoning/tools/simulate_exec.py
new file mode 100644
index 0000000..2bb7cbd
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/tools/simulate_exec.py
@@ -0,0 +1,132 @@
+from __future__ import annotations
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import json
+
+from db.config.database import DatabaseConfig
+from db import get_simulation_path, store_simulation_results
+from reasoning.base import BaseSimulationTool
+
+# Import run_simulation function
+from execute.run.simulation_runner import SimulationRunner
+from execute.run.batch_runner import BatchRunner
+
+
+class SimulateTools(BaseSimulationTool):
+ """
+ Orchestrates simulation runs for a given model_id and persists results.
+
+ - Keeps a default model_id (optional).
+ - Resolves script paths through db.store.get_simulation_path.
+ - Executes via your existing run_simulation(script_path, params).
+ - Appends rows to DB via store_simulation_results.
+ - Provides JSON tool adapters for agent/function-calling integrations.
+ """
+
+ def __init__(self, db_config: DatabaseConfig, default_model_id: Optional[str] = None):
+ super().__init__(db_path=db_config.database_path, default_model_id=default_model_id)
+ self.db_config = db_config
+
+ # ------------------------ core helpers ------------------------
+
+ def resolve_script_path(self, model_id: Optional[str] = None) -> Path:
+ mid = model_id or self.default_model_id
+ if not mid:
+ raise ValueError("model_id is required (no default_model_id set).")
+ return Path(get_simulation_path(mid, db_path=self.db_path))
+
+ def run_simulation_for_model(self, params: Dict[str, Any], model_id: Optional[str] = None) -> Dict[str, Any]:
+ """
+ Resolve script for model_id, run simulate(**params), store a single row, and return it.
+ """
+ mid = model_id or self.default_model_id
+ if not mid:
+ raise ValueError("model_id is required (no default_model_id set).")
+
+ script_path = self.resolve_script_path(mid)
+ runner = SimulationRunner()
+ row = runner.run(script_path, params)
+
+ # Persist (append) this row to the model’s results
+ param_keys = list(params.keys())
+ store_simulation_results(model_id=mid, rows=[row], param_keys=param_keys, db_path=self.db_path)
+ return row
+
+ def run_batch_for_model(self, grid: List[Dict[str, Any]], model_id: Optional[str] = None) -> List[Dict[str, Any]]:
+ """
+ Run multiple param combinations for model_id and append all results to DB.
+ """
+ mid = model_id or self.default_model_id
+ if not mid:
+ raise ValueError("model_id is required (no default_model_id set).")
+
+ script_path = self.resolve_script_path(mid)
+ runner = SimulationRunner()
+ rows: List[Dict[str, Any]] = [runner.run(script_path, p) for p in grid]
+
+ param_keys = list(grid[0].keys()) if grid and isinstance(grid[0], dict) else []
+ if rows:
+ store_simulation_results(model_id=mid, rows=rows, param_keys=param_keys, db_path=self.db_path)
+ return rows
+
+ # ------------------------ optional utilities ------------------------
+
+ @staticmethod
+ def cartesian_grid(param_to_values: Dict[str, List[Any]]) -> List[Dict[str, Any]]:
+ """
+ Build a cartesian product grid: {'a':[1,2], 'b':[10]} -> [{'a':1,'b':10},{'a':2,'b':10}]
+ """
+ from itertools import product
+ keys = list(param_to_values.keys())
+ vals = [param_to_values[k] for k in keys]
+ return [dict(zip(keys, combo)) for combo in product(*vals)]
+
+ # ------------------------ tool adapters (for agents) ------------------------
+
+ def tool_run_simulation(self, payload_json: str) -> str:
+ """
+ JSON adapter for agents / function-calling.
+
+ Input JSON:
+ {
+ "model_id": "my_model_abc123", // optional if default_model_id set
+ "params": { "N": 20, "dt": 0.1 },
+ "db_path": "mcp.db" // optional: overrides ctor setting
+ }
+
+ Returns JSON: the stored row (includes _ok/_error fields, stdout/stderr, etc.).
+ """
+ payload = json.loads(payload_json)
+ model_id = payload.get("model_id") or self.default_model_id
+ if not model_id:
+ raise ValueError("tool_run_simulation: 'model_id' is required (no default_model_id set).")
+ if "db_path" in payload and payload["db_path"] != self.db_path:
+ self.db_path = payload["db_path"]
+
+ params = payload.get("params", {})
+ row = self.run_simulation_for_model(params=params, model_id=model_id)
+ return json.dumps(row)
+
+ def tool_run_batch(self, payload_json: str) -> str:
+ """
+ JSON adapter for agents / function-calling.
+
+ Input JSON:
+ {
+ "model_id": "my_model_abc123", // optional if default_model_id set
+ "grid": [ {"N": 10}, {"N": 20} ],
+ "db_path": "mcp.db" // optional: overrides ctor setting
+ }
+
+ Returns JSON: list of stored rows.
+ """
+ payload = json.loads(payload_json)
+ model_id = payload.get("model_id") or self.default_model_id
+ if not model_id:
+ raise ValueError("tool_run_batch: 'model_id' is required (no default_model_id set).")
+ if "db_path" in payload and payload["db_path"] != self.db_path:
+ self.db_path = payload["db_path"]
+
+ grid = payload.get("grid", [])
+ rows = self.run_batch_for_model(grid=grid, model_id=model_id)
+ return json.dumps(rows)
diff --git a/modules/research-framework/simexr_mod/reasoning/utils/__init__.py b/modules/research-framework/simexr_mod/reasoning/utils/__init__.py
new file mode 100644
index 0000000..9ff2ad8
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/utils/__init__.py
@@ -0,0 +1,8 @@
+"""Utility modules for reasoning support."""
+
+from .extract_code_map import extract_code_map
+from .history import prune_history
+from .json_utils import _safe_parse
+from .load_results import load_results
+
+__all__ = ["extract_code_map", "prune_history", "_safe_parse", "load_results"]
diff --git a/modules/research-framework/simexr_mod/reasoning/utils/extract_code_map.py b/modules/research-framework/simexr_mod/reasoning/utils/extract_code_map.py
new file mode 100644
index 0000000..aa1cf29
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/utils/extract_code_map.py
@@ -0,0 +1,20 @@
+import json
+from typing import List, Dict, Any
+
+def extract_code_map(history: List[Dict[str, Any]]) -> Dict[int, str]:
+ """
+ Scan the agent’s conversation history and pull out every python_exec
+ call’s `code` snippet, keyed by the step index in `history`.
+ """
+ code_map: Dict[int, str] = {}
+ for idx, entry in enumerate(history):
+ # assistant function‐call entries look like:
+ # {"role":"assistant", "function_call": {"name":"python_exec", "arguments": {"code": "..."}}, ...}
+ fc = entry.get("function_call")
+ if fc and isinstance(fc, dict):
+ args = fc.get("arguments", {})
+ args = json.loads(args)
+ code = args.get("code")
+ if code:
+ code_map[idx] = code
+ return code_map
diff --git a/modules/research-framework/simexr_mod/reasoning/utils/history.py b/modules/research-framework/simexr_mod/reasoning/utils/history.py
new file mode 100644
index 0000000..3f5d12b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/utils/history.py
@@ -0,0 +1,26 @@
+from typing import List, Dict, Any
+
+def prune_history(msgs: List[Dict[str, Any]], max_assistant_bundles: int = 2) -> List[Dict[str, Any]]:
+ """Prune conversation history to keep it manageable."""
+ sys = next((m for m in msgs if m["role"] == "system"), None)
+ usr = next((m for m in msgs if m["role"] == "user"), None)
+ bundles = []
+ i = 0
+ while i < len(msgs):
+ m = msgs[i]
+ if m.get("role") == "assistant":
+ b = [m]
+ j = i + 1
+ while j < len(msgs) and msgs[j].get("role") == "tool":
+ b.append(msgs[j])
+ j += 1
+ bundles.append(b)
+ i = j
+ else:
+ i += 1
+ out = []
+ if sys: out.append(sys)
+ if usr: out.append(usr)
+ for b in bundles[-max_assistant_bundles:]:
+ out.extend(b)
+ return out
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/reasoning/utils/json_utils.py b/modules/research-framework/simexr_mod/reasoning/utils/json_utils.py
new file mode 100644
index 0000000..55bdc2d
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/utils/json_utils.py
@@ -0,0 +1,9 @@
+import json
+
+
+def _safe_parse(d: str | None) -> dict:
+ try:
+ val = json.loads(d)
+ return val if isinstance(val, dict) else {}
+ except Exception:
+ return {}
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/reasoning/utils/load_results.py b/modules/research-framework/simexr_mod/reasoning/utils/load_results.py
new file mode 100644
index 0000000..d9f0ddc
--- /dev/null
+++ b/modules/research-framework/simexr_mod/reasoning/utils/load_results.py
@@ -0,0 +1,56 @@
+import sqlite3
+from pathlib import Path
+
+import pandas as pd
+
+from db.utils.json_utils import _safe_parse
+
+
+def load_results(
+ db_path: str | Path = "mcp.db",
+ model_id: str | None = None,
+) -> pd.DataFrame:
+ """
+ Load the `results` table, expand JSON columns, and optionally
+ filter to a single model_id.
+
+ Returns a DataFrame with columns:
+ model_id, ts, <all params fields>, <all output fields>
+ """
+ # 1) fetch raw rows
+ con = sqlite3.connect(str(db_path))
+ raw_df = pd.read_sql("SELECT model_id, params, outputs, ts FROM results", con)
+ con.close()
+
+ # print(raw_df.head())
+ # 2) optionally filter to only the given model_id
+ print("===========",model_id,"==========")
+ if model_id is not None:
+ raw_df = raw_df[raw_df["model_id"] == model_id]
+ # print(raw_df.head())
+ # 3) parse the JSON columns safely
+ raw_df["params"] = raw_df["params"].apply(_safe_parse)
+ raw_df["outputs"] = raw_df["outputs"].apply(_safe_parse)
+ # print(raw_df.head())
+ # 4) drop any rows where parsing failed (empty dict)
+ filtered = raw_df[
+ raw_df["params"].apply(bool) & raw_df["outputs"].apply(bool)
+ ].reset_index(drop=True)
+
+ # 5) expand the dict columns into separate DataFrame columns
+ params_df = pd.json_normalize(filtered["params"])
+ outputs_df = pd.json_normalize(filtered["outputs"])
+
+ # 6) concatenate model_id, ts, parameters, and outputs
+ print(params_df.head())
+ print(outputs_df.head())
+ final = pd.concat(
+ [
+ filtered[["model_id", "ts"]],
+ params_df,
+ outputs_df
+ ],
+ axis=1
+ )
+
+ return final
\ No newline at end of file
diff --git a/modules/research-framework/simexr_mod/requirements.txt b/modules/research-framework/simexr_mod/requirements.txt
new file mode 100644
index 0000000..b081c3b
--- /dev/null
+++ b/modules/research-framework/simexr_mod/requirements.txt
@@ -0,0 +1,43 @@
+# SimExR Framework Dependencies
+# Core Framework
+fastapi>=0.104.0
+uvicorn[standard]>=0.24.0
+pydantic>=2.5.0
+
+# OpenAI Integration
+openai>=1.3.0
+
+# Scientific Computing
+numpy>=1.24.0
+scipy>=1.11.0
+pandas>=2.1.0
+matplotlib>=3.7.0
+
+# Database
+sqlalchemy>=2.0.0
+
+# Progress Bars
+tqdm>=4.66.0
+
+# HTTP Client
+requests>=2.31.0
+httpx>=0.25.0
+
+# Code Processing
+ast>=0.0.2
+astunparse>=1.6.3
+
+# Utilities
+python-multipart>=0.0.6
+python-dotenv>=1.0.0
+pyyaml>=6.0.1
+
+# Optional: LangChain for advanced reasoning
+langchain>=0.1.0
+langchain-openai>=0.0.5
+
+# Development and Testing
+pytest>=7.4.0
+pytest-asyncio>=0.21.0
+black>=23.0.0
+flake8>=6.0.0
diff --git a/modules/research-framework/simexr_mod/setup.sh b/modules/research-framework/simexr_mod/setup.sh
new file mode 100755
index 0000000..a739694
--- /dev/null
+++ b/modules/research-framework/simexr_mod/setup.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# SimExR Framework Setup Script
+# This script automates the installation and configuration of the SimExR framework
+
+set -e # Exit on any error
+
+echo "🚀 SimExR Framework Setup"
+echo "=========================="
+
+# Check if Python is installed
+if ! command -v python3 &> /dev/null; then
+ echo "❌ Python 3 is not installed. Please install Python 3.8+ first."
+ exit 1
+fi
+
+# Check Python version
+PYTHON_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
+echo "✅ Python version: $PYTHON_VERSION"
+
+# Create virtual environment
+echo "📦 Creating virtual environment..."
+if [ -d "simexr_venv" ]; then
+ echo "⚠️ Virtual environment already exists. Removing..."
+ rm -rf simexr_venv
+fi
+
+python3 -m venv simexr_venv
+echo "✅ Virtual environment created"
+
+# Activate virtual environment
+echo "🔧 Activating virtual environment..."
+source simexr_venv/bin/activate
+
+# Upgrade pip
+echo "⬆️ Upgrading pip..."
+pip install --upgrade pip
+
+# Install dependencies
+echo "📚 Installing dependencies..."
+if [ -f "requirements.txt" ]; then
+ pip install -r requirements.txt
+else
+ echo "⚠️ requirements.txt not found. Installing common dependencies..."
+ pip install fastapi uvicorn openai pandas numpy scipy matplotlib tqdm sqlalchemy
+fi
+
+# Create config directory if it doesn't exist
+echo "⚙️ Setting up configuration..."
+mkdir -p utils
+
+# Create config file if it doesn't exist
+if [ ! -f "utils/config.yaml" ]; then
+ echo "📝 Creating config.yaml template..."
+ cat > utils/config.yaml << EOF
+# SimExR Configuration
+openai:
+ api_key: "your-openai-api-key-here"
+
+# Database configuration
+database:
+ path: "mcp.db"
+
+# Logging configuration
+logging:
+ level: "INFO"
+ format: "%(asctime)s | %(levelname)-8s | %(name)-20s | %(message)s"
+EOF
+ echo "✅ Configuration file created at utils/config.yaml"
+ echo "⚠️ Please update utils/config.yaml with your OpenAI API key"
+else
+ echo "✅ Configuration file already exists"
+fi
+
+# Create external_models directory
+echo "📁 Creating directories..."
+mkdir -p external_models
+mkdir -p systems/models
+mkdir -p logs
+
+# Set up database
+echo "🗄️ Setting up database..."
+if [ ! -f "mcp.db" ]; then
+ echo "✅ Database will be created on first run"
+else
+ echo "✅ Database already exists"
+fi
+
+# Test the installation
+echo "🧪 Testing installation..."
+python3 -c "
+import sys
+print('✅ Python path:', sys.executable)
+try:
+ import fastapi
+ print('✅ FastAPI installed')
+except ImportError:
+ print('❌ FastAPI not installed')
+ sys.exit(1)
+try:
+ import openai
+ print('✅ OpenAI installed')
+except ImportError:
+ print('❌ OpenAI not installed')
+ sys.exit(1)
+try:
+ import pandas
+ print('✅ Pandas installed')
+except ImportError:
+ print('❌ Pandas not installed')
+ sys.exit(1)
+print('✅ All core dependencies installed successfully')
+"
+
+echo ""
+echo "🎉 Setup completed successfully!"
+echo ""
+echo "📋 Next steps:"
+echo "1. Update utils/config.yaml with your OpenAI API key"
+echo "2. Activate the virtual environment: source simexr_venv/bin/activate"
+echo "3. Start the API server: python start_api.py --host 127.0.0.1 --port 8001"
+echo "4. Visit http://127.0.0.1:8001/docs for API documentation"
+echo ""
+echo "🔗 Quick start commands:"
+echo "source simexr_venv/bin/activate"
+echo "python start_api.py --host 127.0.0.1 --port 8001"
+echo ""
+echo "📖 For more information, see README.md"
diff --git a/modules/research-framework/simexr_mod/start_api.py b/modules/research-framework/simexr_mod/start_api.py
new file mode 100755
index 0000000..ed89ae6
--- /dev/null
+++ b/modules/research-framework/simexr_mod/start_api.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+SimExR API Startup Script
+
+This script starts the SimExR API server with proper configuration.
+"""
+
+import sys
+import os
+import argparse
+from pathlib import Path
+
+# Add project root to Python path
+project_root = Path(__file__).parent
+sys.path.insert(0, str(project_root))
+
+
+def main():
+ # Clear any cached environment variables and set OpenAI API key globally
+ print("🧹 Clearing environment variable cache...")
+
+ # Clear any existing OpenAI-related environment variables
+ openai_vars_to_clear = [
+ "OPENAI_API_KEY", "OPENAI_API_KEY_OLD", "OPENAI_API_KEY_CACHE",
+ "PYTHONPATH", "PYTHONHOME", "PYTHONUNBUFFERED"
+ ]
+
+ for var in openai_vars_to_clear:
+ if var in os.environ:
+ old_value = os.environ.pop(var)
+ print(f"🗑️ Cleared {var}: {old_value[:20] if old_value else 'None'}...")
+
+ # Force reload any cached modules that might have old API keys
+ import importlib
+ modules_to_reload = ['openai', 'utils.config']
+ for module_name in modules_to_reload:
+ if module_name in sys.modules:
+ importlib.reload(sys.modules[module_name])
+ print(f"🔄 Reloaded module: {module_name}")
+
+ # Now set the OpenAI API key from config using the dedicated module
+ try:
+ from utils.openai_config import ensure_openai_api_key
+ api_key = ensure_openai_api_key()
+ print("✅ OpenAI API key configuration completed successfully")
+ except Exception as e:
+ print(f"❌ Error configuring OpenAI API key: {e}")
+ api_key = None
+
+ parser = argparse.ArgumentParser(description="Start SimExR API Server")
+ parser.add_argument("--host", default="0.0.0.0", help="Host to bind to")
+ parser.add_argument("--port", type=int, default=8000, help="Port to bind to")
+ parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development")
+ parser.add_argument("--workers", type=int, default=1, help="Number of worker processes")
+ parser.add_argument("--log-level", default="info", help="Log level")
+ parser.add_argument("--db-path", help="Path to database file")
+
+ args = parser.parse_args()
+
+ # Set environment variables
+ if args.db_path:
+ os.environ["SIMEXR_DATABASE_PATH"] = args.db_path
+
+ # Import after setting environment
+ import uvicorn
+ from api.main import app
+
+ print("🚀 Starting SimExR API Server")
+ print(f"📡 Host: {args.host}:{args.port}")
+ print(f"🔄 Reload: {args.reload}")
+ print(f"👥 Workers: {args.workers}")
+ print(f"📊 Database: {os.environ.get('SIMEXR_DATABASE_PATH', 'default')}")
+ print(f"📖 Docs: http://{args.host}:{args.port}/docs")
+ print()
+
+ # Start server
+ uvicorn.run(
+ "api.main:app",
+ host=args.host,
+ port=args.port,
+ reload=args.reload,
+ workers=args.workers if not args.reload else 1, # reload mode requires single worker
+ log_level=args.log_level,
+ access_log=True
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/modules/research-framework/simexr_mod/start_streamlit.py b/modules/research-framework/simexr_mod/start_streamlit.py
new file mode 100755
index 0000000..23d84bd
--- /dev/null
+++ b/modules/research-framework/simexr_mod/start_streamlit.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+SimExR Streamlit App Launcher
+
+This script starts the Streamlit app with the correct configuration.
+Make sure the API server is running before starting the Streamlit app.
+"""
+
+import subprocess
+import sys
+import time
+import requests
+from pathlib import Path
+
+def check_api_server():
+ """Check if the API server is running."""
+ try:
+ response = requests.get("http://127.0.0.1:8000/health/status", timeout=5)
+ return response.status_code == 200
+ except:
+ return False
+
+def main():
+ print("🚀 SimExR Streamlit App Launcher")
+ print("=" * 40)
+
+ # Check if API server is running
+ print("🔍 Checking API server status...")
+ if not check_api_server():
+ print("❌ API server is not running!")
+ print("💡 Please start the API server first with:")
+ print(" python start_api.py --host 127.0.0.1 --port 8000")
+ print("\n🔄 Starting API server automatically...")
+
+ # Try to start the API server
+ try:
+ api_process = subprocess.Popen([
+ sys.executable, "start_api.py", "--host", "127.0.0.1", "--port", "8000"
+ ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ # Wait for server to start
+ print("⏳ Waiting for API server to start...")
+ for i in range(30): # Wait up to 30 seconds
+ time.sleep(1)
+ if check_api_server():
+ print("✅ API server started successfully!")
+ break
+ else:
+ print("❌ Failed to start API server")
+ return 1
+
+ except Exception as e:
+ print(f"❌ Error starting API server: {e}")
+ return 1
+ else:
+ print("✅ API server is running!")
+
+ # Check if app.py exists
+ if not Path("app.py").exists():
+ print("❌ app.py not found!")
+ print("💡 Make sure you're in the correct directory")
+ return 1
+
+ # Start Streamlit app
+ print("\n🌐 Starting Streamlit app...")
+ print("📱 The app will be available at: http://localhost:8501")
+ print("🔗 API server: http://127.0.0.1:8000")
+ print("\n" + "=" * 40)
+
+ try:
+ # Start Streamlit with the app
+ subprocess.run([
+ sys.executable, "-m", "streamlit", "run", "app.py",
+ "--server.port", "8501",
+ "--server.address", "localhost",
+ "--browser.gatherUsageStats", "false"
+ ])
+ except KeyboardInterrupt:
+ print("\n🛑 Streamlit app stopped by user")
+ except Exception as e:
+ print(f"❌ Error starting Streamlit app: {e}")
+ return 1
+
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/modules/research-framework/simexr_mod/test_all_apis.py b/modules/research-framework/simexr_mod/test_all_apis.py
new file mode 100755
index 0000000..8e42669
--- /dev/null
+++ b/modules/research-framework/simexr_mod/test_all_apis.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python3
+"""
+SimExR Framework - Complete API Testing Script
+
+This script demonstrates all the APIs and their functionality with real examples.
+Run this script to test the complete workflow from GitHub import to AI analysis.
+"""
+
+import requests
+import json
+import time
+import sys
+from typing import Dict, Any, List
+
+# Configuration
+BASE_URL = "http://127.0.0.1:8000"
+GITHUB_URL = "https://github.com/vash02/physics-systems-dataset/blob/main/vanderpol.py"
+MODEL_NAME = "vanderpol_transform_test"
+
+class SimExRAPITester:
+ def __init__(self, base_url: str = BASE_URL):
+ self.base_url = base_url
+ self.model_id = None
+ self.test_results = {}
+
+ def print_header(self, title: str):
+ """Print a formatted header."""
+ print(f"\n{'='*60}")
+ print(f"🧪 {title}")
+ print(f"{'='*60}")
+
+ def print_success(self, message: str):
+ """Print a success message."""
+ print(f"✅ {message}")
+
+ def print_error(self, message: str):
+ """Print an error message."""
+ print(f"❌ {message}")
+
+ def print_info(self, message: str):
+ """Print an info message."""
+ print(f"ℹ️ {message}")
+
+ def make_request(self, method: str, endpoint: str, data: Dict = None) -> Dict:
+ """Make an HTTP request and return the response."""
+ url = f"{self.base_url}{endpoint}"
+ headers = {"Content-Type": "application/json"}
+
+ try:
+ if method.upper() == "GET":
+ response = requests.get(url, headers=headers)
+ elif method.upper() == "POST":
+ response = requests.post(url, headers=headers, json=data)
+ elif method.upper() == "DELETE":
+ response = requests.delete(url, headers=headers)
+ else:
+ raise ValueError(f"Unsupported method: {method}")
+
+ response.raise_for_status()
+ return response.json()
+
+ except requests.exceptions.RequestException as e:
+ self.print_error(f"Request failed: {e}")
+ return {"error": str(e)}
+
+ def test_health_api(self) -> bool:
+ """Test the health check API."""
+ self.print_header("Testing Health API")
+
+ # Test health status
+ result = self.make_request("GET", "/health/status")
+ if "error" not in result:
+ self.print_success("Health status API working")
+ self.print_info(f"Status: {result.get('status', 'unknown')}")
+ return True
+ else:
+ self.print_error("Health status API failed")
+ return False
+
+ def test_github_import(self) -> bool:
+ """Test GitHub script import and transformation."""
+ self.print_header("Testing GitHub Import & Transformation")
+
+ data = {
+ "github_url": GITHUB_URL,
+ "model_name": MODEL_NAME,
+ "max_smoke_iters": 3
+ }
+
+ self.print_info(f"Importing from: {GITHUB_URL}")
+ result = self.make_request("POST", "/simulation/transform/github", data)
+
+ if "error" not in result and "model_id" in result:
+ self.model_id = result["model_id"]
+ self.print_success(f"Successfully imported model: {self.model_id}")
+ self.print_info(f"Model name: {result.get('model_name', 'unknown')}")
+ self.print_info(f"Script path: {result.get('script_path', 'unknown')}")
+ return True
+ else:
+ self.print_error("GitHub import failed")
+ return False
+
+ def test_single_simulation(self) -> bool:
+ """Test single simulation execution."""
+ self.print_header("Testing Single Simulation")
+
+ if not self.model_id:
+ self.print_error("No model ID available")
+ return False
+
+ data = {
+ "model_id": self.model_id,
+ "parameters": {
+ "mu": 1.5,
+ "z0": [1.5, 0.5],
+ "eval_time": 25,
+ "t_iteration": 250,
+ "plot": False
+ }
+ }
+
+ self.print_info("Running single simulation...")
+ result = self.make_request("POST", "/simulation/run", data)
+
+ if "error" not in result and "status" in result:
+ self.print_success("Single simulation completed")
+ self.print_info(f"Status: {result.get('status', 'unknown')}")
+ self.print_info(f"Execution time: {result.get('execution_time', 'unknown')}s")
+ return True
+ else:
+ self.print_error("Single simulation failed")
+ return False
+
+ def test_batch_simulation(self) -> bool:
+ """Test batch simulation execution."""
+ self.print_header("Testing Batch Simulation")
+
+ if not self.model_id:
+ self.print_error("No model ID available")
+ return False
+
+ data = {
+ "model_id": self.model_id,
+ "parameter_grid": [
+ {
+ "mu": 1.0,
+ "z0": [2, 0],
+ "eval_time": 30,
+ "t_iteration": 300,
+ "plot": False
+ },
+ {
+ "mu": 1.5,
+ "z0": [1.5, 0.5],
+ "eval_time": 25,
+ "t_iteration": 250,
+ "plot": False
+ }
+ ]
+ }
+
+ self.print_info("Running batch simulation...")
+ result = self.make_request("POST", "/simulation/batch", data)
+
+ if "error" not in result and "results" in result:
+ self.print_success("Batch simulation completed")
+ self.print_info(f"Total simulations: {len(result.get('results', []))}")
+ successful = sum(1 for r in result.get('results', []) if r.get('status') == 'completed')
+ self.print_info(f"Successful: {successful}")
+ return True
+ else:
+ self.print_error("Batch simulation failed")
+ return False
+
+ def test_model_management(self) -> bool:
+ """Test model management APIs."""
+ self.print_header("Testing Model Management APIs")
+
+ # Test list models
+ self.print_info("Testing list models...")
+ result = self.make_request("GET", "/simulation/models")
+ if "error" not in result and "models" in result:
+ self.print_success(f"Listed {len(result['models'])} models")
+ else:
+ self.print_error("List models failed")
+ return False
+
+ # Test fuzzy search
+ self.print_info("Testing fuzzy search...")
+ result = self.make_request("GET", "/simulation/models/search?name=vanderpol&limit=3")
+ if "error" not in result and "models" in result:
+ self.print_success(f"Found {len(result['models'])} matching models")
+ else:
+ self.print_error("Fuzzy search failed")
+ return False
+
+ # Test get model info
+ if self.model_id:
+ self.print_info("Testing get model info...")
+ result = self.make_request("GET", f"/simulation/models/{self.model_id}")
+ if "error" not in result and "model" in result:
+ self.print_success("Retrieved model information")
+ else:
+ self.print_error("Get model info failed")
+ return False
+
+ return True
+
+ def test_results_apis(self) -> bool:
+ """Test results retrieval APIs."""
+ self.print_header("Testing Results APIs")
+
+ if not self.model_id:
+ self.print_error("No model ID available")
+ return False
+
+ # Test get model results
+ self.print_info("Testing get model results...")
+ result = self.make_request("GET", f"/simulation/models/{self.model_id}/results?limit=5")
+ if "error" not in result and "results" in result:
+ self.print_success(f"Retrieved {len(result['results'])} results")
+ self.print_info(f"Total count: {result.get('total_count', 0)}")
+ else:
+ self.print_error("Get model results failed")
+ return False
+
+ # Test database results
+ self.print_info("Testing database results...")
+ result = self.make_request("GET", f"/database/results?model_id={self.model_id}&limit=3")
+ if "error" not in result and "results" in result:
+ self.print_success(f"Retrieved {len(result['results'])} database results")
+ else:
+ self.print_error("Database results failed")
+ return False
+
+ return True
+
+ def test_reasoning_apis(self) -> bool:
+ """Test AI reasoning APIs."""
+ self.print_header("Testing AI Reasoning APIs")
+
+ if not self.model_id:
+ self.print_error("No model ID available")
+ return False
+
+ # Test ask reasoning question
+ self.print_info("Testing AI reasoning...")
+ data = {
+ "model_id": self.model_id,
+ "question": "What is the behavior of the van der Pol oscillator for mu=1.0 and mu=1.5? How do the trajectories differ?",
+ "max_steps": 3
+ }
+
+ result = self.make_request("POST", "/reasoning/ask", data)
+ if "error" not in result and "answer" in result:
+ self.print_success("AI reasoning completed")
+ self.print_info(f"Execution time: {result.get('execution_time', 'unknown')}s")
+ self.print_info(f"Answer length: {len(result.get('answer', ''))} characters")
+ else:
+ self.print_error("AI reasoning failed")
+ return False
+
+ # Test get reasoning history
+ self.print_info("Testing reasoning history...")
+ result = self.make_request("GET", f"/reasoning/history/{self.model_id}?limit=3")
+ if "error" not in result and "conversations" in result:
+ self.print_success(f"Retrieved {len(result['conversations'])} conversations")
+ else:
+ self.print_error("Reasoning history failed")
+ return False
+
+ # Test get all conversations
+ self.print_info("Testing all conversations...")
+ result = self.make_request("GET", "/reasoning/conversations?limit=5")
+ if "error" not in result and "conversations" in result:
+ self.print_success(f"Retrieved {len(result['conversations'])} total conversations")
+ else:
+ self.print_error("All conversations failed")
+ return False
+
+ # Test reasoning statistics
+ self.print_info("Testing reasoning statistics...")
+ result = self.make_request("GET", "/reasoning/stats")
+ if "error" not in result and "total_conversations" in result:
+ self.print_success("Retrieved reasoning statistics")
+ self.print_info(f"Total conversations: {result.get('total_conversations', 0)}")
+ self.print_info(f"Unique models: {result.get('unique_models', 0)}")
+ else:
+ self.print_error("Reasoning statistics failed")
+ return False
+
+ return True
+
+ def test_database_apis(self) -> bool:
+ """Test database read-only APIs."""
+ self.print_header("Testing Database APIs")
+
+ # Test database stats
+ self.print_info("Testing database stats...")
+ result = self.make_request("GET", "/database/stats")
+ if "error" not in result:
+ self.print_success("Retrieved database statistics")
+ self.print_info(f"Total models: {result.get('total_models', 0)}")
+ self.print_info(f"Total results: {result.get('total_results', 0)}")
+ else:
+ self.print_error("Database stats failed")
+ return False
+
+ # Test database models
+ self.print_info("Testing database models...")
+ result = self.make_request("GET", "/database/models?limit=5")
+ if "error" not in result and "models" in result:
+ self.print_success(f"Retrieved {len(result['models'])} database models")
+ else:
+ self.print_error("Database models failed")
+ return False
+
+ return True
+
+ def run_complete_test(self) -> Dict[str, Any]:
+ """Run the complete API test suite."""
+ self.print_header("SimExR Framework - Complete API Test Suite")
+
+ tests = [
+ ("Health API", self.test_health_api),
+ ("GitHub Import", self.test_github_import),
+ ("Single Simulation", self.test_single_simulation),
+ ("Batch Simulation", self.test_batch_simulation),
+ ("Model Management", self.test_model_management),
+ ("Results APIs", self.test_results_apis),
+ ("AI Reasoning", self.test_reasoning_apis),
+ ("Database APIs", self.test_database_apis),
+ ]
+
+ results = {}
+ total_tests = len(tests)
+ passed_tests = 0
+
+ for test_name, test_func in tests:
+ try:
+ success = test_func()
+ results[test_name] = success
+ if success:
+ passed_tests += 1
+ time.sleep(1) # Brief pause between tests
+ except Exception as e:
+ self.print_error(f"Test {test_name} failed with exception: {e}")
+ results[test_name] = False
+
+ # Print summary
+ self.print_header("Test Results Summary")
+ print(f"📊 Total Tests: {total_tests}")
+ print(f"✅ Passed: {passed_tests}")
+ print(f"❌ Failed: {total_tests - passed_tests}")
+ print(f"📈 Success Rate: {(passed_tests/total_tests)*100:.1f}%")
+
+ print("\n📋 Detailed Results:")
+ for test_name, success in results.items():
+ status = "✅ PASS" if success else "❌ FAIL"
+ print(f" {status} {test_name}")
+
+ return {
+ "total_tests": total_tests,
+ "passed_tests": passed_tests,
+ "failed_tests": total_tests - passed_tests,
+ "success_rate": (passed_tests/total_tests)*100,
+ "results": results,
+ "model_id": self.model_id
+ }
+
+def main():
+ """Main function to run the API tests."""
+ print("🚀 SimExR Framework API Testing")
+ print("=================================")
+
+ # Check if server is running
+ try:
+ response = requests.get(f"{BASE_URL}/health/status", timeout=5)
+ if response.status_code != 200:
+ print(f"❌ Server is not responding properly. Status code: {response.status_code}")
+ sys.exit(1)
+ except requests.exceptions.RequestException:
+ print(f"❌ Cannot connect to server at {BASE_URL}")
+ print("💡 Make sure the server is running with: python start_api.py --host 127.0.0.1 --port 8000")
+ sys.exit(1)
+
+ # Run tests
+ tester = SimExRAPITester()
+ results = tester.run_complete_test()
+
+ # Save results
+ with open("test_results.json", "w") as f:
+ json.dump(results, f, indent=2)
+
+ print(f"\n📄 Test results saved to test_results.json")
+
+ if results["passed_tests"] == results["total_tests"]:
+ print("\n🎉 All tests passed! The SimExR framework is working perfectly.")
+ sys.exit(0)
+ else:
+ print(f"\n⚠️ {results['failed_tests']} test(s) failed. Please check the logs above.")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
diff --git a/modules/research-framework/simexr_mod/utils/config.py b/modules/research-framework/simexr_mod/utils/config.py
new file mode 100644
index 0000000..ab97432
--- /dev/null
+++ b/modules/research-framework/simexr_mod/utils/config.py
@@ -0,0 +1,153 @@
+# utils/config.py
+import os
+import yaml
+import importlib
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+
+class Settings:
+ def __init__(self, file_name: str = "config.yaml"):
+ # Clear cached environment variables at initialization
+ self._clear_cached_env_vars()
+
+ # Candidate directories to search (in order)
+ self._roots = [
+ Path.cwd(),
+ Path(__file__).resolve().parent.parent, # project root
+ Path(__file__).resolve().parent, # module folder
+ ]
+
+ self._cfg: Dict[str, Any] = {}
+ self._config_path: Path | None = None
+ for root in self._roots:
+ p = root / file_name
+ if p.is_file():
+ try:
+ self._cfg = yaml.safe_load(p.read_text()) or {}
+ self._config_path = p
+ break
+ except Exception:
+ print(f"⚠️ Failed to parse {p}")
+ else:
+ print(f"⚠️ `{file_name}` not found in {self._roots}. Falling back to env vars.")
+
+ # ---------- helpers ----------
+ def _clear_cached_env_vars(self):
+ """Clear any cached or conflicting environment variables."""
+ # List of environment variables that might conflict with our config
+ env_vars_to_clear = [
+ "OPENAI_API_KEY_OLD", "OPENAI_API_KEY_CACHE", "OPENAI_API_KEY_BACKUP",
+ "PYTHONPATH", "PYTHONHOME", "PYTHONUNBUFFERED",
+ "SIMEXR_OPENAI_API_KEY", "SIMEXR_CONFIG_CACHE"
+ ]
+
+ cleared_vars = []
+ for var in env_vars_to_clear:
+ if var in os.environ:
+ old_value = os.environ.pop(var)
+ cleared_vars.append(f"{var}: {old_value[:20] if old_value else 'None'}")
+
+ if cleared_vars:
+ print(f"🧹 Cleared cached environment variables: {', '.join(cleared_vars)}")
+
+ def check_env_vars(self) -> Dict[str, Any]:
+ """Check current environment variables and return a summary."""
+ env_summary = {}
+
+ # Check OpenAI-related environment variables
+ openai_vars = {k: v for k, v in os.environ.items() if 'openai' in k.lower() or 'api_key' in k.lower()}
+ if openai_vars:
+ env_summary['openai_vars'] = {k: v[:20] + '...' if v and len(v) > 20 else v for k, v in openai_vars.items()}
+
+ # Check Python-related environment variables
+ python_vars = {k: v for k, v in os.environ.items() if 'python' in k.lower()}
+ if python_vars:
+ env_summary['python_vars'] = {k: v[:20] + '...' if v and len(v) > 20 else v for k, v in python_vars.items()}
+
+ return env_summary
+
+ def clear_specific_env_var(self, var_name: str) -> bool:
+ """Clear a specific environment variable if it exists."""
+ if var_name in os.environ:
+ old_value = os.environ.pop(var_name)
+ print(f"🗑️ Cleared {var_name}: {old_value[:20] if old_value else 'None'}")
+ return True
+ return False
+
+ def reload_modules(self, module_names: list = None):
+ """Reload specified modules to clear any cached configurations."""
+ if module_names is None:
+ module_names = ['openai', 'utils.config', 'utils.openai_config']
+
+ reloaded = []
+ for module_name in module_names:
+ if module_name in sys.modules:
+ try:
+ importlib.reload(sys.modules[module_name])
+ reloaded.append(module_name)
+ except Exception as e:
+ print(f"⚠️ Could not reload {module_name}: {e}")
+
+ if reloaded:
+ print(f"🔄 Reloaded modules: {', '.join(reloaded)}")
+
+ @property
+ def project_root(self) -> Path:
+ # pick the second entry from _roots (your intended project root)
+ return self._roots[1]
+
+ def _get(self, *keys: str, default: Any = None) -> Any:
+ """Safe nested lookup: _get('database','path', default=None)"""
+ cur = self._cfg
+ for k in keys:
+ if not isinstance(cur, dict) or k not in cur:
+ return default
+ cur = cur[k]
+ return cur
+
+ # ---------- keys ----------
+ @property
+ def openai_api_key(self) -> str:
+ # 1) config.yaml
+ key = self._get("openai", "api_key")
+ if key:
+ return key
+ # 2) env var
+ return os.environ.get("OPENAI_API_KEY", "") or ""
+
+ @property
+ def db_path(self) -> Path:
+ """
+ Database path priority:
+ 1) config.yaml: database.path
+ 2) env: SIMEXR_DB_PATH
+ 3) default: <project_root>/mcp.db
+ """
+ from_env = os.environ.get("SIMEXR_DB_PATH")
+ val = self._get("database", "path") or from_env
+ if not val:
+ val = str(self.project_root / "mcp.db")
+ p = Path(val).expanduser()
+ # Don't force-create here; let callers decide. Just normalize to absolute.
+ return p if p.is_absolute() else p.resolve()
+
+ @property
+ def media_root(self) -> Path:
+ """
+ Root directory for saving figures/animations, if you want one:
+ 1) config.yaml: media.root
+ 2) env: SIMEXR_MEDIA_ROOT
+ 3) default: <project_root>/results_media
+ """
+ from_env = os.environ.get("SIMEXR_MEDIA_ROOT")
+ val = self._get("media", "root") or from_env
+ if not val:
+ val = str(self.project_root / "results_media")
+ p = Path(val).expanduser()
+ return p if p.is_absolute() else p.resolve()
+
+
+# a singleton you can import everywhere
+settings = Settings()
diff --git a/modules/research-framework/simexr_mod/utils/logger.py b/modules/research-framework/simexr_mod/utils/logger.py
new file mode 100644
index 0000000..24da92e
--- /dev/null
+++ b/modules/research-framework/simexr_mod/utils/logger.py
@@ -0,0 +1,11 @@
+# delete the failing import
+# from utils.logger import setup_logging
+
+import logging
+
+def setup_logging(name: str):
+ logging.basicConfig(level=logging.INFO,
+ format="%(asctime)s %(levelname)s %(message)s")
+ return logging.getLogger(name)
+
+logger = setup_logging("sandbox_executor")
diff --git a/modules/research-framework/simexr_mod/utils/openai_config.py b/modules/research-framework/simexr_mod/utils/openai_config.py
new file mode 100644
index 0000000..ad5bd60
--- /dev/null
+++ b/modules/research-framework/simexr_mod/utils/openai_config.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""
+OpenAI API Key Configuration Manager
+
+This module ensures the OpenAI API key is properly set and available throughout the application.
+"""
+
+import os
+import openai
+from .config import settings
+
+def ensure_openai_api_key():
+ """
+ Ensure the OpenAI API key is set globally.
+ This function should be called at startup and whenever the API key needs to be refreshed.
+ """
+ # Get API key from config
+ api_key = settings.openai_api_key
+
+ if not api_key:
+ raise ValueError("No OpenAI API key found in configuration")
+
+ # Set in environment
+ os.environ["OPENAI_API_KEY"] = api_key
+
+ # Set in openai module
+ openai.api_key = api_key
+
+ print(f"🔑 OpenAI API key configured globally: {api_key[:20]}...")
+
+ return api_key
+
+def get_openai_api_key():
+ """
+ Get the current OpenAI API key.
+ """
+ return openai.api_key or os.environ.get("OPENAI_API_KEY")
+
+def is_openai_configured():
+ """
+ Check if OpenAI is properly configured.
+ """
+ api_key = get_openai_api_key()
+ return bool(api_key and api_key.startswith("sk-"))
+
+# Initialize at module import
+try:
+ ensure_openai_api_key()
+except Exception as e:
+ print(f"⚠️ Warning: Could not initialize OpenAI API key: {e}")
