scheduler/scripts/validate-full-functionality.sh

#!/bin/bash

# Complete cold-start validation script
# This script performs a full validation of the airavata-scheduler system
# including cold-start setup, unit tests, and integration tests

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
UNIT_TEST_TIMEOUT="30m"
INTEGRATION_TEST_TIMEOUT="60m"
COLD_START_TIMEOUT="10m"

# Test result files
UNIT_TEST_RESULTS="unit-test-results.log"
INTEGRATION_TEST_RESULTS="integration-test-results.log"
COLD_START_RESULTS="cold-start-results.log"

# Cleanup function
cleanup() {
    log_info "Cleaning up test result files..."
    rm -f "$UNIT_TEST_RESULTS" "$INTEGRATION_TEST_RESULTS" "$COLD_START_RESULTS"
}

# Set up trap for cleanup
trap cleanup EXIT

# Main validation function
main() {
    log_info "Starting complete functionality validation for airavata-scheduler"
    log_info "Project root: $PROJECT_ROOT"
    
    cd "$PROJECT_ROOT"
    
    # Phase 1: Cold-start setup
    log_info "Phase 1: Performing cold-start setup..."
    if ! perform_cold_start; then
        log_error "Cold-start setup failed"
        exit 1
    fi
    
    # Phase 2: Unit tests
    log_info "Phase 2: Running unit tests..."
    if ! run_unit_tests; then
        log_error "Unit tests failed"
        exit 1
    fi
    
    # Phase 3: Integration tests
    log_info "Phase 3: Running integration tests..."
    if ! run_integration_tests; then
        log_error "Integration tests failed"
        exit 1
    fi
    
    # Phase 4: Generate summary report
    log_info "Phase 4: Generating summary report..."
    generate_summary_report
    
    log_success "Complete functionality validation completed successfully!"
}

# Perform cold-start setup
perform_cold_start() {
    log_info "Running cold-start setup script..."
    
    if [ ! -f "scripts/setup-cold-start.sh" ]; then
        log_error "Cold-start script not found: scripts/setup-cold-start.sh"
        return 1
    fi
    
    # Make script executable
    chmod +x scripts/setup-cold-start.sh
    
    # Run cold-start with timeout
    if timeout "$COLD_START_TIMEOUT" ./scripts/setup-cold-start.sh 2>&1 | tee "$COLD_START_RESULTS"; then
        log_success "Cold-start setup completed successfully"
        return 0
    else
        log_error "Cold-start setup failed or timed out"
        return 1
    fi
}

# Run unit tests
run_unit_tests() {
    log_info "Running unit tests with timeout: $UNIT_TEST_TIMEOUT"
    
    # Check if Go is available
    if ! command -v go &> /dev/null; then
        log_error "Go is not installed or not in PATH"
        return 1
    fi
    
    # Run unit tests
    if go test -v -timeout "$UNIT_TEST_TIMEOUT" ./tests/unit/... 2>&1 | tee "$UNIT_TEST_RESULTS"; then
        log_success "Unit tests completed successfully"
        return 0
    else
        log_error "Unit tests failed"
        return 1
    fi
}

# Run integration tests
run_integration_tests() {
    log_info "Running integration tests with timeout: $INTEGRATION_TEST_TIMEOUT"
    
    # Check if Docker is available
    if ! command -v docker &> /dev/null; then
        log_error "Docker is not installed or not in PATH"
        return 1
    fi
    
    # Check if Docker Compose is available
    if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
        log_error "Docker Compose is not installed or not in PATH"
        return 1
    fi
    
    # Run integration tests
    if go test -v -timeout "$INTEGRATION_TEST_TIMEOUT" ./tests/integration/... 2>&1 | tee "$INTEGRATION_TEST_RESULTS"; then
        log_success "Integration tests completed successfully"
        return 0
    else
        log_error "Integration tests failed"
        return 1
    fi
}

# Generate summary report
generate_summary_report() {
    log_info "Generating summary report..."
    
    echo "=========================================="
    echo "AIRAVATA-SCHEDULER VALIDATION SUMMARY"
    echo "=========================================="
    echo "Timestamp: $(date)"
    echo "Project Root: $PROJECT_ROOT"
    echo ""
    
    # Cold-start results
    echo "COLD-START SETUP:"
    if [ -f "$COLD_START_RESULTS" ]; then
        if grep -q "Cold-start setup completed successfully" "$COLD_START_RESULTS"; then
            echo "  Status: SUCCESS"
        else
            echo "  Status: FAILED"
        fi
    else
        echo "  Status: NO RESULTS"
    fi
    echo ""
    
    # Unit test results
    echo "UNIT TESTS:"
    if [ -f "$UNIT_TEST_RESULTS" ]; then
        unit_passed=$(grep -c "PASS:" "$UNIT_TEST_RESULTS" || echo "0")
        unit_failed=$(grep -c "FAIL:" "$UNIT_TEST_RESULTS" || echo "0")
        unit_skipped=$(grep -c "SKIP:" "$UNIT_TEST_RESULTS" || echo "0")
        
        echo "  Passed: $unit_passed"
        echo "  Failed: $unit_failed"
        echo "  Skipped: $unit_skipped"
        
        if [ "$unit_failed" -eq 0 ]; then
            echo "  Status: SUCCESS"
        else
            echo "  Status: FAILED"
        fi
    else
        echo "  Status: NO RESULTS"
    fi
    echo ""
    
    # Integration test results
    echo "INTEGRATION TESTS:"
    if [ -f "$INTEGRATION_TEST_RESULTS" ]; then
        int_passed=$(grep -c "PASS:" "$INTEGRATION_TEST_RESULTS" || echo "0")
        int_failed=$(grep -c "FAIL:" "$INTEGRATION_TEST_RESULTS" || echo "0")
        int_skipped=$(grep -c "SKIP:" "$INTEGRATION_TEST_RESULTS" || echo "0")
        
        echo "  Passed: $int_passed"
        echo "  Failed: $int_failed"
        echo "  Skipped: $int_skipped"
        
        if [ "$int_failed" -eq 0 ]; then
            echo "  Status: SUCCESS"
        else
            echo "  Status: FAILED"
        fi
    else
        echo "  Status: NO RESULTS"
    fi
    echo ""
    
    # Overall status
    echo "OVERALL STATUS:"
    if [ -f "$COLD_START_RESULTS" ] && [ -f "$UNIT_TEST_RESULTS" ] && [ -f "$INTEGRATION_TEST_RESULTS" ]; then
        if grep -q "Cold-start setup completed successfully" "$COLD_START_RESULTS" && \
           [ "$(grep -c "FAIL:" "$UNIT_TEST_RESULTS" || echo "0")" -eq 0 ] && \
           [ "$(grep -c "FAIL:" "$INTEGRATION_TEST_RESULTS" || echo "0")" -eq 0 ]; then
            echo "  Result: ALL TESTS PASSED"
            echo "  Recommendation: System is ready for production use"
        else
            echo "  Result: SOME TESTS FAILED"
            echo "  Recommendation: Review failed tests before deployment"
        fi
    else
        echo "  Result: INCOMPLETE VALIDATION"
        echo "  Recommendation: Re-run validation script"
    fi
    echo ""
    
    # Test coverage summary
    echo "TEST COVERAGE:"
    echo "  - Cold-start functionality: Validated"
    echo "  - Unit test coverage: $(grep -c "PASS:" "$UNIT_TEST_RESULTS" 2>/dev/null || echo "0") tests"
    echo "  - Integration test coverage: $(grep -c "PASS:" "$INTEGRATION_TEST_RESULTS" 2>/dev/null || echo "0") tests"
    echo "  - End-to-end workflow: Validated"
    echo "  - Data staging: Validated"
    echo "  - Output collection: Validated"
    echo "  - Worker spawning: Validated"
    echo ""
    
    echo "=========================================="
}

# Help function
show_help() {
    echo "Usage: $0 [OPTIONS]"
    echo ""
    echo "Complete functionality validation for airavata-scheduler"
    echo ""
    echo "OPTIONS:"
    echo "  -h, --help     Show this help message"
    echo "  -v, --verbose  Enable verbose output"
    echo "  --unit-only    Run only unit tests"
    echo "  --integration-only  Run only integration tests"
    echo "  --cold-start-only   Run only cold-start setup"
    echo ""
    echo "EXAMPLES:"
    echo "  $0                    # Run complete validation"
    echo "  $0 --unit-only        # Run only unit tests"
    echo "  $0 --integration-only # Run only integration tests"
    echo ""
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            show_help
            exit 0
            ;;
        -v|--verbose)
            set -x
            shift
            ;;
        --unit-only)
            log_info "Running unit tests only..."
            run_unit_tests
            exit $?
            ;;
        --integration-only)
            log_info "Running integration tests only..."
            run_integration_tests
            exit $?
            ;;
        --cold-start-only)
            log_info "Running cold-start setup only..."
            perform_cold_start
            exit $?
            ;;
        *)
            log_error "Unknown option: $1"
            show_help
            exit 1
            ;;
    esac
done

# Run main function
main