#!/bin/bash
################################################################################
# Oracle 19c RAC Database Administration - Data Guard Switchover Functions
# Description: Data Guard switchover with automatic rollback capability
# Created: 2025-11-02
# Updated: 2025-11-14 - Integrated with database loading fix and wrapper functions
################################################################################
################################################################################
# Function: perform_dg_switchover
# Description: Main wrapper function for Data Guard switchover
# Parameters: $1 - Primary database name
# $2 - Primary SCAN address
# $3 - Primary service name
# $4 - Standby database name
# $5 - Standby SCAN address
# $6 - Standby service name
# Returns: 0 on success, 1 on failure
################################################################################
perform_dg_switchover() {
local primary_db=$1
local primary_scan=$2
local primary_service=$3
local standby_db=$4
local standby_scan=$5
local standby_service=$6
log_message "INFO" "Starting Data Guard switchover process"
log_message "INFO" "Primary: ${primary_db} @ ${primary_scan}/${primary_service}"
log_message "INFO" "Standby: ${standby_db} @ ${standby_scan}/${standby_service}"
# Validate connections to both databases
echo ""
echo "Validating database connections..."
if ! test_db_connection "${primary_scan}" "${primary_service}"; then
log_message "ERROR" "Cannot connect to primary database ${primary_db}"
echo "ERROR: Cannot connect to primary database ${primary_db}"
return 1
fi
echo "✓ Primary database connection OK"
if ! test_db_connection "${standby_scan}" "${standby_service}"; then
log_message "ERROR" "Cannot connect to standby database ${standby_db}"
echo "ERROR: Cannot connect to standby database ${standby_db}"
return 1
fi
echo "✓ Standby database connection OK"
# Call the actual switchover function
perform_switchover "$primary_db" "$standby_db"
return $?
}
################################################################################
# Function: perform_switchover
# Description: Perform Data Guard switchover with validation and rollback
# Parameters: $1 - Current primary database name
# $2 - Target standby database name (new primary)
# Returns: 0 on success, 1 on failure
################################################################################
perform_switchover() {
local current_primary=$1
local target_standby=$2
log_message "INFO" "=========================================="
log_message "INFO" "Starting Data Guard Switchover"
log_message "INFO" "Current Primary: $current_primary"
log_message "INFO" "Target Primary: $target_standby"
log_message "INFO" "=========================================="
# Generate switchover report file
local timestamp=$(date '+%Y%m%d_%H%M%S')
local report_file="${REPORT_BASE_DIR}/switchover_${current_primary}_${timestamp}.log"
# Create restore point for rollback
local restore_point="SWITCHOVER_${timestamp}"
echo ""
echo "=========================================="
echo "Data Guard Switchover Process"
echo "=========================================="
echo ""
echo "Phase 1: Pre-Switchover Validation"
echo "-----------------------------------"
# Pre-switchover validation
if ! validate_switchover_prerequisites "$current_primary" "$target_standby"; then
log_message "ERROR" "Pre-switchover validation failed"
echo "✗ Pre-switchover validation FAILED"
echo " Switchover cannot proceed safely"
return 1
fi
echo "✓ Pre-switchover validation PASSED"
log_message "INFO" "Pre-switchover validation passed"
# Create restore points for rollback
echo ""
echo "Phase 2: Creating Restore Points"
echo "---------------------------------"
if ! create_switchover_restore_points "$current_primary" "$restore_point"; then
log_message "ERROR" "Failed to create restore points"
echo "✗ Failed to create restore points"
echo " Switchover aborted for safety"
return 1
fi
echo "✓ Restore points created successfully"
log_message "INFO" "Restore points created: $restore_point"
# Perform the switchover
echo ""
echo "Phase 3: Executing Switchover"
echo "------------------------------"
if ! execute_switchover "$current_primary" "$target_standby"; then
log_message "ERROR" "Switchover execution failed"
echo "✗ Switchover FAILED"
echo ""
echo "Attempting automatic rollback..."
if perform_switchover_rollback "$current_primary" "$target_standby" "$restore_point"; then
echo "✓ Rollback successful - databases restored to original state"
log_message "INFO" "Switchover rolled back successfully"
else
echo "✗ Rollback FAILED - manual intervention required"
log_message "ERROR" "Switchover rollback failed - manual intervention needed"
fi
return 1
fi
echo "✓ Switchover executed successfully"
log_message "INFO" "Switchover execution completed"
# Post-switchover validation
echo ""
echo "Phase 4: Post-Switchover Validation"
echo "------------------------------------"
if ! validate_post_switchover "$target_standby" "$current_primary"; then
log_message "ERROR" "Post-switchover validation failed"
echo "✗ Post-switchover validation FAILED"
echo " WARNING: Databases may not be in sync"
echo ""
echo "Attempting automatic rollback..."
if perform_switchover_rollback "$target_standby" "$current_primary" "$restore_point"; then
echo "✓ Rollback successful - databases restored"
log_message "INFO" "Post-validation failure - rolled back successfully"
else
echo "✗ Rollback FAILED - manual intervention required"
log_message "ERROR" "Post-validation rollback failed"
fi
return 1
fi
echo "✓ Post-switchover validation PASSED"
log_message "INFO" "Post-switchover validation successful"
# Generate switchover report
generate_switchover_report "$current_primary" "$target_standby" "$timestamp" "SUCCESS"
echo ""
echo "=========================================="
echo "Switchover Completed Successfully"
echo "=========================================="
echo ""
echo "New Configuration:"
echo " Primary: $target_standby"
echo " Standby: $current_primary"
echo ""
echo "Report: $report_file"
echo ""
log_message "INFO" "=========================================="
log_message "INFO" "Switchover completed successfully"
log_message "INFO" "New Primary: $target_standby"
log_message "INFO" "=========================================="
return 0
}
################################################################################
# Function: validate_switchover_prerequisites
# Description: Validate that switchover can be performed safely
################################################################################
validate_switchover_prerequisites() {
local current_primary=$1
local target_standby=$2
log_message "INFO" "Validating switchover prerequisites"
echo " Checking primary database status..."
# Check primary database role
local primary_role=$(sqlplus -s / as sysdba << EOF
SET HEADING OFF
SET FEEDBACK OFF
SET PAGESIZE 0
SELECT database_role FROM v\$database;
EXIT;
EOF
)
primary_role=$(echo "$primary_role" | xargs)
if [[ "$primary_role" != "PRIMARY" ]]; then
echo " ✗ Database is not in PRIMARY role (current: $primary_role)"
log_message "ERROR" "Database not in PRIMARY role: $primary_role"
return 1
fi
echo " ✓ Database is in PRIMARY role"
# Check switchover status
local switchover_status=$(sqlplus -s / as sysdba << EOF
SET HEADING OFF
SET FEEDBACK OFF
SET PAGESIZE 0
SELECT switchover_status FROM v\$database;
EXIT;
EOF
)
switchover_status=$(echo "$switchover_status" | xargs)
if [[ "$switchover_status" != "TO STANDBY" && "$switchover_status" != "SESSIONS ACTIVE" ]]; then
echo " ✗ Switchover not allowed (status: $switchover_status)"
log_message "ERROR" "Invalid switchover status: $switchover_status"
return 1
fi
echo " ✓ Switchover status is valid: $switchover_status"
# Check for active sessions if needed
if [[ "$switchover_status" == "SESSIONS ACTIVE" ]]; then
echo " ⚠ Warning: Active sessions present - will be terminated during switchover"
log_message "WARN" "Active sessions present during switchover"
fi
# Check Data Guard configuration
echo " Checking Data Guard configuration..."
local dg_status=$(dgmgrl / << EOF
SHOW CONFIGURATION;
EXIT;
EOF
)
if echo "$dg_status" | grep -q "SUCCESS"; then
echo " ✓ Data Guard configuration is valid"
else
echo " ✗ Data Guard configuration has issues"
log_message "ERROR" "Data Guard configuration validation failed"
return 1
fi
# Check standby database status via DGMGRL
echo " Checking standby database status..."
local standby_status=$(dgmgrl / << EOF
SHOW DATABASE $target_standby;
EXIT;
EOF
)
if echo "$standby_status" | grep -q "SUCCESS"; then
echo " ✓ Standby database is ready"
else
echo " ✗ Standby database has issues"
echo "$standby_status"
log_message "ERROR" "Standby database validation failed"
return 1
fi
# Check apply lag
echo " Checking apply lag..."
local apply_lag=$(sqlplus -s / as sysdba << EOF
SET HEADING OFF
SET FEEDBACK OFF
SET PAGESIZE 0
SELECT value FROM v\$dataguard_stats WHERE name = 'apply lag';
EXIT;
EOF
)
apply_lag=$(echo "$apply_lag" | xargs)
if [[ -n "$apply_lag" ]]; then
echo " Apply lag: $apply_lag"
log_message "INFO" "Current apply lag: $apply_lag"
else
echo " ✓ No significant apply lag"
fi
# Check for archive gaps
echo " Checking for archive log gaps..."
local gap_count=$(sqlplus -s / as sysdba << EOF
SET HEADING OFF
SET FEEDBACK OFF
SET PAGESIZE 0
SELECT COUNT(*) FROM v\$archive_gap;
EXIT;
EOF
)
gap_count=$(echo "$gap_count" | xargs)
if [[ $gap_count -gt 0 ]]; then
echo " ✗ Archive log gaps detected: $gap_count"
log_message "ERROR" "Archive log gaps present: $gap_count"
return 1
fi
echo " ✓ No archive log gaps"
log_message "INFO" "All prerequisite checks passed"
return 0
}
################################################################################
# Function: create_switchover_restore_points
# Description: Create restore points for rollback capability
################################################################################
create_switchover_restore_points() {
local current_primary=$1
local restore_point=$2
log_message "INFO" "Creating restore points for rollback"
echo " Creating restore point on primary: $restore_point"
# Create guaranteed restore point on primary
sqlplus -s / as sysdba << EOF
WHENEVER SQLERROR EXIT 1
CREATE RESTORE POINT ${restore_point} GUARANTEE FLASHBACK DATABASE;
EXIT;
EOF
if [[ $? -eq 0 ]]; then
echo " ✓ Restore point created on primary"
log_message "INFO" "Restore point created on primary: $restore_point"
else
echo " ✗ Failed to create restore point on primary"
log_message "ERROR" "Failed to create restore point on primary"
return 1
fi
# Note: Restore point will be automatically created on standby via redo apply
echo " Restore point will replicate to standby via redo apply"
return 0
}
################################################################################
# Function: execute_switchover
# Description: Execute the actual switchover operation
################################################################################
execute_switchover() {
local current_primary=$1
local target_standby=$2
log_message "INFO" "Executing switchover from $current_primary to $target_standby"
echo " Step 1: Switching $current_primary to standby role..."
# Switchover current primary to standby
local switch_result=$(dgmgrl / << EOF
SWITCHOVER TO $target_standby;
EXIT;
EOF
)
echo "$switch_result"
if echo "$switch_result" | grep -q "SUCCESS"; then
echo " ✓ Switchover command executed successfully"
log_message "INFO" "Switchover command successful"
else
echo " ✗ Switchover command failed"
log_message "ERROR" "Switchover command failed"
return 1
fi
# Wait for switchover to complete
echo " Step 2: Waiting for switchover to complete..."
sleep 10
# Verify new primary is open
echo " Step 3: Verifying new primary database..."
local retries=0
local max_retries=30
while [[ $retries -lt $max_retries ]]; do
local db_status=$(dgmgrl / << EOF
SHOW DATABASE $target_standby;
EXIT;
EOF
)
if echo "$db_status" | grep -q "PRIMARY" && echo "$db_status" | grep -q "SUCCESS"; then
echo " ✓ New primary database is open and ready"
log_message "INFO" "New primary database confirmed: $target_standby"
return 0
fi
((retries++))
echo " Waiting for database to transition... (attempt $retries/$max_retries)"
sleep 5
done
echo " ✗ Timeout waiting for database transition"
log_message "ERROR" "Timeout waiting for new primary to be ready"
return 1
}
################################################################################
# Function: validate_post_switchover
# Description: Validate switchover completed successfully
################################################################################
validate_post_switchover() {
local new_primary=$1
local new_standby=$2
log_message "INFO" "Validating post-switchover state"
echo " Checking new configuration..."
# Show configuration
local config=$(dgmgrl / << EOF
SHOW CONFIGURATION;
EXIT;
EOF
)
echo "$config"
if echo "$config" | grep -q "$new_primary" | grep -q "PRIMARY"; then
echo " ✓ New primary confirmed: $new_primary"
else
echo " ✗ New primary validation failed"
log_message "ERROR" "New primary validation failed"
return 1
fi
if echo "$config" | grep -q "$new_standby" | grep -q "STANDBY"; then
echo " ✓ New standby confirmed: $new_standby"
else
echo " ✗ New standby validation failed"
log_message "ERROR" "New standby validation failed"
return 1
fi
# Check for errors
if echo "$config" | grep -qi "error\|warning"; then
echo " ⚠ Configuration warnings/errors detected"
log_message "WARN" "Post-switchover configuration has warnings"
else
echo " ✓ No configuration errors detected"
fi
# Verify redo transport is working
echo " Checking redo transport..."
sleep 5
local transport_check=$(dgmgrl / << EOF
SHOW DATABASE $new_standby;
EXIT;
EOF
)
if echo "$transport_check" | grep -q "SUCCESS"; then
echo " ✓ Redo transport is active"
else
echo " ⚠ Redo transport may have issues"
log_message "WARN" "Redo transport validation inconclusive"
fi
log_message "INFO" "Post-switchover validation completed"
return 0
}
################################################################################
# Function: perform_switchover_rollback
# Description: Rollback switchover using restore points
################################################################################
perform_switchover_rollback() {
local current_primary=$1
local current_standby=$2
local restore_point=$3
log_message "INFO" "=========================================="
log_message "INFO" "Starting Switchover Rollback"
log_message "INFO" "=========================================="
echo ""
echo "Rollback Process:"
echo "================="
# Attempt to switchover back
echo " Attempting to switch back to original configuration..."
local rollback_result=$(dgmgrl / << EOF
SWITCHOVER TO $current_standby;
EXIT;
EOF
)
if echo "$rollback_result" | grep -q "SUCCESS"; then
echo " ✓ Switchover rollback successful"
log_message "INFO" "Switchover rolled back successfully"
# Wait for completion
sleep 10
# Verify original configuration
echo " Verifying original configuration restored..."
local verify=$(dgmgrl / << EOF
SHOW CONFIGURATION;
EXIT;
EOF
)
if echo "$verify" | grep -q "$current_standby" | grep -q "PRIMARY"; then
echo " ✓ Original configuration confirmed"
log_message "INFO" "Original configuration restored"
return 0
else
echo " ⚠ Configuration verification inconclusive"
log_message "WARN" "Rollback verification inconclusive"
return 1
fi
else
echo " ✗ Automatic rollback failed"
log_message "ERROR" "Automatic switchover rollback failed"
echo ""
echo "Manual Rollback Required:"
echo "========================"
echo "1. Connect to $current_primary"
echo "2. Execute: FLASHBACK DATABASE TO RESTORE POINT $restore_point;"
echo "3. Execute: ALTER DATABASE OPEN RESETLOGS;"
echo "4. Reconfigure Data Guard"
echo ""
return 1
fi
}
################################################################################
# Function: generate_switchover_report
# Description: Generate detailed switchover report
################################################################################
generate_switchover_report() {
local old_primary=$1
local new_primary=$2
local timestamp=$3
local status=$4
local report_file="${REPORT_BASE_DIR}/switchover_${old_primary}_${timestamp}.html"
{
cat << EOF
Data Guard Switchover Report
Switchover Summary
| Item | Value |
|---|
| Original Primary | $old_primary |
| New Primary | $new_primary |
| Switchover Time | $(date '+%Y-%m-%d %H:%M:%S') |
| Status | $status |
Current Configuration
EOF
dgmgrl / << DGEOF
SHOW CONFIGURATION;
EXIT;
DGEOF
echo ""
echo "
"
generate_html_footer
} > "$report_file"
log_message "INFO" "Switchover report generated: $report_file"
}
################################################################################
# Export functions
################################################################################
export -f perform_switchover
export -f validate_switchover_prerequisites
export -f execute_switchover
export -f perform_switchover_rollback
log_message "INFO" "Data Guard switchover functions loaded"
