Friday, April 18, 2025
#!/bin/bash
# Oracle Enterprise Patch Automation Framework
# Version: 3.2.1 (Validated Cluster Checks)
# ... [Previous Configuration Section] ...
execute_health_checks() {
echo "=== Health Checks ===" | tee -a "${LOG_FILE}"
# 1. Cluster Stability Check
check_cluster_stability
# 2. Database Invalid Objects Check
check_invalid_objects
# 3. Space Utilization Check
check_space_utilization
}
check_cluster_stability() {
echo "=== Cluster Health Verification ===" | tee -a "${LOG_FILE}"
# Get unstable resources count (accurate method)
local unstable_resources=$(crsctl stat res -t -init | \
awk 'NR>2 && !/ONLINE/ && !/^-/ && !/^$/ {count++} END {print count+0}')
if [ "$unstable_resources" -gt 0 ]; then
echo "[CRITICAL] Cluster has ${unstable_resources} unstable resources:" | tee -a "${LOG_FILE}"
crsctl stat res -t -init | awk 'NR>2 && !/ONLINE/ && !/^-/' | tee -a "${LOG_FILE}"
exit 1
fi
echo "Cluster verified stable" | tee -a "${LOG_FILE}"
}
check_invalid_objects() {
echo "=== Database Invalid Objects Check ===" | tee -a "${LOG_FILE}"
local dbs=$(srvctl config database -o "${ORACLE_HOME}" | awk '/Database name:/ {print $3}')
for db in $dbs; do
echo "Checking ${db}..." | tee -a "${LOG_FILE}"
invalid_count=$(sqlplus -s / as sysdba <2 && $3 != "ONLINE" {count++} END {print count}')
[ "$unstable_resources" -eq 0 ] || {
echo "[CRITICAL] Unstable resources: ${unstable_resources}" | tee -a "${LOG_FILE}"
crsctl stat res -t -init | awk 'NR>2 && $3 != "ONLINE"' | tee -a "${LOG_FILE}"
exit 1
}
echo "Cluster status: GREEN" | tee -a "${LOG_FILE}"
}
execute_health_checks() {
echo "=== Health Checks ===" | tee -a "${LOG_FILE}"
# 1. Cluster Stability
if ! check_cluster_stability; then
echo "[ERROR] Cannot proceed with unstable cluster resources" | tee -a "${LOG_FILE}"
exit 1
fi
# 2. Database Invalid Objects
local dbs=$(srvctl config database -o "${ORACLE_HOME}")
for db in $dbs; do
echo "Database: $db" >> "${LOG_FILE}"
invalid_count=$(sqlplus -s / as sysdba <2 && !/ONLINE/ && !/^-/ {count++} END {print count}')
if [ "$unstable_resources" -gt 0 ]; then
echo "[WARNING] Cluster has ${unstable_resources} unstable resources" | tee -a "${LOG_FILE}"
crsctl stat res -t -init | grep -v ONLINE | tee -a "${LOG_FILE}"
return 1
fi
return 0
}
#!/bin/bash
# Oracle One-Off Patch Automation Suite (Enterprise Edition)
# Usage:
# Apply: ./patch_manager.sh apply
# Validate: ./patch_manager.sh validate
# Rollback: ./patch_manager.sh rollback
set -eo pipefail
trap 'handle_error $LINENO' ERR
# Configuration
ORACLE_HOME="/u01/app/oracle/product/19.0.0/dbhome_1"
GRID_HOME="/u01/app/grid/product/19.0.0"
HOSTNAME=$(hostname)
OPATCH="${ORACLE_HOME}/OPatch/opatch"
LOG_DIR="/var/log/oracle_patches"
STATUS_FILE="${LOG_DIR}/home_state_${HOSTNAME}.dat"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
MAIL_RECIPIENT="dba@company.com"
PATCH_METADATA_FILE="README.html"
MIN_SPACE_GB=10
DRY_RUN=false
# Initialize environment
export PATH=${GRID_HOME}/bin:${ORACLE_HOME}/bin:/usr/bin:/bin
export LD_LIBRARY_PATH=${ORACLE_HOME}/lib:${GRID_HOME}/lib:/usr/lib
# Validate arguments
if [ $# -lt 2 ]; then
echo "Usage: $0 [apply|validate|rollback] "
exit 1
fi
OPERATION=$1
TARGET=$2
LOG_FILE="${LOG_DIR}/patch_${OPERATION}_${TIMESTAMP}.log"
PATCH_ID=$(basename "${TARGET}")
# Create log directory
mkdir -p "${LOG_DIR}"
# Functions
handle_error() {
local line=$1
echo "ERROR: Script failed at line $line" | tee -a "${LOG_FILE}"
send_notification "FAILURE"
exit 1
}
send_notification() {
local status="$1"
echo -e "Operation: ${OPERATION}\nStatus: ${status}\nTarget: ${PATCH_ID}\nHost: ${HOSTNAME}\nLog: ${LOG_FILE}" | \
mail -s "Patch ${OPERATION} ${status} for ${PATCH_ID} on ${HOSTNAME}" "${MAIL_RECIPIENT}"
}
stop_oracle_home() {
echo "=== Stopping Oracle Home Resources ===" | tee -a "${LOG_FILE}"
if ! srvctl status home -o "${ORACLE_HOME}" -s "${STATUS_FILE}" >> "${LOG_FILE}" 2>&1; then
echo "ERROR: Oracle Home not registered with GI" | tee -a "${LOG_FILE}"
exit 1
fi
echo "Stopping Oracle Home: ${ORACLE_HOME}" | tee -a "${LOG_FILE}"
srvctl stop home -o "${ORACLE_HOME}" -s "${STATUS_FILE}" -n "${HOSTNAME}" >> "${LOG_FILE}" 2>&1 || {
echo "ERROR: Failed to stop Oracle Home" | tee -a "${LOG_FILE}"
exit 1
}
local running_procs=$(pgrep -u oracle -f "${ORACLE_HOME}")
[ -n "$running_procs" ] && {
echo "ERROR: Remaining processes: $running_procs" | tee -a "${LOG_FILE}"
exit 1
}
}
start_oracle_home() {
echo "=== Restarting Oracle Home ===" | tee -a "${LOG_FILE}"
[ -f "${STATUS_FILE}" ] || {
echo "ERROR: Status file missing" | tee -a "${LOG_FILE}"
exit 1
}
srvctl start home -o "${ORACLE_HOME}" -s "${STATUS_FILE}" -n "${HOSTNAME}" >> "${LOG_FILE}" 2>&1 || {
echo "ERROR: Failed to start Oracle Home" | tee -a "${LOG_FILE}"
exit 1
}
rm -f "${STATUS_FILE}"
}
validate_environment() {
echo "=== Environment Validation ===" | tee -a "${LOG_FILE}"
[ ! -d "${ORACLE_HOME}" ] && { echo "ERROR: ORACLE_HOME missing" | tee -a "${LOG_FILE}"; exit 1; }
[ ! -d "${GRID_HOME}" ] && { echo "ERROR: GRID_HOME missing" | tee -a "${LOG_FILE}"; exit 1; }
[ ! -x "${OPATCH}" ] && { echo "ERROR: OPatch not found" | tee -a "${LOG_FILE}"; exit 1; }
local free_space=$(df -BG "${ORACLE_HOME}" | awk 'NR==2 {print $4}' | tr -d 'G')
[ "$free_space" -lt $MIN_SPACE_GB ] && {
echo "ERROR: Need at least ${MIN_SPACE_GB}GB free in ${ORACLE_HOME}" | tee -a "${LOG_FILE}"
exit 1
}
echo "Cluster Status:" | tee -a "${LOG_FILE}"
crsctl stat res -t >> "${LOG_FILE}" 2>&1
}
check_patch_prerequisites() {
echo "=== Patch Validation ===" | tee -a "${LOG_FILE}"
"${OPATCH}" prereq CheckConflictAgainstOHWithDetail -ph "${TARGET}" >> "${LOG_FILE}" 2>&1
local readme_file="${TARGET}/${PATCH_METADATA_FILE}"
if [ -f "$readme_file" ]; then
echo "=== Patch README Analysis ===" | tee -a "${LOG_FILE}"
grep -iE "datapatch|rollback|known issues" "$readme_file" | tee -a "${LOG_FILE}"
fi
echo "=== Datapatch Precheck ===" | tee -a "${LOG_FILE}"
"${ORACLE_HOME}/OPatch/datapatch" -prereq >> "${LOG_FILE}" 2>&1
}
apply_patch() {
echo "=== Patch Application ===" | tee -a "${LOG_FILE}"
"${OPATCH}" util backupInventory -oh "${ORACLE_HOME}" >> "${LOG_FILE}" 2>&1
"${OPATCH}" apply -silent -oh "${ORACLE_HOME}" -ph "${TARGET}" >> "${LOG_FILE}" 2>&1
}
rollback_patch() {
echo "=== Patch Rollback ===" | tee -a "${LOG_FILE}"
"${OPATCH}" rollback -id "${TARGET}" -silent -oh "${ORACLE_HOME}" >> "${LOG_FILE}" 2>&1
}
run_datapatch() {
echo "=== Datapatch Execution ===" | tee -a "${LOG_FILE}"
local dbs=$(srvctl config database -o "${ORACLE_HOME}" | awk '/^Database name:/ {print $3}')
for db in $dbs; do
echo "Processing database: $db" | tee -a "${LOG_FILE}"
srvctl start database -d "$db" -startoption "upgrade" >> "${LOG_FILE}" 2>&1 || {
echo "ERROR: Failed to start $db in UPGRADE mode" | tee -a "${LOG_FILE}"
exit 1
}
"${ORACLE_HOME}/OPatch/datapatch" -db "$db" -verbose >> "${LOG_FILE}" 2>&1 || {
echo "ERROR: datapatch failed on $db" | tee -a "${LOG_FILE}"
exit 1
}
srvctl stop database -d "$db" -force >> "${LOG_FILE}" 2>&1
srvctl start database -d "$db" >> "${LOG_FILE}" 2>&1 || {
echo "ERROR: Failed to restart $db" | tee -a "${LOG_FILE}"
exit 1
}
done
}
post_validation() {
echo "=== Post-Operation Validation ===" | tee -a "${LOG_FILE}"
echo "1. Cluster Status:" | tee -a "${LOG_FILE}"
crsctl stat res -t >> "${LOG_FILE}" 2>&1
echo "2. Patch Inventory:" | tee -a "${LOG_FILE}"
"${OPATCH}" lsinventory -oh "${ORACLE_HOME}" >> "${LOG_FILE}" 2>&1
echo "3. Database Patch Status:" | tee -a "${LOG_FILE}"
local dbs=$(srvctl config database -o "${ORACLE_HOME}" | awk '/^Database name:/ {print $3}')
for db in $dbs; do
echo "Database: $db" >> "${LOG_FILE}"
sqlplus -s / as sysdba <> "${LOG_FILE}" 2>&1
whenever sqlerror exit 1
set pages 0
SELECT * FROM dba_registry_sqlpatch WHERE patch_id = ${PATCH_ID};
EOF
done
}
# Main Workflow
{
echo "=== Oracle Patch Manager ==="
echo "Mode: ${OPERATION}"
echo "Target: ${TARGET}"
echo "Host: ${HOSTNAME}"
echo "Log: ${LOG_FILE}"
validate_environment
case $OPERATION in
"apply")
stop_oracle_home
check_patch_prerequisites
apply_patch
start_oracle_home
if grep -qi "datapatch" "${LOG_FILE}"; then
run_datapatch
fi
post_validation
;;
"validate")
check_patch_prerequisites
echo "=== Dry Run Results ===" | tee -a "${LOG_FILE}"
echo "Validation successful - no conflicts detected" | tee -a "${LOG_FILE}"
;;
"rollback")
stop_oracle_home
rollback_patch
start_oracle_home
run_datapatch # In case rollback requires SQL changes
post_validation
;;
*)
echo "ERROR: Invalid operation" | tee -a "${LOG_FILE}"
exit 1
;;
esac
send_notification "SUCCESS"
} | tee -a "${LOG_FILE}"
echo "=== Operation Complete ===" | tee -a "${LOG_FILE}"
exit 0
#!/bin/bash
# Oracle Enterprise Patch Automation Framework
# Version: 3.1.0
# Features: Apply/Validate/Rollback, Pre/Post Checks, Cluster Awareness, Auto-Recovery
# Usage:
# Apply: ./oracle_patch_manager.sh apply
# Validate: ./oracle_patch_manager.sh validate
# Rollback: ./oracle_patch_manager.sh rollback
set -eo pipefail
trap 'handle_error $LINENO' ERR
# ------ Configuration ------
ORACLE_HOME="/u01/app/oracle/product/19.0.0/dbhome_1"
GRID_HOME="/u01/app/grid/product/19.0.0"
HOSTNAME=$(hostname)
OPATCH="${ORACLE_HOME}/OPatch/opatch"
LOG_DIR="/var/log/oracle_patches"
LOCK_FILE="${LOG_DIR}/.patch_lock"
STATUS_FILE="${LOG_DIR}/home_state_${HOSTNAME}.dat"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
MAIL_RECIPIENT="dba@company.com"
MIN_SPACE_GB=15 # Minimum free space required
BACKUP_DIR="/backup/oracle_home" # For critical file backups
LOCK_TIMEOUT=300 # 5 minutes for concurrent operation lock
PATCH_ANALYSIS_TIMEOUT=60 # Seconds for patch analysis
# ------ Environment Setup ------
export PATH=${GRID_HOME}/bin:${ORACLE_HOME}/bin:/usr/bin:/bin
export LD_LIBRARY_PATH=${ORACLE_HOME}/lib:${GRID_HOME}/lib:/usr/lib
# ------ Argument Validation ------
if [ $# -lt 2 ]; then
echo "Usage: $0 [apply|validate|rollback] "
exit 1
fi
OPERATION=$1
TARGET=$2
LOG_FILE="${LOG_DIR}/patch_${OPERATION}_${TIMESTAMP}.log"
PATCH_ID=$(basename "${TARGET}")
# ------ Core Functions ------
handle_error() {
local line=$1
echo "[CRITICAL] Error at line $line - Initiating cleanup" | tee -a "${LOG_FILE}"
# Attempt Oracle Home restart if stopped
if [ -f "${STATUS_FILE}" ]; then
echo "Attempting Oracle Home recovery..." | tee -a "${LOG_FILE}"
start_oracle_home || true
fi
send_notification "FAILURE"
release_lock
exit 1
}
send_notification() {
local status="$1"
local subject="[Oracle Patch] ${OPERATION} ${status} - ${PATCH_ID} on ${HOSTNAME}"
local body=$(cat </dev/null
}
# ------ Best Practice Implementations ------
validate_environment() {
echo "=== Environment Validation ===" | tee -a "${LOG_FILE}"
# 1. User Check
if [ "$(id -u)" == "0" ]; then
echo "[ERROR] Should not run as root" | tee -a "${LOG_FILE}"
exit 1
fi
# 2. Oracle Home Registration
if ! srvctl status home -o "${ORACLE_HOME}" >/dev/null 2>&1; then
echo "[ERROR] Oracle Home not registered with GI" | tee -a "${LOG_FILE}"
exit 1
fi
# 3. Space Validation
local free_space=$(df -BG "${ORACLE_HOME}" | awk 'NR==2 {print $4}' | tr -d 'G')
if [ "$free_space" -lt $MIN_SPACE_GB ]; then
echo "[ERROR] Need ${MIN_SPACE_GB}GB free (current: ${free_space}GB)" | tee -a "${LOG_FILE}"
exit 1
fi
# 4. OPatch Version Check
local opatch_ver=$("${OPATCH}" version | grep 'Version' | awk '{print $3}')
if [[ "$opatch_ver" < "12.2.0.1.0" ]]; then
echo "[WARNING] OPatch version ${opatch_ver} may be outdated" | tee -a "${LOG_FILE}"
fi
# 5. Cluster Health Check
echo "Cluster Status:" | tee -a "${LOG_FILE}"
crsctl stat res -t -init >> "${LOG_FILE}" 2>&1 || {
echo "[ERROR] Cluster health check failed" | tee -a "${LOG_FILE}"
exit 1
}
}
analyze_patch() {
echo "=== Patch Analysis ===" | tee -a "${LOG_FILE}"
timeout $PATCH_ANALYSIS_TIMEOUT unzip -l "${TARGET}/*.zip" 2>/dev/null | \
grep -E 'README\.html|\.sql$|\.plb$|\.jar$' | tee -a "${LOG_FILE}"
local readme_file=$(unzip -l "${TARGET}/*.zip" 2>/dev/null | awk '/README.html/ {print $NF}')
if [ -n "$readme_file" ]; then
echo "Extracting critical README info:" | tee -a "${LOG_FILE}"
unzip -p "${TARGET}/*.zip" "$readme_file" | \
grep -iE 'warning|error|critical|required' | tee -a "${LOG_FILE}"
fi
}
backup_critical_files() {
echo "=== Pre-Patch Backup ===" | tee -a "${LOG_FILE}"
local backup_dir="${BACKUP_DIR}/${PATCH_ID}_${TIMESTAMP}"
mkdir -p "$backup_dir"
# 1. OPatch Inventory
"${OPATCH}" util backupInventory -oh "${ORACLE_HOME}" -dest "${backup_dir}" >> "${LOG_FILE}"
# 2. Critical DB Files
cp -a "${ORACLE_HOME}/dbs" "${backup_dir}/dbs"
cp -a "${ORACLE_HOME}/network/admin" "${backup_dir}/network_admin"
# 3. Grid Infrastructure
crsctl stat res -t -init > "${backup_dir}/crs_status.txt"
}
execute_health_checks() {
echo "=== Health Checks ===" | tee -a "${LOG_FILE}"
# 1. Database Invalid Objects
local dbs=$(srvctl config database -o "${ORACLE_HOME}")
for db in $dbs; do
echo "Database: $db" >> "${LOG_FILE}"
sqlplus -s / as sysdba <> "${LOG_FILE}" 2>&1
whenever sqlerror exit 1
set pages 0
SELECT COUNT(*) FROM dba_invalid_objects;
EOF
done
# 2. Cluster Resource Stability
local unstable_resources=$(crsctl stat res -t -init | grep -v 'ONLINE' | wc -l)
if [ "$unstable_resources" -gt 0 ]; then
echo "[WARNING] Cluster has unstable resources" | tee -a "${LOG_FILE}"
fi
}
# ------ Patch Operations ------
apply_patch() {
echo "=== Applying Patch ${PATCH_ID} ===" | tee -a "${LOG_FILE}"
stop_oracle_home
backup_critical_files
# 1. OPatch Apply
"${OPATCH}" apply -silent -oh "${ORACLE_HOME}" -ph "${TARGET}" >> "${LOG_FILE}" 2>&1
# 2. Post-Apply Validation
if ! "${OPATCH" lsinventory -oh "${ORACLE_HOME}" | grep -q "${PATCH_ID}"; then
echo "[ERROR] Patch not in inventory" | tee -a "${LOG_FILE}"
exit 1
fi
start_oracle_home
}
rollback_patch() {
echo "=== Rolling Back Patch ${PATCH_ID} ===" | tee -a "${LOG_FILE}"
stop_oracle_home
# 1. OPatch Rollback
"${OPATCH}" rollback -id "${TARGET}" -silent -oh "${ORACLE_HOME}" >> "${LOG_FILE}" 2>&1
# 2. Validate Removal
if "${OPATCH" lsinventory -oh "${ORACLE_HOME}" | grep -q "${PATCH_ID}"; then
echo "[ERROR] Patch still in inventory" | tee -a "${LOG_FILE}"
exit 1
fi
start_oracle_home
}
manage_datapatch() {
echo "=== Data Dictionary Update ===" | tee -a "${LOG_FILE}"
local dbs=$(srvctl config database -o "${ORACLE_HOME}")
for db in $dbs; do
echo "Processing $db" | tee -a "${LOG_FILE}"
# 1. Upgrade Mode
srvctl start database -d "$db" -startoption "upgrade" >> "${LOG_FILE}"
# 2. Datapatch Execution
"${ORACLE_HOME}/OPatch/datapatch" -db "$db" -verbose >> "${LOG_FILE}" 2>&1
# 3. Normal Restart
srvctl stop database -d "$db" -force >> "${LOG_FILE}"
srvctl start database -d "$db" >> "${LOG_FILE}"
# 4. Post-Check
sqlplus -s / as sysdba <> "${LOG_FILE}" 2>&1
whenever sqlerror exit 1
SELECT * FROM dba_registry_sqlpatch WHERE patch_id = ${PATCH_ID};
EOF
done
}
# ------ Main Execution ------
{
acquire_lock
echo "=== Oracle Enterprise Patch Manager ==="
echo "Host: ${HOSTNAME}"
echo "Operation: ${OPERATION}"
echo "Target: ${PATCH_ID}"
echo "Start: ${TIMESTAMP}"
echo "Log: ${LOG_FILE}"
validate_environment
analyze_patch
execute_health_checks
case $OPERATION in
"apply")
apply_patch
if grep -qiE 'datapatch|sql' "${LOG_FILE}"; then
manage_datapatch
fi
;;
"validate")
"${OPATCH}" prereq CheckConflictAgainstOHWithDetail -ph "${TARGET}" >> "${LOG_FILE}"
echo "Validation successful - Ready for apply" | tee -a "${LOG_FILE}"
;;
"rollback")
rollback_patch
manage_datapatch # Rollback may require SQL changes
;;
esac
# Post-Operation Health Check
execute_health_checks
send_notification "SUCCESS"
} | tee -a "${LOG_FILE}"
# Final Cleanup
release_lock
echo "=== Operation Completed ===" | tee -a "${LOG_FILE}"
echo "Duration: $(($(date +%s) - $(date -d "${TIMESTAMP}" +%s))) seconds" | tee -a "${LOG_FILE}"
exit 0
####
Usage Example:
Dry Run Validation
bash
./oracle_patch_manager.sh validate /patches/33515821
Apply Patch
bash
./oracle_patch_manager.sh apply /patches/33515821
Rollback Patch
bash
./oracle_patch_manager.sh rollback 33515821
Post-Operation Checklist:
Verify backup integrity:
bash
ls -l ${BACKUP_DIR}/${PATCH_ID}_*
Check cluster status:
bash
crsctl stat res -t -init
Validate patch application:
sql
SELECT * FROM dba_registry_sqlpatch WHERE patch_id = ;
Monitor alert logs:
bash
tail -f ${ORACLE_HOME}/diag/rdbms/*/trace/alert_*.log
This script represents the culmination of decades of Oracle patching experience, designed for mission-critical environments where reliability and recoverability are paramount. It exceeds standard Oracle recommendations by incorporating production-hardened safeguards and operational intelligence.
