#!/bin/bash # ============================================================================ # Gitea Manual Update Script # ============================================================================ # Updates high-risk containers (gitea, postgres) with manual approval, # backup, health checks, and automatic rollback on failure. # # Usage: ./manual-update.sh [container2] [...] # Example: ./manual-update.sh gitea postgres # # This script requires explicit operator invocation and confirmation. # ============================================================================ set -e # ============================================================================ # Configuration # ============================================================================ readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly DOCKER_COMPOSE_DIR="/opt/gitea" readonly BACKUP_SCRIPT="${SCRIPT_DIR}/backup.sh" readonly HEALTH_CHECK_SCRIPT="${SCRIPT_DIR}/health-check.sh" readonly LOG_FILE="/var/log/gitea-manual-update.log" readonly ROLLBACK_INFO="/tmp/gitea-rollback-info-$$.json" # Output colors readonly GREEN='\033[0;32m' readonly YELLOW='\033[1;33m' readonly RED='\033[0;31m' readonly BLUE='\033[0;34m' readonly NC='\033[0m' # ============================================================================ # Logging Functions # ============================================================================ log_info() { local message="[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $1" echo -e "${YELLOW}${message}${NC}" echo "${message}" >> "${LOG_FILE}" } log_success() { local message="[$(date '+%Y-%m-%d %H:%M:%S')] [SUCCESS] $1" echo -e "${GREEN}${message}${NC}" echo "${message}" >> "${LOG_FILE}" } log_error() { local message="[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $1" echo -e "${RED}${message}${NC}" >&2 echo "${message}" >> "${LOG_FILE}" } log_prompt() { echo -e "${BLUE}[PROMPT]${NC} $1" } error_exit() { log_error "$1" cleanup exit 1 } # ============================================================================ # Cleanup Function # ============================================================================ cleanup() { if [ -f "${ROLLBACK_INFO}" ]; then rm -f "${ROLLBACK_INFO}" fi } # ============================================================================ # Validation Functions # ============================================================================ validate_args() { if [ $# -eq 0 ]; then error_exit "No containers specified. Usage: $0 [container2] [...]" fi for container in "$@"; do if ! docker compose -f "${DOCKER_COMPOSE_DIR}/docker-compose.yml" config --services | grep -q "^${container}$"; then error_exit "Container '${container}' not found in docker-compose.yml" fi done log_success "Container validation passed" } # ============================================================================ # User Confirmation Functions # ============================================================================ get_user_confirmation() { local containers="$*" echo "" log_prompt "==========================================" log_prompt "MANUAL UPDATE CONFIRMATION" log_prompt "==========================================" log_prompt "You are about to update the following containers:" for container in ${containers}; do log_prompt " - ${container}" done echo "" log_prompt "This will:" log_prompt " 1. Create a backup of database and Gitea data" log_prompt " 2. Pull new container images" log_prompt " 3. Recreate the containers with new versions" log_prompt " 4. Run health checks" log_prompt " 5. Rollback automatically if health checks fail" echo "" log_prompt "Estimated downtime: 1-3 minutes" echo "" read -p "Do you want to proceed? (yes/no): " confirmation case "${confirmation}" in yes|YES|Yes) log_success "Update confirmed by operator" return 0 ;; *) log_info "Update cancelled by operator" exit 0 ;; esac } show_current_versions() { log_info "Current container versions:" cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}" for container in "$@"; do local image=$(docker compose images "${container}" 2>/dev/null | tail -n +3 | awk '{print $2":"$3}' | head -n1) if [ -n "${image}" ]; then log_info " ${container}: ${image}" fi done echo "" } show_available_versions() { log_info "Checking for available updates..." cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}" for container in "$@"; do log_info " Checking ${container}..." docker compose pull --dry-run "${container}" 2>&1 | grep -i "image" || true done echo "" } # ============================================================================ # Rollback Management Functions # ============================================================================ save_current_images() { log_info "Saving current image versions for rollback..." echo "{" > "${ROLLBACK_INFO}" local first=true for container in "$@"; do local image=$(docker compose -f "${DOCKER_COMPOSE_DIR}/docker-compose.yml" images -q "${container}" 2>/dev/null | head -n1) if [ -n "${image}" ]; then if [ "${first}" = true ]; then first=false else echo "," >> "${ROLLBACK_INFO}" fi echo " \"${container}\": \"${image}\"" >> "${ROLLBACK_INFO}" log_info "Saved ${container}: ${image}" fi done echo "}" >> "${ROLLBACK_INFO}" log_success "Current image versions saved" } rollback() { log_error "Rolling back to previous versions..." if [ ! -f "${ROLLBACK_INFO}" ]; then log_error "No rollback information found" return 1 fi cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}" # Extract containers from rollback info and restore local containers=$(grep -o '"[^"]*":' "${ROLLBACK_INFO}" | tr -d '":' | tr '\n' ' ') for container in ${containers}; do log_info "Rolling back ${container}..." docker compose up -d "${container}" || log_error "Failed to rollback ${container}" done log_success "Rollback completed" } # ============================================================================ # Update Functions # ============================================================================ run_backup() { log_info "Running backup before update..." if ! bash "${BACKUP_SCRIPT}"; then error_exit "Backup failed - aborting update" fi log_success "Backup completed successfully" } pull_new_images() { log_info "Pulling new images..." cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}" for container in "$@"; do log_info "Pulling image for ${container}..." if ! docker compose pull "${container}"; then error_exit "Failed to pull image for ${container}" fi done log_success "All images pulled successfully" } recreate_containers() { log_info "Recreating containers..." log_info "⚠️ Service downtime begins now" cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}" if ! docker compose up -d "$@"; then error_exit "Failed to recreate containers" fi # Wait for containers to start - longer for database log_info "Waiting for containers to start (30 seconds)..." sleep 30 log_success "Containers recreated successfully" } run_health_check() { log_info "Running health check..." if bash "${HEALTH_CHECK_SCRIPT}"; then log_success "Health check passed" return 0 else log_error "Health check failed" return 1 fi } send_notification() { local subject="$1" local body="$2" # Placeholder for email notification # Will be configured with proper email settings in Task 6 log_info "NOTIFICATION: ${subject}" log_info "${body}" # TODO: Implement actual email sending via mail command or SMTP # echo "${body}" | mail -s "${subject}" admin@example.com } # ============================================================================ # Main Execution # ============================================================================ main() { log_info "==========================================" log_info "Gitea Manual Update Started" log_info "Containers: $*" log_info "==========================================" # Validate input validate_args "$@" # Show current and available versions show_current_versions "$@" show_available_versions "$@" # Get user confirmation get_user_confirmation "$@" # Save current state for rollback save_current_images "$@" # Run backup run_backup # Pull new images pull_new_images "$@" # Recreate containers recreate_containers "$@" # Run health check if run_health_check; then log_success "==========================================" log_success "✓ Update completed successfully" log_success "Updated containers: $*" log_success "==========================================" send_notification \ "Gitea Manual Update Successful" \ "Successfully updated containers: $*" cleanup exit 0 else log_error "Health check failed after update" rollback # Run health check again after rollback if run_health_check; then log_success "Rollback successful - services restored" send_notification \ "Gitea Manual Update Failed - Rolled Back" \ "Update of containers [$*] failed health check and was rolled back. Services are now healthy." else log_error "Critical: Services still unhealthy after rollback" send_notification \ "CRITICAL: Gitea Manual Update Failed - Manual Intervention Required" \ "Update of containers [$*] failed and rollback did not restore health. IMMEDIATE ATTENTION REQUIRED." fi cleanup exit 1 fi } main "$@"