b8eb8e991c
All checks were successful
Update Automation Tests / Integration Tests (pull_request) Successful in 37s
- Create restore.sh for automated S3 backup recovery - Fetches backups, stops services, restores database/data/config, restarts & validates - Successfully tested on production system - Document procedures in backup-strategy.md - Add Test 6: Full backup/restore cycle with disaster simulation - Rename test-update.sh → test-integration.sh
359 lines
11 KiB
Bash
359 lines
11 KiB
Bash
#!/bin/bash
|
|
# ============================================================================
|
|
# Gitea Manual Update Script
|
|
# ============================================================================
|
|
# Updates high-risk containers (gitea, postgres) with manual approval,
|
|
# backup, health checks, and automatic rollback on failure.
|
|
#
|
|
# Usage: ./manual-update.sh <container1> [container2] [...]
|
|
# Example: ./manual-update.sh gitea postgres
|
|
#
|
|
# This script requires explicit operator invocation and confirmation.
|
|
# ============================================================================
|
|
|
|
set -e
|
|
|
|
# ============================================================================
|
|
# Configuration
|
|
# ============================================================================
|
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
readonly DOCKER_COMPOSE_DIR="/opt/gitea"
|
|
readonly COMPOSE_FILE="${DOCKER_COMPOSE_DIR}/docker-compose.yml"
|
|
readonly BACKUP_SCRIPT="${SCRIPT_DIR}/backup.sh"
|
|
readonly HEALTH_CHECK_SCRIPT="${SCRIPT_DIR}/health-check.sh"
|
|
readonly LOG_FILE="/var/log/gitea-manual-update.log"
|
|
readonly ROLLBACK_INFO="/tmp/gitea-rollback-info-$$.json"
|
|
|
|
# Wait timeouts (seconds)
|
|
readonly CONTAINER_STARTUP_WAIT=30
|
|
|
|
# Output colors
|
|
readonly GREEN='\033[0;32m'
|
|
readonly YELLOW='\033[1;33m'
|
|
readonly RED='\033[0;31m'
|
|
readonly BLUE='\033[0;34m'
|
|
readonly NC='\033[0m'
|
|
|
|
# ============================================================================
|
|
# Logging Functions
|
|
# ============================================================================
|
|
get_timestamp() {
|
|
date '+%Y-%m-%d %H:%M:%S'
|
|
}
|
|
|
|
log_info() {
|
|
local message="[$(get_timestamp)] [INFO] $1"
|
|
echo -e "${YELLOW}${message}${NC}"
|
|
echo "${message}" >> "${LOG_FILE}"
|
|
}
|
|
|
|
log_success() {
|
|
local message="[$(get_timestamp)] [SUCCESS] $1"
|
|
echo -e "${GREEN}${message}${NC}"
|
|
echo "${message}" >> "${LOG_FILE}"
|
|
}
|
|
|
|
log_error() {
|
|
local message="[$(get_timestamp)] [ERROR] $1"
|
|
echo -e "${RED}${message}${NC}" >&2
|
|
echo "${message}" >> "${LOG_FILE}"
|
|
}
|
|
|
|
log_prompt() {
|
|
echo -e "${BLUE}[PROMPT]${NC} $1"
|
|
}
|
|
|
|
error_exit() {
|
|
log_error "$1"
|
|
cleanup
|
|
exit 1
|
|
}
|
|
|
|
# ============================================================================
|
|
# Cleanup Function
|
|
# ============================================================================
|
|
cleanup() {
|
|
if [ -f "${ROLLBACK_INFO}" ]; then
|
|
rm -f "${ROLLBACK_INFO}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Helper Functions
|
|
# ============================================================================
|
|
change_to_compose_dir() {
|
|
cd "${DOCKER_COMPOSE_DIR}" || error_exit "Failed to change to ${DOCKER_COMPOSE_DIR}"
|
|
}
|
|
|
|
run_compose() {
|
|
docker compose -f "${COMPOSE_FILE}" "$@"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Validation Functions
|
|
# ============================================================================
|
|
validate_args() {
|
|
if [ $# -eq 0 ]; then
|
|
error_exit "No containers specified. Usage: $0 <container1> [container2] [...]"
|
|
fi
|
|
|
|
for container in "$@"; do
|
|
if ! run_compose config --services | grep -q "^${container}$"; then
|
|
error_exit "Container '${container}' not found in docker-compose.yml"
|
|
fi
|
|
done
|
|
|
|
log_success "Container validation passed"
|
|
}
|
|
|
|
# ============================================================================
|
|
# User Confirmation Functions
|
|
# ============================================================================
|
|
get_user_confirmation() {
|
|
local containers="$*"
|
|
|
|
echo ""
|
|
log_prompt "=========================================="
|
|
log_prompt "MANUAL UPDATE CONFIRMATION"
|
|
log_prompt "=========================================="
|
|
log_prompt "You are about to update the following containers:"
|
|
for container in ${containers}; do
|
|
log_prompt " - ${container}"
|
|
done
|
|
echo ""
|
|
log_prompt "This will:"
|
|
log_prompt " 1. Create a backup of database and Gitea data"
|
|
log_prompt " 2. Pull new container images"
|
|
log_prompt " 3. Recreate the containers with new versions"
|
|
log_prompt " 4. Run health checks"
|
|
log_prompt " 5. Rollback automatically if health checks fail"
|
|
echo ""
|
|
log_prompt "Estimated downtime: 1-3 minutes"
|
|
echo ""
|
|
|
|
read -p "Do you want to proceed? (yes/no): " confirmation
|
|
|
|
case "${confirmation}" in
|
|
yes|YES|Yes)
|
|
log_success "Update confirmed by operator"
|
|
return 0
|
|
;;
|
|
*)
|
|
log_info "Update cancelled by operator"
|
|
exit 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
show_current_versions() {
|
|
log_info "Current container versions:"
|
|
|
|
change_to_compose_dir
|
|
|
|
for container in "$@"; do
|
|
local image=$(run_compose images "${container}" 2>/dev/null | tail -n +3 | awk '{print $2":"$3}' | head -n1)
|
|
if [ -n "${image}" ]; then
|
|
log_info " ${container}: ${image}"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
}
|
|
|
|
show_available_versions() {
|
|
log_info "Checking for available updates..."
|
|
|
|
change_to_compose_dir
|
|
|
|
for container in "$@"; do
|
|
log_info " Checking ${container}..."
|
|
run_compose pull --dry-run "${container}" 2>&1 | grep -i "image" || true
|
|
done
|
|
|
|
echo ""
|
|
}
|
|
|
|
# ============================================================================
|
|
# Rollback Management Functions
|
|
# ============================================================================
|
|
save_current_images() {
|
|
log_info "Saving current image versions for rollback..."
|
|
|
|
echo "{" > "${ROLLBACK_INFO}"
|
|
local first=true
|
|
|
|
for container in "$@"; do
|
|
local image=$(run_compose images -q "${container}" 2>/dev/null | head -n1)
|
|
|
|
if [ -n "${image}" ]; then
|
|
if [ "${first}" = true ]; then
|
|
first=false
|
|
else
|
|
echo "," >> "${ROLLBACK_INFO}"
|
|
fi
|
|
echo " \"${container}\": \"${image}\"" >> "${ROLLBACK_INFO}"
|
|
log_info "Saved ${container}: ${image}"
|
|
fi
|
|
done
|
|
|
|
echo "}" >> "${ROLLBACK_INFO}"
|
|
log_success "Current image versions saved"
|
|
}
|
|
|
|
rollback() {
|
|
log_error "Rolling back to previous versions..."
|
|
|
|
if [ ! -f "${ROLLBACK_INFO}" ]; then
|
|
log_error "No rollback information found"
|
|
return 1
|
|
fi
|
|
|
|
change_to_compose_dir
|
|
|
|
# Extract containers from rollback info and restore
|
|
local containers=$(grep -o '"[^"]*":' "${ROLLBACK_INFO}" | tr -d '":' | tr '\n' ' ')
|
|
|
|
for container in ${containers}; do
|
|
log_info "Rolling back ${container}..."
|
|
run_compose up -d "${container}" || log_error "Failed to rollback ${container}"
|
|
done
|
|
|
|
log_success "Rollback completed"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Update Functions
|
|
# ============================================================================
|
|
run_backup() {
|
|
log_info "Running backup before update..."
|
|
|
|
if ! bash "${BACKUP_SCRIPT}"; then
|
|
error_exit "Backup failed - aborting update"
|
|
fi
|
|
|
|
log_success "Backup completed successfully"
|
|
}
|
|
|
|
pull_new_images() {
|
|
log_info "Pulling new images..."
|
|
|
|
change_to_compose_dir
|
|
|
|
for container in "$@"; do
|
|
log_info "Pulling image for ${container}..."
|
|
if ! run_compose pull "${container}"; then
|
|
error_exit "Failed to pull image for ${container}"
|
|
fi
|
|
done
|
|
|
|
log_success "All images pulled successfully"
|
|
}
|
|
|
|
recreate_containers() {
|
|
log_info "Recreating containers..."
|
|
log_info "⚠️ Service downtime begins now"
|
|
|
|
change_to_compose_dir
|
|
|
|
if ! run_compose up -d "$@"; then
|
|
error_exit "Failed to recreate containers"
|
|
fi
|
|
|
|
# Wait for containers to start - longer for database
|
|
log_info "Waiting for containers to start (${CONTAINER_STARTUP_WAIT} seconds)..."
|
|
sleep "${CONTAINER_STARTUP_WAIT}"
|
|
|
|
log_success "Containers recreated successfully"
|
|
}
|
|
|
|
run_health_check() {
|
|
log_info "Running health check..."
|
|
|
|
if bash "${HEALTH_CHECK_SCRIPT}"; then
|
|
log_success "Health check passed"
|
|
return 0
|
|
else
|
|
log_error "Health check failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
send_notification() {
|
|
local subject="$1"
|
|
local body="$2"
|
|
|
|
# Placeholder for email notification
|
|
# Will be configured with proper email settings in Task 6
|
|
log_info "NOTIFICATION: ${subject}"
|
|
log_info "${body}"
|
|
|
|
# TODO: Implement actual email sending via mail command or SMTP
|
|
# echo "${body}" | mail -s "${subject}" admin@example.com
|
|
}
|
|
|
|
# ============================================================================
|
|
# Main Execution
|
|
# ============================================================================
|
|
main() {
|
|
log_info "=========================================="
|
|
log_info "Gitea Manual Update Started"
|
|
log_info "Containers: $*"
|
|
log_info "=========================================="
|
|
|
|
# Validate input
|
|
validate_args "$@"
|
|
|
|
# Show current and available versions
|
|
show_current_versions "$@"
|
|
show_available_versions "$@"
|
|
|
|
# Get user confirmation
|
|
get_user_confirmation "$@"
|
|
|
|
# Save current state for rollback
|
|
save_current_images "$@"
|
|
|
|
# Run backup
|
|
run_backup
|
|
|
|
# Pull new images
|
|
pull_new_images "$@"
|
|
|
|
# Recreate containers
|
|
recreate_containers "$@"
|
|
|
|
# Run health check
|
|
if run_health_check; then
|
|
log_success "=========================================="
|
|
log_success "✓ Update completed successfully"
|
|
log_success "Updated containers: $*"
|
|
log_success "=========================================="
|
|
send_notification \
|
|
"Gitea Manual Update Successful" \
|
|
"Successfully updated containers: $*"
|
|
cleanup
|
|
exit 0
|
|
else
|
|
log_error "Health check failed after update"
|
|
rollback
|
|
|
|
# Run health check again after rollback
|
|
if run_health_check; then
|
|
log_success "Rollback successful - services restored"
|
|
send_notification \
|
|
"Gitea Manual Update Failed - Rolled Back" \
|
|
"Update of containers [$*] failed health check and was rolled back. Services are now healthy."
|
|
else
|
|
log_error "Critical: Services still unhealthy after rollback"
|
|
send_notification \
|
|
"CRITICAL: Gitea Manual Update Failed - Manual Intervention Required" \
|
|
"Update of containers [$*] failed and rollback did not restore health. IMMEDIATE ATTENTION REQUIRED."
|
|
fi
|
|
|
|
cleanup
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
main "$@"
|