gitea_admin
685de1816d
- Diun monitors Docker images - Automated updates for nginx, manual approval for gitea/postgres - Weekly cert renewal automation via cron - Health checks with automatic rollback on failure - AWS SES email notifications on update failures - Daily S3 backups + pre-update snapshots - Integration tests with Gitea Actions quality gate - Change domain from gitea.poll-streams.com to git.poll-streams.com - Add diagrams
481 lines
15 KiB
Bash
Executable File
481 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ============================================================================
|
|
# Update Automation Integration Tests
|
|
# ============================================================================
|
|
# Tests script integration with Docker components in isolated environment.
|
|
# Does NOT touch production infrastructure or AWS services.
|
|
#
|
|
# Requirements:
|
|
# - Docker daemon running
|
|
# - docker compose plugin installed
|
|
#
|
|
# Tests:
|
|
# 1. Script syntax validation (static)
|
|
# 2. Docker Compose configuration validity (static)
|
|
# 3. Backup creates valid archives (integration)
|
|
# 4. Health checks detect container failures (integration)
|
|
# 5. Update workflow with rollback (integration)
|
|
#
|
|
# Usage: ./test-update.sh
|
|
# Exit: 0 if all tests pass, 1 if any test fails
|
|
# ============================================================================
|
|
|
|
set -e
|
|
|
|
# ============================================================================
|
|
# Configuration
|
|
# ============================================================================
|
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
readonly DOCKER_COMPOSE_DIR="$(cd "${SCRIPT_DIR}/../docker" && pwd)"
|
|
readonly BACKUP_SCRIPT="${SCRIPT_DIR}/backup.sh"
|
|
readonly HEALTH_CHECK_SCRIPT="${SCRIPT_DIR}/health-check.sh"
|
|
readonly AUTO_UPDATE_SCRIPT="${SCRIPT_DIR}/auto-update.sh"
|
|
readonly MANUAL_UPDATE_SCRIPT="${SCRIPT_DIR}/manual-update.sh"
|
|
readonly COMPOSE_FILE="${DOCKER_COMPOSE_DIR}/docker-compose.yml"
|
|
readonly TEST_LOG="/tmp/test-update-$$.log"
|
|
readonly TEST_DIR="/tmp/test-gitea-$$"
|
|
|
|
# Test images and credentials
|
|
readonly PG_IMAGE="postgres:18.4"
|
|
readonly PG_USER="testuser"
|
|
readonly PG_PASS="testpass"
|
|
readonly PG_DB="testdb"
|
|
readonly NGINX_IMAGE="nginx:1.27-alpine"
|
|
readonly ALPINE_OLD="alpine:3.19"
|
|
readonly ALPINE_NEW="alpine:3.20"
|
|
|
|
# Wait timeouts (seconds)
|
|
readonly WAIT_TIMEOUT=30
|
|
readonly WAIT_INTERVAL=0.5
|
|
|
|
# Output colors
|
|
readonly GREEN='\033[0;32m'
|
|
readonly RED='\033[0;31m'
|
|
readonly BLUE='\033[0;34m'
|
|
readonly NC='\033[0m' # No Color
|
|
|
|
# Test counters
|
|
TESTS_PASSED=0
|
|
TESTS_FAILED=0
|
|
|
|
# Cleanup tracking
|
|
CONTAINERS_TO_CLEANUP=()
|
|
|
|
# ============================================================================
|
|
# Cleanup Functions
|
|
# ============================================================================
|
|
cleanup() {
|
|
log_info "Cleaning up test environment..."
|
|
|
|
# Stop and remove test containers
|
|
if [[ ${#CONTAINERS_TO_CLEANUP[@]} -gt 0 ]]; then
|
|
for container in "${CONTAINERS_TO_CLEANUP[@]}"; do
|
|
docker rm -f "${container}" &>/dev/null || true
|
|
done
|
|
fi
|
|
|
|
# Remove test directory
|
|
if [[ -d "${TEST_DIR}" ]]; then
|
|
rm -rf "${TEST_DIR}"
|
|
fi
|
|
|
|
log_info "Cleanup complete"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
# ============================================================================
|
|
# Output Functions
|
|
# ============================================================================
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $*" | tee -a "${TEST_LOG}"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[PASS]${NC} $*" | tee -a "${TEST_LOG}"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[FAIL]${NC} $*" | tee -a "${TEST_LOG}"
|
|
}
|
|
|
|
pass_test() {
|
|
local message="$1"
|
|
TESTS_PASSED=$((TESTS_PASSED + 1))
|
|
log_success "${message}"
|
|
}
|
|
|
|
fail_test() {
|
|
local message="$1"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
log_error "${message}"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Helper Functions
|
|
# ============================================================================
|
|
|
|
wait_for_postgres() {
|
|
local container=$1
|
|
local attempts=0
|
|
local max_attempts=$((WAIT_TIMEOUT * 2)) # Check every 0.5s
|
|
|
|
# First wait for container to be running
|
|
while ! docker ps --filter "name=${container}" --format "{{.Names}}" | grep -q "^${container}$"; do
|
|
((attempts++))
|
|
if [[ $attempts -ge $max_attempts ]]; then
|
|
return 1
|
|
fi
|
|
sleep "${WAIT_INTERVAL}"
|
|
done
|
|
|
|
# Then wait for postgres to be ready
|
|
attempts=0
|
|
while ! docker exec "${container}" pg_isready -U "${PG_USER}" &>/dev/null; do
|
|
((attempts++))
|
|
if [[ $attempts -ge $max_attempts ]]; then
|
|
return 1
|
|
fi
|
|
sleep "${WAIT_INTERVAL}"
|
|
done
|
|
|
|
# Give it a moment to fully initialize
|
|
sleep 1
|
|
return 0
|
|
}
|
|
|
|
wait_for_container() {
|
|
local container=$1
|
|
local attempts=0
|
|
local max_attempts=$((WAIT_TIMEOUT * 2))
|
|
|
|
while ! docker ps --filter "name=${container}" --format "{{.Names}}" | grep -q "^${container}$"; do
|
|
((attempts++))
|
|
if [[ $attempts -ge $max_attempts ]]; then
|
|
return 1
|
|
fi
|
|
sleep "${WAIT_INTERVAL}"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
start_postgres_container() {
|
|
local name=$1
|
|
|
|
docker run -d \
|
|
--name "${name}" \
|
|
-e POSTGRES_USER="${PG_USER}" \
|
|
-e POSTGRES_PASSWORD="${PG_PASS}" \
|
|
-e POSTGRES_DB="${PG_DB}" \
|
|
"${PG_IMAGE}" &>> "${TEST_LOG}"
|
|
|
|
CONTAINERS_TO_CLEANUP+=("${name}")
|
|
wait_for_postgres "${name}"
|
|
}
|
|
|
|
start_container() {
|
|
local name=$1
|
|
local image=$2
|
|
shift 2
|
|
local extra_args=("$@")
|
|
|
|
docker run -d \
|
|
--name "${name}" \
|
|
"${image}" \
|
|
"${extra_args[@]}" &>> "${TEST_LOG}"
|
|
|
|
CONTAINERS_TO_CLEANUP+=("${name}")
|
|
wait_for_container "${name}"
|
|
}
|
|
|
|
validate_sql_archive() {
|
|
local file=$1
|
|
local pattern=$2
|
|
|
|
gunzip -t "${file}" 2>> "${TEST_LOG}" && \
|
|
zcat "${file}" | grep -q "${pattern}"
|
|
}
|
|
|
|
validate_tar_archive() {
|
|
local file=$1
|
|
local pattern=$2
|
|
|
|
tar -tzf "${file}" &>> "${TEST_LOG}" && \
|
|
tar -tzf "${file}" | grep -q "${pattern}"
|
|
}
|
|
|
|
get_container_image() {
|
|
local container=$1
|
|
docker inspect --format='{{.Config.Image}}' "${container}"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Test Functions
|
|
# ============================================================================
|
|
|
|
test_script_syntax() {
|
|
log_info "Test 1: Script syntax validation..."
|
|
|
|
local scripts=(
|
|
"${BACKUP_SCRIPT}"
|
|
"${HEALTH_CHECK_SCRIPT}"
|
|
"${AUTO_UPDATE_SCRIPT}"
|
|
"${MANUAL_UPDATE_SCRIPT}"
|
|
)
|
|
|
|
for script in "${scripts[@]}"; do
|
|
if [[ ! -f "${script}" ]]; then
|
|
fail_test "Script not found: ${script}"
|
|
continue
|
|
fi
|
|
|
|
if bash -n "${script}" 2>> "${TEST_LOG}"; then
|
|
pass_test "Syntax valid: $(basename "${script}")"
|
|
else
|
|
fail_test "Syntax error in: $(basename "${script}")"
|
|
fi
|
|
done
|
|
}
|
|
|
|
test_docker_compose_validity() {
|
|
log_info "Test 2: Docker Compose configuration..."
|
|
|
|
if [[ ! -f "${COMPOSE_FILE}" ]]; then
|
|
fail_test "docker-compose.yml not found"
|
|
return
|
|
fi
|
|
|
|
# Validate compose file syntax
|
|
if ! docker compose -f "${COMPOSE_FILE}" config &>> "${TEST_LOG}"; then
|
|
fail_test "docker-compose.yml has syntax errors"
|
|
return
|
|
fi
|
|
pass_test "docker-compose.yml is valid"
|
|
|
|
# Check for latest tags (anti-pattern)
|
|
if grep -E "image:.*:latest" "${COMPOSE_FILE}" &>> "${TEST_LOG}"; then
|
|
fail_test "Found 'latest' tags (versions should be pinned)"
|
|
else
|
|
pass_test "No 'latest' tags (versions properly pinned)"
|
|
fi
|
|
}
|
|
|
|
test_backup_creates_valid_archives() {
|
|
log_info "Test 3: Backup creates valid archives..."
|
|
|
|
# Create test environment
|
|
mkdir -p "${TEST_DIR}/backups"
|
|
mkdir -p "${TEST_DIR}/gitea-data"
|
|
echo "test data" > "${TEST_DIR}/gitea-data/test-file.txt"
|
|
|
|
# Start test postgres container
|
|
local db_container="test-postgres-$$"
|
|
if ! start_postgres_container "${db_container}"; then
|
|
fail_test "Failed to start postgres container"
|
|
return
|
|
fi
|
|
|
|
# Create test table with data
|
|
docker exec "${db_container}" psql -U "${PG_USER}" -d "${PG_DB}" -c \
|
|
"CREATE TABLE test_data (id SERIAL PRIMARY KEY, value TEXT);" &>> "${TEST_LOG}"
|
|
docker exec "${db_container}" psql -U "${PG_USER}" -d "${PG_DB}" -c \
|
|
"INSERT INTO test_data (value) VALUES ('test value');" &>> "${TEST_LOG}"
|
|
|
|
# Test database backup
|
|
local backup_file="${TEST_DIR}/backups/test-backup.sql.gz"
|
|
if ! docker exec "${db_container}" pg_dump -U "${PG_USER}" "${PG_DB}" | gzip > "${backup_file}" 2>> "${TEST_LOG}"; then
|
|
fail_test "Database backup failed"
|
|
return
|
|
fi
|
|
|
|
if ! validate_sql_archive "${backup_file}" "test_data"; then
|
|
fail_test "Database backup archive is invalid"
|
|
return
|
|
fi
|
|
pass_test "Database backup creates valid SQL archive"
|
|
|
|
# Test Gitea data backup
|
|
local data_backup="${TEST_DIR}/backups/test-data.tar.gz"
|
|
if ! tar -czf "${data_backup}" -C "${TEST_DIR}" gitea-data 2>> "${TEST_LOG}"; then
|
|
fail_test "Gitea data backup failed"
|
|
return
|
|
fi
|
|
|
|
if ! validate_tar_archive "${data_backup}" "test-file.txt"; then
|
|
fail_test "Gitea data backup archive is invalid"
|
|
return
|
|
fi
|
|
pass_test "Gitea data backup creates valid tar archive"
|
|
}
|
|
|
|
test_health_checks_detect_failures() {
|
|
log_info "Test 4: Health checks detect container failures..."
|
|
|
|
# Start healthy test container
|
|
local test_container="test-nginx-$$"
|
|
if ! start_container "${test_container}" "${NGINX_IMAGE}"; then
|
|
fail_test "Failed to start nginx container"
|
|
return
|
|
fi
|
|
|
|
# Test 1: Detect running container
|
|
if docker ps --filter "name=${test_container}" --format "{{.Names}}" | grep -q "^${test_container}$"; then
|
|
pass_test "Health check detects running container"
|
|
else
|
|
fail_test "Health check failed to detect running container"
|
|
fi
|
|
|
|
# Test 2: Stop container and verify detection
|
|
docker stop "${test_container}" &>> "${TEST_LOG}"
|
|
sleep 1
|
|
|
|
if ! docker ps --filter "name=${test_container}" --format "{{.Names}}" | grep -q "^${test_container}$"; then
|
|
pass_test "Health check detects stopped container"
|
|
else
|
|
fail_test "Health check failed to detect stopped container"
|
|
fi
|
|
|
|
# Test 3: Start postgres and verify health check
|
|
local pg_container="test-pg-health-$$"
|
|
if ! start_postgres_container "${pg_container}"; then
|
|
fail_test "Failed to start postgres for health check"
|
|
return
|
|
fi
|
|
|
|
# Test pg_isready (how health-check.sh validates postgres)
|
|
if docker exec "${pg_container}" pg_isready -U "${PG_USER}" &>> "${TEST_LOG}"; then
|
|
pass_test "Postgres health check (pg_isready) works"
|
|
else
|
|
fail_test "Postgres health check failed"
|
|
fi
|
|
}
|
|
|
|
test_update_workflow_with_rollback() {
|
|
log_info "Test 5: Update workflow with rollback simulation..."
|
|
|
|
# Create test container with versioned images
|
|
local test_container="test-rollback-$$"
|
|
|
|
# Start with old version
|
|
if ! start_container "${test_container}" "${ALPINE_OLD}" tail -f /dev/null; then
|
|
fail_test "Failed to start container with initial image"
|
|
return
|
|
fi
|
|
|
|
# Verify initial version
|
|
local initial_image=$(get_container_image "${test_container}")
|
|
if [[ "${initial_image}" == "${ALPINE_OLD}" ]]; then
|
|
pass_test "Container starts with correct initial image"
|
|
else
|
|
fail_test "Container has wrong initial image: ${initial_image}"
|
|
fi
|
|
|
|
# Simulate update: save current image info (like auto-update.sh does)
|
|
local saved_image="${initial_image}"
|
|
|
|
# "Update" to new version
|
|
docker rm -f "${test_container}" &>> "${TEST_LOG}"
|
|
if ! start_container "${test_container}" "${ALPINE_NEW}" tail -f /dev/null; then
|
|
fail_test "Failed to update container"
|
|
return
|
|
fi
|
|
|
|
local updated_image=$(get_container_image "${test_container}")
|
|
if [[ "${updated_image}" == "${ALPINE_NEW}" ]]; then
|
|
pass_test "Container updates to new image"
|
|
else
|
|
fail_test "Container update failed"
|
|
fi
|
|
|
|
# Simulate rollback (health check failed scenario)
|
|
docker rm -f "${test_container}" &>> "${TEST_LOG}"
|
|
if ! start_container "${test_container}" "${saved_image}" tail -f /dev/null; then
|
|
fail_test "Failed to rollback container"
|
|
return
|
|
fi
|
|
|
|
local rolled_back_image=$(get_container_image "${test_container}")
|
|
if [[ "${rolled_back_image}" == "${saved_image}" ]]; then
|
|
pass_test "Rollback restores previous image"
|
|
else
|
|
fail_test "Rollback failed: got ${rolled_back_image}, expected ${saved_image}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Main Execution
|
|
# ============================================================================
|
|
main() {
|
|
echo "=========================================="
|
|
echo "Update Automation Integration Tests"
|
|
echo "=========================================="
|
|
echo ""
|
|
log_info "Starting tests at $(date)"
|
|
log_info "Test environment: ${TEST_DIR}"
|
|
echo ""
|
|
|
|
# Check Docker is available
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "Docker is not installed or not in PATH"
|
|
exit 1
|
|
fi
|
|
|
|
if ! docker ps &> /dev/null; then
|
|
log_error "Docker daemon is not running or not accessible"
|
|
exit 1
|
|
fi
|
|
|
|
# Create log file
|
|
: > "${TEST_LOG}"
|
|
|
|
# Create test directory
|
|
mkdir -p "${TEST_DIR}"
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Static Analysis Tests"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
test_script_syntax
|
|
echo ""
|
|
|
|
test_docker_compose_validity
|
|
echo ""
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Integration Tests (Docker Required)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
test_backup_creates_valid_archives
|
|
echo ""
|
|
|
|
test_health_checks_detect_failures
|
|
echo ""
|
|
|
|
test_update_workflow_with_rollback
|
|
echo ""
|
|
|
|
# Summary
|
|
echo "=========================================="
|
|
echo "Test Summary"
|
|
echo "=========================================="
|
|
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
|
|
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
|
|
echo ""
|
|
|
|
if [[ ${TESTS_FAILED} -eq 0 ]]; then
|
|
echo -e "${GREEN}All integration tests passed!${NC}"
|
|
echo ""
|
|
log_info "Full log: ${TEST_LOG}"
|
|
exit 0
|
|
else
|
|
echo -e "${RED}${TESTS_FAILED} test(s) failed${NC}"
|
|
echo ""
|
|
log_error "Full log: ${TEST_LOG}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
main "$@"
|