qvest-task/scripts/test-integration.sh
gitea_admin 2e368a3a7c feat: implement disaster recovery with automated restore (#2)
- Create restore.sh for automated S3 backup recovery
  - Fetches backups, stops services, restores database/data/config, restarts & validates
- Successfully tested on production system
- Document procedures in backup-strategy.md
- Add Test 6: Full backup/restore cycle with disaster simulation
- Rename test-update.sh → test-integration.sh

Co-authored-by: aviyadeveloper <aviya.developer@gmail.com>
Reviewed-on: #2
2026-06-11 17:29:55 +00:00

640 lines
20 KiB
Bash
Executable File

#!/bin/bash
# ============================================================================
# Integration Test Suite
# ============================================================================
# Tests script integration with Docker components in isolated environment.
# Does NOT touch production infrastructure or AWS services.
#
# Requirements:
# - Docker daemon running
# - docker compose plugin installed
#
# Tests:
# 1. Script syntax validation (static)
# 2. Docker Compose configuration validity (static)
# 3. Backup creates valid archives (integration)
# 4. Health checks detect container failures (integration)
# 5. Update workflow with rollback (integration)
# 6. Full backup and restore cycle (integration)
#
# Usage: ./test-integration.sh
# Exit: 0 if all tests pass, 1 if any test fails
# ============================================================================
set -e
# ============================================================================
# Configuration
# ============================================================================
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly DOCKER_COMPOSE_DIR="$(cd "${SCRIPT_DIR}/../docker" && pwd)"
readonly BACKUP_SCRIPT="${SCRIPT_DIR}/backup.sh"
readonly HEALTH_CHECK_SCRIPT="${SCRIPT_DIR}/health-check.sh"
readonly AUTO_UPDATE_SCRIPT="${SCRIPT_DIR}/auto-update.sh"
readonly MANUAL_UPDATE_SCRIPT="${SCRIPT_DIR}/manual-update.sh"
readonly COMPOSE_FILE="${DOCKER_COMPOSE_DIR}/docker-compose.yml"
readonly TEST_LOG="/tmp/test-integration-$$.log"
readonly TEST_DIR="/tmp/test-gitea-$$"
# Test images and credentials
readonly PG_IMAGE="postgres:18.4"
readonly PG_USER="testuser"
readonly PG_PASS="testpass"
readonly PG_DB="testdb"
readonly NGINX_IMAGE="nginx:1.27-alpine"
readonly ALPINE_OLD="alpine:3.19"
readonly ALPINE_NEW="alpine:3.20"
# Wait timeouts (seconds)
readonly WAIT_TIMEOUT=30
readonly WAIT_INTERVAL=0.5
readonly POSTGRES_INIT_DELAY=1
# Output colors
readonly GREEN='\033[0;32m'
readonly RED='\033[0;31m'
readonly BLUE='\033[0;34m'
readonly NC='\033[0m' # No Color
# Test counters
TESTS_PASSED=0
TESTS_FAILED=0
# Cleanup tracking
CONTAINERS_TO_CLEANUP=()
# ============================================================================
# Cleanup Functions
# ============================================================================
cleanup() {
log_info "Cleaning up test environment..."
# Stop and remove test containers
if [[ ${#CONTAINERS_TO_CLEANUP[@]} -gt 0 ]]; then
for container in "${CONTAINERS_TO_CLEANUP[@]}"; do
docker rm -f "${container}" &>/dev/null || true
done
fi
# Remove test directory
if [[ -d "${TEST_DIR}" ]]; then
rm -rf "${TEST_DIR}"
fi
log_info "Cleanup complete"
}
trap cleanup EXIT
# ============================================================================
# Output Functions
# ============================================================================
log_info() {
echo -e "${BLUE}[INFO]${NC} $*" | tee -a "${TEST_LOG}"
}
log_success() {
echo -e "${GREEN}[PASS]${NC} $*" | tee -a "${TEST_LOG}"
}
log_error() {
echo -e "${RED}[FAIL]${NC} $*" | tee -a "${TEST_LOG}"
}
pass_test() {
local message="$1"
TESTS_PASSED=$((TESTS_PASSED + 1))
log_success "${message}"
}
fail_test() {
local message="$1"
TESTS_FAILED=$((TESTS_FAILED + 1))
log_error "${message}"
}
# ============================================================================
# Helper Functions
# ============================================================================
wait_for_postgres() {
local container=$1
local attempts=0
local max_attempts=$((WAIT_TIMEOUT * 2)) # Check every 0.5s
# First wait for container to be running
while ! docker ps --filter "name=${container}" --format "{{.Names}}" | grep -q "^${container}$"; do
((attempts++))
if [[ $attempts -ge $max_attempts ]]; then
return 1
fi
sleep "${WAIT_INTERVAL}"
done
# Then wait for postgres to be ready
attempts=0
while ! docker exec "${container}" pg_isready -U "${PG_USER}" &>/dev/null; do
((attempts++))
if [[ $attempts -ge $max_attempts ]]; then
return 1
fi
sleep "${WAIT_INTERVAL}"
done
# Give it a moment to fully initialize
sleep "${POSTGRES_INIT_DELAY}"
return 0
}
wait_for_container() {
local container=$1
local attempts=0
local max_attempts=$((WAIT_TIMEOUT * 2))
while ! docker ps --filter "name=${container}" --format "{{.Names}}" | grep -q "^${container}$"; do
((attempts++))
if [[ $attempts -ge $max_attempts ]]; then
return 1
fi
sleep "${WAIT_INTERVAL}"
done
return 0
}
start_postgres_container() {
local name=$1
docker run -d \
--name "${name}" \
-e POSTGRES_USER="${PG_USER}" \
-e POSTGRES_PASSWORD="${PG_PASS}" \
-e POSTGRES_DB="${PG_DB}" \
"${PG_IMAGE}" &>> "${TEST_LOG}"
CONTAINERS_TO_CLEANUP+=("${name}")
wait_for_postgres "${name}"
}
start_container() {
local name=$1
local image=$2
shift 2
local extra_args=("$@")
docker run -d \
--name "${name}" \
"${image}" \
"${extra_args[@]}" &>> "${TEST_LOG}"
CONTAINERS_TO_CLEANUP+=("${name}")
wait_for_container "${name}"
}
validate_sql_archive() {
local file=$1
local pattern=$2
gunzip -t "${file}" 2>> "${TEST_LOG}" && \
zcat "${file}" | grep -q "${pattern}"
}
validate_tar_archive() {
local file=$1
local pattern=$2
tar -tzf "${file}" &>> "${TEST_LOG}" && \
tar -tzf "${file}" | grep -q "${pattern}"
}
get_container_image() {
local container=$1
docker inspect --format='{{.Config.Image}}' "${container}"
}
is_container_running() {
local container=$1
docker ps --filter "name=${container}" --format "{{.Names}}" | grep -q "^${container}$"
}
exec_psql() {
local container=$1
local database=$2
local sql=$3
docker exec "${container}" psql -U "${PG_USER}" -d "${database}" -c "${sql}" &>> "${TEST_LOG}"
}
exec_psql_query() {
local container=$1
local database=$2
local query=$3
docker exec "${container}" psql -U "${PG_USER}" -d "${database}" -t -c "${query}" 2>> "${TEST_LOG}" | xargs
}
# ============================================================================
# Test Functions
# ============================================================================
test_script_syntax() {
log_info "Test 1: Script syntax validation..."
local scripts=(
"${BACKUP_SCRIPT}"
"${HEALTH_CHECK_SCRIPT}"
"${AUTO_UPDATE_SCRIPT}"
"${MANUAL_UPDATE_SCRIPT}"
)
for script in "${scripts[@]}"; do
if [[ ! -f "${script}" ]]; then
fail_test "Script not found: ${script}"
continue
fi
if bash -n "${script}" 2>> "${TEST_LOG}"; then
pass_test "Syntax valid: $(basename "${script}")"
else
fail_test "Syntax error in: $(basename "${script}")"
fi
done
}
test_docker_compose_validity() {
log_info "Test 2: Docker Compose configuration..."
if [[ ! -f "${COMPOSE_FILE}" ]]; then
fail_test "docker-compose.yml not found"
return
fi
# Validate compose file syntax
if ! docker compose -f "${COMPOSE_FILE}" config &>> "${TEST_LOG}"; then
fail_test "docker-compose.yml has syntax errors"
return
fi
pass_test "docker-compose.yml is valid"
# Check for latest tags (anti-pattern)
if grep -E "image:.*:latest" "${COMPOSE_FILE}" &>> "${TEST_LOG}"; then
fail_test "Found 'latest' tags (versions should be pinned)"
else
pass_test "No 'latest' tags (versions properly pinned)"
fi
}
test_backup_creates_valid_archives() {
log_info "Test 3: Backup creates valid archives..."
# Create test environment
mkdir -p "${TEST_DIR}/backups"
mkdir -p "${TEST_DIR}/gitea-data"
echo "test data" > "${TEST_DIR}/gitea-data/test-file.txt"
# Start test postgres container
local db_container="test-postgres-$$"
if ! start_postgres_container "${db_container}"; then
fail_test "Failed to start postgres container"
return
fi
# Create test table with data
exec_psql "${db_container}" "${PG_DB}" \
"CREATE TABLE test_data (id SERIAL PRIMARY KEY, value TEXT);"
exec_psql "${db_container}" "${PG_DB}" \
"INSERT INTO test_data (value) VALUES ('test value');"
# Test database backup
local backup_file="${TEST_DIR}/backups/test-backup.sql.gz"
if ! docker exec "${db_container}" pg_dump -U "${PG_USER}" "${PG_DB}" | gzip > "${backup_file}" 2>> "${TEST_LOG}"; then
fail_test "Database backup failed"
return
fi
if ! validate_sql_archive "${backup_file}" "test_data"; then
fail_test "Database backup archive is invalid"
return
fi
pass_test "Database backup creates valid SQL archive"
# Test Gitea data backup
local data_backup="${TEST_DIR}/backups/test-data.tar.gz"
if ! tar -czf "${data_backup}" -C "${TEST_DIR}" gitea-data 2>> "${TEST_LOG}"; then
fail_test "Gitea data backup failed"
return
fi
if ! validate_tar_archive "${data_backup}" "test-file.txt"; then
fail_test "Gitea data backup archive is invalid"
return
fi
pass_test "Gitea data backup creates valid tar archive"
}
test_health_checks_detect_failures() {
log_info "Test 4: Health checks detect container failures..."
# Start healthy test container
local test_container="test-nginx-$$"
if ! start_container "${test_container}" "${NGINX_IMAGE}"; then
fail_test "Failed to start nginx container"
return
fi
# Test 1: Detect running container
if is_container_running "${test_container}"; then
pass_test "Health check detects running container"
else
fail_test "Health check failed to detect running container"
fi
# Test 2: Stop container and verify detection
docker stop "${test_container}" &>> "${TEST_LOG}"
sleep 1
if ! is_container_running "${test_container}"; then
pass_test "Health check detects stopped container"
else
fail_test "Health check failed to detect stopped container"
fi
# Test 3: Start postgres and verify health check
local pg_container="test-pg-health-$$"
if ! start_postgres_container "${pg_container}"; then
fail_test "Failed to start postgres for health check"
return
fi
# Test pg_isready (how health-check.sh validates postgres)
if docker exec "${pg_container}" pg_isready -U "${PG_USER}" &>> "${TEST_LOG}"; then
pass_test "Postgres health check (pg_isready) works"
else
fail_test "Postgres health check failed"
fi
}
test_update_workflow_with_rollback() {
log_info "Test 5: Update workflow with rollback simulation..."
# Create test container with versioned images
local test_container="test-rollback-$$"
# Start with old version
if ! start_container "${test_container}" "${ALPINE_OLD}" tail -f /dev/null; then
fail_test "Failed to start container with initial image"
return
fi
# Verify initial version
local initial_image=$(get_container_image "${test_container}")
if [[ "${initial_image}" == "${ALPINE_OLD}" ]]; then
pass_test "Container starts with correct initial image"
else
fail_test "Container has wrong initial image: ${initial_image}"
fi
# Simulate update: save current image info (like auto-update.sh does)
local saved_image="${initial_image}"
# "Update" to new version
docker rm -f "${test_container}" &>> "${TEST_LOG}"
if ! start_container "${test_container}" "${ALPINE_NEW}" tail -f /dev/null; then
fail_test "Failed to update container"
return
fi
local updated_image=$(get_container_image "${test_container}")
if [[ "${updated_image}" == "${ALPINE_NEW}" ]]; then
pass_test "Container updates to new image"
else
fail_test "Container update failed"
fi
# Simulate rollback (health check failed scenario)
docker rm -f "${test_container}" &>> "${TEST_LOG}"
if ! start_container "${test_container}" "${saved_image}" tail -f /dev/null; then
fail_test "Failed to rollback container"
return
fi
local rolled_back_image=$(get_container_image "${test_container}")
if [[ "${rolled_back_image}" == "${saved_image}" ]]; then
pass_test "Rollback restores previous image"
else
fail_test "Rollback failed: got ${rolled_back_image}, expected ${saved_image}"
fi
}
test_backup_and_restore_cycle() {
log_info "Test 6: Full backup and restore cycle..."
# Create test database container
local db_container="test-restore-db-$$"
if ! start_postgres_container "${db_container}"; then
fail_test "Failed to start postgres for restore test"
return
fi
# Create test data and directory structure
mkdir -p "${TEST_DIR}/restore-test/data"
mkdir -p "${TEST_DIR}/restore-test/backups"
echo "original content" > "${TEST_DIR}/restore-test/data/test-file.txt"
echo "config data" > "${TEST_DIR}/restore-test/data/config.yml"
# Create database with test data
exec_psql "${db_container}" "${PG_DB}" \
"CREATE TABLE restore_test (id SERIAL PRIMARY KEY, data TEXT, created_at TIMESTAMP DEFAULT NOW());"
exec_psql "${db_container}" "${PG_DB}" \
"INSERT INTO restore_test (data) VALUES ('original data'), ('test record 1'), ('test record 2');"
# Verify original data exists
local original_count=$(exec_psql_query "${db_container}" "${PG_DB}" \
"SELECT COUNT(*) FROM restore_test;")
if [[ "${original_count}" -ne 3 ]]; then
fail_test "Failed to create test data (expected 3 rows, got ${original_count})"
return
fi
pass_test "Test data created successfully (3 rows)"
# Step 1: Create backups
local timestamp="test-$$"
local db_backup="${TEST_DIR}/restore-test/backups/database-${timestamp}.sql.gz"
local data_backup="${TEST_DIR}/restore-test/backups/data-${timestamp}.tar.gz"
if ! docker exec "${db_container}" pg_dump -U "${PG_USER}" "${PG_DB}" | gzip > "${db_backup}" 2>> "${TEST_LOG}"; then
fail_test "Database backup failed"
return
fi
if ! tar -czf "${data_backup}" -C "${TEST_DIR}/restore-test" data 2>> "${TEST_LOG}"; then
fail_test "Data directory backup failed"
return
fi
pass_test "Backups created successfully"
# Step 2: Corrupt/destroy the data (simulate disaster)
exec_psql "${db_container}" "${PG_DB}" \
"DELETE FROM restore_test;"
exec_psql "${db_container}" "${PG_DB}" \
"INSERT INTO restore_test (data) VALUES ('corrupted data');"
rm -f "${TEST_DIR}/restore-test/data/test-file.txt"
echo "corrupted content" > "${TEST_DIR}/restore-test/data/test-file.txt"
# Verify data is corrupted
local corrupted_count=$(exec_psql_query "${db_container}" "${PG_DB}" \
"SELECT COUNT(*) FROM restore_test;")
if [[ "${corrupted_count}" -ne 1 ]]; then
fail_test "Data corruption simulation failed"
return
fi
pass_test "Data corruption simulated (1 row instead of 3)"
# Step 3: Restore database from backup
if ! zcat "${db_backup}" | docker exec -i "${db_container}" psql -U "${PG_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${PG_DB};" &>> "${TEST_LOG}"; then
fail_test "Failed to drop database"
return
fi
if ! exec_psql "${db_container}" postgres "CREATE DATABASE ${PG_DB};"; then
fail_test "Failed to recreate database"
return
fi
if ! zcat "${db_backup}" | docker exec -i "${db_container}" psql -U "${PG_USER}" -d "${PG_DB}" &>> "${TEST_LOG}"; then
fail_test "Database restore failed"
return
fi
pass_test "Database restored from backup"
# Step 4: Restore data directory
rm -rf "${TEST_DIR}/restore-test/data"
if ! tar -xzf "${data_backup}" -C "${TEST_DIR}/restore-test" 2>> "${TEST_LOG}"; then
fail_test "Data directory restore failed"
return
fi
pass_test "Data directory restored from backup"
# Step 5: Verify restored data matches original
local restored_count=$(exec_psql_query "${db_container}" "${PG_DB}" \
"SELECT COUNT(*) FROM restore_test;")
if [[ "${restored_count}" -ne 3 ]]; then
fail_test "Restored data count mismatch (expected 3, got ${restored_count})"
return
fi
local restored_data=$(exec_psql_query "${db_container}" "${PG_DB}" \
"SELECT data FROM restore_test ORDER BY id LIMIT 1;")
if [[ "${restored_data}" != "original data" ]]; then
fail_test "Restored data content mismatch (expected 'original data', got '${restored_data}')"
return
fi
pass_test "Database data restored correctly (3 rows, original content)"
# Verify file content
local restored_file_content=$(cat "${TEST_DIR}/restore-test/data/test-file.txt")
if [[ "${restored_file_content}" != "original content" ]]; then
fail_test "Restored file content mismatch"
return
fi
if [[ ! -f "${TEST_DIR}/restore-test/data/config.yml" ]]; then
fail_test "Config file missing after restore"
return
fi
pass_test "File system data restored correctly"
# Step 6: Verify database is operational after restore
if ! exec_psql "${db_container}" "${PG_DB}" \
"INSERT INTO restore_test (data) VALUES ('post-restore test');"; then
fail_test "Database not operational after restore"
return
fi
local final_count=$(exec_psql_query "${db_container}" "${PG_DB}" \
"SELECT COUNT(*) FROM restore_test;")
if [[ "${final_count}" -ne 4 ]]; then
fail_test "Post-restore database operations failed"
return
fi
pass_test "Database fully operational after restore"
}
# ============================================================================
# Main Execution
# ============================================================================
main() {
echo "=========================================="
echo "Integration Test Suite"
echo "=========================================="
echo ""
log_info "Starting tests at $(date)"
log_info "Test environment: ${TEST_DIR}"
echo ""
# Check Docker is available
if ! command -v docker &> /dev/null; then
log_error "Docker is not installed or not in PATH"
exit 1
fi
if ! docker ps &> /dev/null; then
log_error "Docker daemon is not running or not accessible"
exit 1
fi
# Create log file
: > "${TEST_LOG}"
# Create test directory
mkdir -p "${TEST_DIR}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Static Analysis Tests"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
test_script_syntax
echo ""
test_docker_compose_validity
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Integration Tests (Docker Required)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
test_backup_creates_valid_archives
echo ""
test_health_checks_detect_failures
echo ""
test_update_workflow_with_rollback
echo ""
test_backup_and_restore_cycle
echo ""
# Summary
echo "=========================================="
echo "Test Summary"
echo "=========================================="
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
echo ""
if [[ ${TESTS_FAILED} -eq 0 ]]; then
echo -e "${GREEN}All integration tests passed!${NC}"
echo ""
log_info "Full log: ${TEST_LOG}"
exit 0
else
echo -e "${RED}${TESTS_FAILED} test(s) failed${NC}"
echo ""
log_error "Full log: ${TEST_LOG}"
exit 1
fi
}
main "$@"