forked from baron/baron-sso
백업/복구로직 변경, 깜빡임 버그 해결
This commit is contained in:
15
scripts/backup/dump-list.sh
Executable file
15
scripts/backup/dump-list.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
backup_root="${BACKUP_ROOT:-$repo_root/backups}"
|
||||
|
||||
if [[ ! -d "$backup_root" ]]; then
|
||||
backup_log "No backup directory found: $backup_root"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
find "$backup_root" -maxdepth 1 -type d -name 'baron-sso-backup-*' | sort
|
||||
69
scripts/backup/dump.sh
Executable file
69
scripts/backup/dump.sh
Executable file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
source "$script_dir/lib/manifest.sh"
|
||||
source "$script_dir/lib/postgres.sh"
|
||||
source "$script_dir/lib/clickhouse.sh"
|
||||
source "$script_dir/lib/config.sh"
|
||||
source "$script_dir/lib/report.sh"
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
services="$(normalize_service_filter "${DUMP_SERVICES:-all}")"
|
||||
mode="${DUMP_MODE:-maintenance}"
|
||||
backup_root="${BACKUP_ROOT:-$repo_root/backups}"
|
||||
backup_dir="${BACKUP:-$backup_root/baron-sso-backup-$(backup_timestamp)}"
|
||||
|
||||
mkdir -p "$backup_dir/reports"
|
||||
create_manifest "$backup_dir" "$mode" "$services"
|
||||
service_timings_json="[]"
|
||||
|
||||
run_backup_step() {
|
||||
local service="$1"
|
||||
shift
|
||||
|
||||
local started_at
|
||||
local finished_at
|
||||
local duration_seconds
|
||||
|
||||
started_at="$(date +%s)"
|
||||
"$@"
|
||||
finished_at="$(date +%s)"
|
||||
duration_seconds="$((finished_at - started_at))"
|
||||
service_timings_json="$(jq -c \
|
||||
--arg service "$service" \
|
||||
--argjson duration "$duration_seconds" \
|
||||
'. + [{service:$service, duration_seconds:$duration}]' \
|
||||
<<<"$service_timings_json")"
|
||||
}
|
||||
|
||||
backup_log "Creating backup at $backup_dir"
|
||||
backup_log "Backup mode: $mode"
|
||||
backup_log "Services: $services"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
run_backup_step postgres dump_baron_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
run_backup_step ory-postgres dump_ory_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
run_backup_step clickhouse dump_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
run_backup_step ory-clickhouse dump_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
run_backup_step config dump_config_snapshot "$backup_dir"
|
||||
fi
|
||||
|
||||
write_backup_markdown_report "$backup_dir" "succeeded" "$services" "$service_timings_json"
|
||||
backup_checksum_file "$backup_dir"
|
||||
BACKUP="$backup_dir" "$script_dir/verify-dump.sh"
|
||||
|
||||
backup_log "Backup complete: $backup_dir"
|
||||
115
scripts/backup/lib/clickhouse.sh
Normal file
115
scripts/backup/lib/clickhouse.sh
Normal file
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
clickhouse_query() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local query="$4"
|
||||
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" --query "$query"
|
||||
}
|
||||
|
||||
render_clickhouse_schema() {
|
||||
local schema_file="$1"
|
||||
|
||||
if head -n 1 "$schema_file" | grep -q '\\n'; then
|
||||
perl -0pe 's{\\n}{\n}g; s{\\t}{\t}g; s{\\\x27}{\x27}g' "$schema_file"
|
||||
return
|
||||
fi
|
||||
|
||||
cat "$schema_file"
|
||||
}
|
||||
|
||||
dump_clickhouse_container() {
|
||||
local backup_dir="$1"
|
||||
local service_name="$2"
|
||||
local container="$3"
|
||||
local user="$4"
|
||||
local password="$5"
|
||||
local output_dir="$backup_dir/clickhouse/$service_name"
|
||||
local table_list="$output_dir/tables.tsv"
|
||||
local database
|
||||
local table
|
||||
local engine
|
||||
local safe_name
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container "$container"
|
||||
mkdir -p "$output_dir/schema" "$output_dir/data" "$backup_dir/reports"
|
||||
|
||||
backup_log "Dumping ClickHouse metadata and Native data: $container"
|
||||
clickhouse_query "$container" "$user" "$password" \
|
||||
"select database, name, engine from system.tables where database not in ('INFORMATION_SCHEMA','information_schema','system') order by database, if(positionCaseInsensitive(engine, 'View') > 0, 1, 0), name format TSV" \
|
||||
>"$table_list"
|
||||
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
safe_name="${database}__${table}"
|
||||
clickhouse_query "$container" "$user" "$password" "show create table \`${database}\`.\`${table}\` FORMAT RawBLOB" >"$output_dir/schema/${safe_name}.sql"
|
||||
if [[ "$engine" != *View* ]]; then
|
||||
clickhouse_query "$container" "$user" "$password" "select * from \`${database}\`.\`${table}\` format Native" >"$output_dir/data/${safe_name}.native"
|
||||
fi
|
||||
clickhouse_query "$container" "$user" "$password" "select '${database}.${table}:' || toString(count()) from \`${database}\`.\`${table}\`" \
|
||||
>>"$backup_dir/reports/${service_name}-row-counts.txt"
|
||||
done <"$table_list"
|
||||
}
|
||||
|
||||
restore_clickhouse_container() {
|
||||
local backup_dir="$1"
|
||||
local service_name="$2"
|
||||
local container="$3"
|
||||
local user="$4"
|
||||
local password="$5"
|
||||
local input_dir="$backup_dir/clickhouse/$service_name"
|
||||
local table_list="$input_dir/tables.tsv"
|
||||
local database
|
||||
local table
|
||||
local engine
|
||||
local safe_name
|
||||
local restored_databases=""
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container "$container"
|
||||
backup_require_path "$table_list"
|
||||
|
||||
backup_log "Restoring ClickHouse tables: $container"
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
if ! grep -qw -- "$database" <<<"$restored_databases"; then
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "drop database if exists \`${database}\`"
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "create database if not exists \`${database}\`"
|
||||
restored_databases="${restored_databases:+$restored_databases }$database"
|
||||
fi
|
||||
done <"$table_list"
|
||||
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
safe_name="${database}__${table}"
|
||||
backup_require_path "$input_dir/schema/${safe_name}.sql"
|
||||
render_clickhouse_schema "$input_dir/schema/${safe_name}.sql" \
|
||||
| docker exec -i "$container" clickhouse-client --user "$user" --password "$password" --multiquery
|
||||
if [[ "$engine" != *View* ]]; then
|
||||
backup_require_path "$input_dir/data/${safe_name}.native"
|
||||
docker exec -i "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "insert into \`${database}\`.\`${table}\` format Native" <"$input_dir/data/${safe_name}.native"
|
||||
fi
|
||||
done <"$table_list"
|
||||
}
|
||||
|
||||
dump_baron_clickhouse() {
|
||||
dump_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
|
||||
}
|
||||
|
||||
dump_ory_clickhouse() {
|
||||
dump_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
|
||||
}
|
||||
|
||||
restore_baron_clickhouse() {
|
||||
restore_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
|
||||
}
|
||||
|
||||
restore_ory_clickhouse() {
|
||||
restore_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
|
||||
}
|
||||
137
scripts/backup/lib/common.sh
Normal file
137
scripts/backup/lib/common.sh
Normal file
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
BACKUP_SUPPORTED_SERVICES="postgres ory-postgres clickhouse ory-clickhouse config"
|
||||
|
||||
backup_repo_root() {
|
||||
if [[ -n "${BACKUP_REPO_ROOT:-}" ]]; then
|
||||
printf '%s\n' "$BACKUP_REPO_ROOT"
|
||||
return
|
||||
fi
|
||||
|
||||
local script_dir
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
printf '%s\n' "$script_dir"
|
||||
}
|
||||
|
||||
backup_log() {
|
||||
printf '==> %s\n' "$*"
|
||||
}
|
||||
|
||||
backup_die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
backup_require_command() {
|
||||
local command_name="$1"
|
||||
command -v "$command_name" >/dev/null 2>&1 || backup_die "required command not found: $command_name"
|
||||
}
|
||||
|
||||
backup_utc_now() {
|
||||
date -u '+%Y-%m-%dT%H:%M:%SZ'
|
||||
}
|
||||
|
||||
backup_timestamp() {
|
||||
date -u '+%Y%m%d-%H%M%SZ'
|
||||
}
|
||||
|
||||
backup_git_commit() {
|
||||
local repo_root="$1"
|
||||
git -c "safe.directory=$repo_root" -C "$repo_root" rev-parse --short=12 HEAD 2>/dev/null || printf 'unknown'
|
||||
}
|
||||
|
||||
normalize_service_filter() {
|
||||
local raw="${1:-all}"
|
||||
local normalized=""
|
||||
local candidate
|
||||
|
||||
if [[ "$raw" == "all" || -z "$raw" ]]; then
|
||||
printf '%s\n' "$BACKUP_SUPPORTED_SERVICES"
|
||||
return
|
||||
fi
|
||||
|
||||
raw="${raw//,/ }"
|
||||
for candidate in $raw; do
|
||||
if ! grep -qw -- "$candidate" <<<"$BACKUP_SUPPORTED_SERVICES"; then
|
||||
backup_die "unknown backup service: $candidate"
|
||||
return 1
|
||||
fi
|
||||
if ! grep -qw -- "$candidate" <<<"$normalized"; then
|
||||
normalized="${normalized:+$normalized }$candidate"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ -n "$normalized" ]] || backup_die "service filter must not be empty"
|
||||
printf '%s\n' "$normalized"
|
||||
}
|
||||
|
||||
service_enabled() {
|
||||
local service="$1"
|
||||
local services="$2"
|
||||
local candidate
|
||||
|
||||
for candidate in $services; do
|
||||
[[ "$candidate" == "$service" ]] && return 0
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
backup_require_path() {
|
||||
local path="$1"
|
||||
[[ -e "$path" ]] || backup_die "required path not found: $path"
|
||||
}
|
||||
|
||||
backup_container_running() {
|
||||
local container="$1"
|
||||
docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null | grep -qx 'true'
|
||||
}
|
||||
|
||||
backup_require_container() {
|
||||
local container="$1"
|
||||
backup_container_running "$container" || backup_die "container is not running: $container"
|
||||
}
|
||||
|
||||
backup_redact_env() {
|
||||
local source_file="$1"
|
||||
local target_file="$2"
|
||||
|
||||
sed -E '/^[[:space:]]*#/! s/^([^=]*(SECRET|PASSWORD|TOKEN|KEY|PRIVATE|CLIENT_SECRET|COOKIE)[^=]*)=.*/\1=REDACTED/I' "$source_file" >"$target_file"
|
||||
}
|
||||
|
||||
backup_checksum_file() {
|
||||
local backup_dir="$1"
|
||||
(
|
||||
cd "$backup_dir"
|
||||
find . -type f ! -name 'checksums.sha256' -print0 \
|
||||
| sort -z \
|
||||
| xargs -0 sha256sum
|
||||
) >"$backup_dir/checksums.sha256"
|
||||
}
|
||||
|
||||
backup_verify_checksums() {
|
||||
local backup_dir="$1"
|
||||
backup_require_path "$backup_dir/checksums.sha256"
|
||||
(
|
||||
cd "$backup_dir"
|
||||
sha256sum -c checksums.sha256
|
||||
)
|
||||
}
|
||||
|
||||
backup_safe_tar() {
|
||||
local source_path="$1"
|
||||
local target_base="$2"
|
||||
local source_parent
|
||||
local source_name
|
||||
|
||||
[[ -e "$source_path" ]] || return 0
|
||||
|
||||
source_parent="$(dirname "$source_path")"
|
||||
source_name="$(basename "$source_path")"
|
||||
|
||||
if command -v zstd >/dev/null 2>&1; then
|
||||
tar --zstd -cf "${target_base}.tar.zst" -C "$source_parent" "$source_name"
|
||||
else
|
||||
tar -czf "${target_base}.tar.gz" -C "$source_parent" "$source_name"
|
||||
fi
|
||||
}
|
||||
37
scripts/backup/lib/config.sh
Normal file
37
scripts/backup/lib/config.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
dump_config_snapshot() {
|
||||
local backup_dir="$1"
|
||||
local repo_root
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
mkdir -p "$backup_dir/config"
|
||||
|
||||
if [[ -f "$repo_root/.env" ]]; then
|
||||
backup_redact_env "$repo_root/.env" "$backup_dir/config/env.redacted"
|
||||
fi
|
||||
|
||||
backup_safe_tar "$repo_root/config/.generated/ory" "$backup_dir/config/generated-ory"
|
||||
backup_safe_tar "$repo_root/gateway" "$backup_dir/config/gateway"
|
||||
|
||||
mkdir -p "$backup_dir/config/compose"
|
||||
for compose_file in compose.infra.yaml compose.ory.yaml docker-compose.yaml; do
|
||||
if [[ -f "$repo_root/$compose_file" ]]; then
|
||||
cp "$repo_root/$compose_file" "$backup_dir/config/compose/$compose_file"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_config_snapshot() {
|
||||
local backup_dir="$1"
|
||||
local repo_root
|
||||
local output_dir
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
output_dir="$repo_root/config-restored"
|
||||
|
||||
backup_require_path "$backup_dir/config"
|
||||
mkdir -p "$output_dir"
|
||||
cp -R "$backup_dir/config/." "$output_dir/"
|
||||
backup_log "Config snapshot was copied to $output_dir for manual review."
|
||||
}
|
||||
42
scripts/backup/lib/manifest.sh
Normal file
42
scripts/backup/lib/manifest.sh
Normal file
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
create_manifest() {
|
||||
local backup_dir="$1"
|
||||
local mode="$2"
|
||||
local services="$3"
|
||||
local repo_root
|
||||
local created_at
|
||||
local git_commit
|
||||
local service
|
||||
local first=1
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
created_at="$(backup_utc_now)"
|
||||
git_commit="$(backup_git_commit "$repo_root")"
|
||||
|
||||
{
|
||||
printf '{\n'
|
||||
printf ' "format_version": "1",\n'
|
||||
printf ' "created_at": "%s",\n' "$created_at"
|
||||
printf ' "git_commit": "%s",\n' "$git_commit"
|
||||
printf ' "mode": "%s",\n' "$mode"
|
||||
printf ' "environment_scope": "same-env-only",\n'
|
||||
printf ' "services": ['
|
||||
for service in $services; do
|
||||
if [[ "$first" -eq 1 ]]; then
|
||||
first=0
|
||||
else
|
||||
printf ', '
|
||||
fi
|
||||
printf '"%s"' "$service"
|
||||
done
|
||||
printf '],\n'
|
||||
printf ' "restore_policy": {\n'
|
||||
printf ' "requires_empty_target": true,\n'
|
||||
printf ' "requires_confirmation": "baron-sso",\n'
|
||||
printf ' "auto_run_migrations": false,\n'
|
||||
printf ' "works_relay_auto_resume": false\n'
|
||||
printf ' }\n'
|
||||
printf '}\n'
|
||||
} >"$backup_dir/manifest.json"
|
||||
}
|
||||
95
scripts/backup/lib/postgres.sh
Normal file
95
scripts/backup/lib/postgres.sh
Normal file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
dump_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
mkdir -p "$backup_dir/postgres" "$backup_dir/reports"
|
||||
|
||||
backup_log "Dumping Baron Postgres database: $db_name"
|
||||
docker exec -e "PGPASSWORD=$db_password" baron_postgres \
|
||||
pg_dump -U "$db_user" -d "$db_name" -Fc >"$backup_dir/postgres/baron.dump"
|
||||
|
||||
docker exec -e "PGPASSWORD=$db_password" baron_postgres \
|
||||
psql -U "$db_user" -d "$db_name" -Atc "select schemaname || '.' || relname || ':' || (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I.%I', schemaname, relname), false, true, '')))[1]::text from pg_stat_user_tables order by 1" \
|
||||
>"$backup_dir/reports/baron-postgres-row-counts.txt"
|
||||
}
|
||||
|
||||
dump_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local hydra_db="${HYDRA_DB:-ory_hydra}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local db_name
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
mkdir -p "$backup_dir/postgres" "$backup_dir/reports"
|
||||
|
||||
backup_log "Dumping Ory Postgres globals"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
pg_dumpall -U "$db_user" --globals-only >"$backup_dir/postgres/globals.sql"
|
||||
|
||||
for db_name in "$kratos_db" "$hydra_db" "$keto_db"; do
|
||||
backup_log "Dumping Ory Postgres database: $db_name"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
pg_dump -U "$db_user" -d "$db_name" -Fc >"$backup_dir/postgres/${db_name}.dump"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$db_name" -Atc "select schemaname || '.' || relname || ':' || (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I.%I', schemaname, relname), false, true, '')))[1]::text from pg_stat_user_tables order by 1" \
|
||||
>"$backup_dir/reports/${db_name}-row-counts.txt"
|
||||
done
|
||||
}
|
||||
|
||||
restore_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
|
||||
backup_require_path "$backup_dir/postgres/baron.dump"
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
|
||||
backup_log "Restoring Baron Postgres database: $db_name"
|
||||
docker exec -i -e "PGPASSWORD=$db_password" baron_postgres \
|
||||
pg_restore -U "$db_user" -d "$db_name" --clean --if-exists <"$backup_dir/postgres/baron.dump"
|
||||
}
|
||||
|
||||
restore_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local hydra_db="${HYDRA_DB:-ory_hydra}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local db_name
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
|
||||
for db_name in "$kratos_db" "$hydra_db" "$keto_db"; do
|
||||
backup_require_path "$backup_dir/postgres/${db_name}.dump"
|
||||
backup_log "Restoring Ory Postgres database: $db_name"
|
||||
docker exec -i -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
pg_restore -U "$db_user" -d "$db_name" --clean --if-exists <"$backup_dir/postgres/${db_name}.dump"
|
||||
done
|
||||
}
|
||||
|
||||
postgres_target_has_data() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container "$container"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -Atc "select exists (select 1 from pg_tables where schemaname not in ('pg_catalog','information_schema') limit 1)" \
|
||||
2>/dev/null | grep -qx 't'
|
||||
}
|
||||
142
scripts/backup/lib/report.sh
Normal file
142
scripts/backup/lib/report.sh
Normal file
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
report_count_from_file() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
|
||||
if [[ ! -f "$file_path" ]]; then
|
||||
printf '0\n'
|
||||
return
|
||||
fi
|
||||
|
||||
awk -F: -v key="$key" '$1 == key {print $2; found=1; exit} END {if (!found) print "0"}' "$file_path"
|
||||
}
|
||||
|
||||
report_first_count() {
|
||||
local key="$1"
|
||||
shift
|
||||
|
||||
local file_path
|
||||
local count
|
||||
|
||||
for file_path in "$@"; do
|
||||
count="$(report_count_from_file "$file_path" "$key")"
|
||||
if [[ "$count" != "0" ]]; then
|
||||
printf '%s\n' "$count"
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
printf '0\n'
|
||||
}
|
||||
|
||||
write_backup_markdown_report() {
|
||||
local backup_dir="$1"
|
||||
local status="$2"
|
||||
local services="$3"
|
||||
local timings_json="${4:-[]}"
|
||||
local reports_dir="$backup_dir/reports"
|
||||
local output_file="$reports_dir/backup-report.md"
|
||||
local created_at
|
||||
local manifest_created_at="unknown"
|
||||
local git_commit="unknown"
|
||||
local users
|
||||
local tenants
|
||||
local relying_parties
|
||||
local hydra_clients
|
||||
local works_org_units
|
||||
local works_users
|
||||
local timings_table
|
||||
|
||||
mkdir -p "$reports_dir"
|
||||
created_at="$(backup_utc_now)"
|
||||
|
||||
if [[ -f "$backup_dir/manifest.json" ]]; then
|
||||
manifest_created_at="$(jq -r '.created_at // "unknown"' "$backup_dir/manifest.json")"
|
||||
git_commit="$(jq -r '.git_commit // "unknown"' "$backup_dir/manifest.json")"
|
||||
fi
|
||||
|
||||
users="$(report_first_count "public.users" "$reports_dir/baron-postgres-row-counts.txt")"
|
||||
tenants="$(report_first_count "public.tenants" "$reports_dir/baron-postgres-row-counts.txt")"
|
||||
relying_parties="$(report_first_count "public.relying_parties" "$reports_dir/baron-postgres-row-counts.txt")"
|
||||
hydra_clients="$(report_first_count "public.hydra_client" "$reports_dir/ory_hydra-row-counts.txt")"
|
||||
works_org_units="$(report_first_count "public.works_org_units" "$reports_dir/baron-postgres-row-counts.txt")"
|
||||
works_users="$(report_first_count "public.works_users" "$reports_dir/baron-postgres-row-counts.txt")"
|
||||
|
||||
timings_table="$(jq -r '
|
||||
if type != "array" or length == 0 then
|
||||
"| 없음 | 0 |"
|
||||
else
|
||||
.[] | "| \(.service) | \(.duration_seconds) |"
|
||||
end
|
||||
' <<<"$timings_json")"
|
||||
|
||||
{
|
||||
printf '# Baron SSO Backup Report\n\n'
|
||||
printf '| 항목 | 값 |\n'
|
||||
printf '| --- | --- |\n'
|
||||
printf '| 생성 시각 | %s |\n' "$created_at"
|
||||
printf '| 백업 시각 | %s |\n' "$manifest_created_at"
|
||||
printf '| 상태 | %s |\n' "$status"
|
||||
printf '| 백업 경로 | `%s` |\n' "$backup_dir"
|
||||
printf '| Git Commit | `%s` |\n' "$git_commit"
|
||||
printf '| 서비스 | `%s` |\n\n' "$services"
|
||||
printf '## 요약\n\n'
|
||||
printf '| 지표 | 값 |\n'
|
||||
printf '| --- | ---: |\n'
|
||||
printf '| 사용자 | %s |\n' "$users"
|
||||
printf '| 테넌트 | %s |\n' "$tenants"
|
||||
printf '| RP | %s |\n' "$relying_parties"
|
||||
printf '| Hydra Client | %s |\n' "$hydra_clients"
|
||||
printf '| WORKS 조직 | %s |\n' "$works_org_units"
|
||||
printf '| WORKS 사용자 | %s |\n\n' "$works_users"
|
||||
printf '## 서비스별 수행 시간\n\n'
|
||||
printf '| 서비스 | 초 |\n'
|
||||
printf '| --- | ---: |\n'
|
||||
printf '%s\n' "$timings_table"
|
||||
} >"$output_file"
|
||||
}
|
||||
|
||||
write_restore_markdown_report() {
|
||||
local restore_json="$1"
|
||||
local output_file
|
||||
|
||||
[[ -f "$restore_json" ]] || return 0
|
||||
output_file="${restore_json%.json}.md"
|
||||
|
||||
jq -r '
|
||||
def services: (.services // [] | join(", "));
|
||||
def verification_rows:
|
||||
(.verification.target_reports // []) as $reports
|
||||
| if ($reports | length) == 0 then
|
||||
"| 없음 | not_run | |"
|
||||
else
|
||||
$reports[]
|
||||
| "| \(.service) | \(.status) | \(.diff_file // "") |"
|
||||
end;
|
||||
"# Baron SSO Restore Report\n",
|
||||
"| 항목 | 값 |",
|
||||
"| --- | --- |",
|
||||
"| 시작 시각 | \(.started_at // "unknown") |",
|
||||
"| 종료 시각 | \(.finished_at // "unknown") |",
|
||||
"| 상태 | \(.status // "unknown") |",
|
||||
"| 메시지 | \(.message // "") |",
|
||||
"| 입력 유형 | \(.backup_source // "unknown") |",
|
||||
"| 백업 경로 | `\(.backup_dir // "")` |",
|
||||
"| Dump 파일 | `\(.dump_file // "")` |",
|
||||
"| 서비스 | `\(services)` |",
|
||||
"",
|
||||
"## 검증",
|
||||
"",
|
||||
"| 항목 | 상태 |",
|
||||
"| --- | --- |",
|
||||
"| Dump checksum | \(.verification.dump_checksum // "not_run") |",
|
||||
"| 대상 row count | \(.verification.target_row_counts // "not_run") |",
|
||||
"",
|
||||
"## 대상별 검증 결과",
|
||||
"",
|
||||
"| 서비스 | 상태 | Diff |",
|
||||
"| --- | --- | --- |",
|
||||
verification_rows
|
||||
' "$restore_json" >"$output_file"
|
||||
}
|
||||
5
scripts/backup/restore-plan.sh
Executable file
5
scripts/backup/restore-plan.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BACKUP="${BACKUP:-${1:-}}" DUMP_FILE="${DUMP_FILE:-}" RESTORE_REPORT="${RESTORE_REPORT:-}" "$script_dir/restore.sh" --dry-run
|
||||
460
scripts/backup/restore.sh
Executable file
460
scripts/backup/restore.sh
Executable file
@@ -0,0 +1,460 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
source "$script_dir/lib/postgres.sh"
|
||||
source "$script_dir/lib/clickhouse.sh"
|
||||
source "$script_dir/lib/config.sh"
|
||||
source "$script_dir/lib/report.sh"
|
||||
|
||||
dry_run=false
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
dry_run=true
|
||||
fi
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
backup_input="${BACKUP:-}"
|
||||
dump_file="${DUMP_FILE:-}"
|
||||
backup_source="directory"
|
||||
temp_extract_dir=""
|
||||
report_path=""
|
||||
report_started_at="$(backup_utc_now)"
|
||||
report_status="started"
|
||||
report_message=""
|
||||
dump_checksum_status="not_run"
|
||||
target_verification_status="not_run"
|
||||
target_verification_reports="[]"
|
||||
|
||||
json_array_from_words() {
|
||||
local words="$1"
|
||||
if [[ -z "$words" ]]; then
|
||||
printf '[]\n'
|
||||
return
|
||||
fi
|
||||
|
||||
printf '%s\n' $words | jq -R . | jq -sc .
|
||||
}
|
||||
|
||||
write_restore_report() {
|
||||
local status="$1"
|
||||
local message="${2:-}"
|
||||
local finished_at
|
||||
local services_json
|
||||
local restore_policy_json="{}"
|
||||
|
||||
[[ -n "$report_path" ]] || return 0
|
||||
|
||||
finished_at="$(backup_utc_now)"
|
||||
services_json="$(json_array_from_words "${services:-}")"
|
||||
if [[ -n "${backup_dir:-}" && -f "$backup_dir/manifest.json" ]]; then
|
||||
restore_policy_json="$(jq -c '.restore_policy // {}' "$backup_dir/manifest.json")"
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$report_path")"
|
||||
jq -n \
|
||||
--arg format_version "1" \
|
||||
--arg started_at "$report_started_at" \
|
||||
--arg finished_at "$finished_at" \
|
||||
--arg status "$status" \
|
||||
--arg message "$message" \
|
||||
--arg backup_source "$backup_source" \
|
||||
--arg backup_dir "${backup_dir:-}" \
|
||||
--arg dump_file "$dump_file" \
|
||||
--argjson services "$services_json" \
|
||||
--arg allow_non_empty_restore "${allow_non_empty:-false}" \
|
||||
--arg dry_run "$dry_run" \
|
||||
--arg dump_checksum "$dump_checksum_status" \
|
||||
--arg target_row_counts "$target_verification_status" \
|
||||
--argjson target_reports "$target_verification_reports" \
|
||||
--argjson restore_policy "$restore_policy_json" \
|
||||
'{
|
||||
format_version: $format_version,
|
||||
started_at: $started_at,
|
||||
finished_at: $finished_at,
|
||||
status: $status,
|
||||
message: $message,
|
||||
backup_source: $backup_source,
|
||||
backup_dir: $backup_dir,
|
||||
dump_file: (if $dump_file == "" then null else $dump_file end),
|
||||
services: $services,
|
||||
allow_non_empty_restore: ($allow_non_empty_restore == "true"),
|
||||
dry_run: ($dry_run == "true"),
|
||||
restore_policy: $restore_policy,
|
||||
verification: {
|
||||
dump_checksum: $dump_checksum,
|
||||
target_row_counts: $target_row_counts,
|
||||
target_reports: $target_reports
|
||||
}
|
||||
}' >"$report_path"
|
||||
write_restore_markdown_report "$report_path"
|
||||
}
|
||||
|
||||
cleanup_restore_input() {
|
||||
if [[ -n "$temp_extract_dir" ]]; then
|
||||
rm -rf "$temp_extract_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
on_restore_error() {
|
||||
local exit_code=$?
|
||||
write_restore_report "failed" "${report_message:-restore failed}"
|
||||
cleanup_restore_input
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
trap on_restore_error ERR
|
||||
trap cleanup_restore_input EXIT
|
||||
|
||||
resolve_backup_input() {
|
||||
if [[ -n "$backup_input" && -n "$dump_file" ]]; then
|
||||
backup_die "set only one of BACKUP or DUMP_FILE for restore."
|
||||
fi
|
||||
|
||||
if [[ -n "$backup_input" ]]; then
|
||||
backup_dir="$backup_input"
|
||||
backup_require_path "$backup_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -z "$dump_file" ]]; then
|
||||
backup_die "BACKUP or DUMP_FILE is required. Example: make restore BACKUP=backups/baron-sso-backup-YYYYMMDD-HHMMSSZ CONFIRM_RESTORE=baron-sso"
|
||||
fi
|
||||
|
||||
backup_require_path "$dump_file"
|
||||
backup_require_command tar
|
||||
temp_extract_dir="$(mktemp -d /tmp/baron-sso-restore.XXXXXX)"
|
||||
backup_source="dump_file"
|
||||
|
||||
case "$dump_file" in
|
||||
*.tar.zst)
|
||||
backup_require_command zstd
|
||||
tar --zstd --no-same-owner -xf "$dump_file" -C "$temp_extract_dir"
|
||||
;;
|
||||
*.tar.gz | *.tgz)
|
||||
tar -xzf "$dump_file" -C "$temp_extract_dir"
|
||||
;;
|
||||
*)
|
||||
backup_die "unsupported DUMP_FILE archive format: $dump_file"
|
||||
;;
|
||||
esac
|
||||
|
||||
mapfile -t manifest_files < <(find "$temp_extract_dir" -type f -name manifest.json | sort)
|
||||
if [[ "${#manifest_files[@]}" -ne 1 ]]; then
|
||||
backup_die "DUMP_FILE must contain exactly one backup directory with manifest.json."
|
||||
fi
|
||||
|
||||
backup_dir="$(dirname "${manifest_files[0]}")"
|
||||
}
|
||||
|
||||
quote_pg_ident() {
|
||||
local raw="$1"
|
||||
printf '"%s"' "${raw//\"/\"\"}"
|
||||
}
|
||||
|
||||
collect_postgres_exact_row_counts() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local output_file="$5"
|
||||
local schema
|
||||
local table
|
||||
local quoted_schema
|
||||
local quoted_table
|
||||
local count
|
||||
|
||||
: >"$output_file"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -F $'\t' \
|
||||
-c "select schemaname, tablename from pg_tables where schemaname not in ('pg_catalog','information_schema') order by 1,2" \
|
||||
| while IFS=$'\t' read -r schema table; do
|
||||
[[ -n "$schema" && -n "$table" ]] || continue
|
||||
quoted_schema="$(quote_pg_ident "$schema")"
|
||||
quoted_table="$(quote_pg_ident "$table")"
|
||||
count="$(docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At \
|
||||
-c "select count(*) from ${quoted_schema}.${quoted_table}")"
|
||||
printf '%s.%s:%s\n' "$schema" "$table" "$count"
|
||||
done | sort >"$output_file"
|
||||
}
|
||||
|
||||
collect_postgres_dump_row_counts() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local dump_path="$5"
|
||||
local output_file="$6"
|
||||
local scratch_db
|
||||
local scratch_ident
|
||||
|
||||
backup_require_path "$dump_path"
|
||||
scratch_db="${database}_restore_verify_$(date -u '+%Y%m%d%H%M%S')_$$"
|
||||
scratch_ident="$(quote_pg_ident "$scratch_db")"
|
||||
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "drop database if exists ${scratch_ident} with (force)" \
|
||||
-c "create database ${scratch_ident}"
|
||||
|
||||
docker exec -i -e "PGPASSWORD=$password" "$container" \
|
||||
pg_restore -U "$user" -d "$scratch_db" --clean --if-exists <"$dump_path"
|
||||
|
||||
collect_postgres_exact_row_counts "$container" "$user" "$password" "$scratch_db" "$output_file"
|
||||
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "drop database if exists ${scratch_ident} with (force)"
|
||||
}
|
||||
|
||||
collect_clickhouse_exact_row_counts() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local table_list="$4"
|
||||
local output_file="$5"
|
||||
local database
|
||||
local table
|
||||
local engine
|
||||
local count
|
||||
|
||||
: >"$output_file"
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
count="$(docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "select count() from \`${database}\`.\`${table}\`")"
|
||||
printf '%s.%s:%s\n' "$database" "$table" "$count"
|
||||
done <"$table_list" | sort >"$output_file"
|
||||
}
|
||||
|
||||
collect_clickhouse_native_stable_row_counts() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local input_dir="$4"
|
||||
local output_file="$5"
|
||||
local scratch_db
|
||||
local database
|
||||
local table
|
||||
local engine
|
||||
local safe_name
|
||||
|
||||
scratch_db="$(basename "$input_dir")_restore_verify_$(date -u '+%Y%m%d%H%M%S')_$$"
|
||||
: >"$output_file"
|
||||
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "drop database if exists \`${scratch_db}\`"
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "create database \`${scratch_db}\`"
|
||||
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
if [[ "$engine" == *View* || "$engine" == *AggregatingMergeTree* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
safe_name="${database}__${table}"
|
||||
backup_require_path "$input_dir/data/${safe_name}.native"
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "create table \`${scratch_db}\`.\`${table}\` as \`${database}\`.\`${table}\`"
|
||||
docker exec -i "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "insert into \`${scratch_db}\`.\`${table}\` format Native" <"$input_dir/data/${safe_name}.native"
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "select '${database}.${table}:' || toString(count()) from \`${scratch_db}\`.\`${table}\`" \
|
||||
>>"$output_file"
|
||||
done <"$input_dir/tables.tsv"
|
||||
|
||||
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
||||
--query "drop database if exists \`${scratch_db}\`"
|
||||
sort -o "$output_file" "$output_file"
|
||||
}
|
||||
|
||||
filter_clickhouse_stable_row_counts() {
|
||||
local table_list="$1"
|
||||
local counts_file="$2"
|
||||
local output_file="$3"
|
||||
local database
|
||||
local table
|
||||
local engine
|
||||
|
||||
: >"$output_file"
|
||||
while IFS=$'\t' read -r database table engine; do
|
||||
[[ -n "$database" && -n "$table" ]] || continue
|
||||
if [[ "$engine" == *View* || "$engine" == *AggregatingMergeTree* ]]; then
|
||||
continue
|
||||
fi
|
||||
grep -F "${database}.${table}:" "$counts_file" >>"$output_file" || true
|
||||
done <"$table_list"
|
||||
sort -o "$output_file" "$output_file"
|
||||
}
|
||||
|
||||
compare_row_count_report() {
|
||||
local label="$1"
|
||||
local expected_file="$2"
|
||||
local actual_file="$3"
|
||||
local diff_file="$4"
|
||||
|
||||
backup_require_path "$expected_file"
|
||||
if diff -u <(sort "$expected_file") <(sort "$actual_file") >"$diff_file"; then
|
||||
jq -n \
|
||||
--arg label "$label" \
|
||||
--arg expected "$expected_file" \
|
||||
--arg actual "$actual_file" \
|
||||
--arg status "passed" \
|
||||
'{label:$label, expected:$expected, actual:$actual, status:$status}'
|
||||
else
|
||||
jq -n \
|
||||
--arg label "$label" \
|
||||
--arg expected "$expected_file" \
|
||||
--arg actual "$actual_file" \
|
||||
--arg diff "$diff_file" \
|
||||
--arg status "failed" \
|
||||
'{label:$label, expected:$expected, actual:$actual, diff:$diff, status:$status}'
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
verify_restored_targets() {
|
||||
local report_dir
|
||||
local report_items=()
|
||||
local item
|
||||
local expected
|
||||
local actual
|
||||
local diff_file
|
||||
local db_name
|
||||
|
||||
report_dir="$(dirname "$report_path")/restore-targets-$(date -u '+%Y%m%d-%H%M%SZ')"
|
||||
mkdir -p "$report_dir"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
expected="$report_dir/baron-postgres-expected-row-counts.txt"
|
||||
actual="$report_dir/baron-postgres-row-counts.txt"
|
||||
diff_file="$report_dir/baron-postgres-row-counts.diff"
|
||||
collect_postgres_dump_row_counts baron_postgres "${DB_USER:-baron}" "${DB_PASSWORD:-password}" "${DB_NAME:-baron_sso}" "$backup_dir/postgres/baron.dump" "$expected"
|
||||
collect_postgres_exact_row_counts baron_postgres "${DB_USER:-baron}" "${DB_PASSWORD:-password}" "${DB_NAME:-baron_sso}" "$actual"
|
||||
item="$(compare_row_count_report "postgres" "$expected" "$actual" "$diff_file")"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
for db_name in "${KRATOS_DB:-ory_kratos}" "${HYDRA_DB:-ory_hydra}" "${KETO_DB:-ory_keto}"; do
|
||||
expected="$report_dir/${db_name}-expected-row-counts.txt"
|
||||
actual="$report_dir/${db_name}-row-counts.txt"
|
||||
diff_file="$report_dir/${db_name}-row-counts.diff"
|
||||
collect_postgres_dump_row_counts ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$db_name" "$backup_dir/postgres/${db_name}.dump" "$expected"
|
||||
collect_postgres_exact_row_counts ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$db_name" "$actual"
|
||||
item="$(compare_row_count_report "ory-postgres/$db_name" "$expected" "$actual" "$diff_file")"
|
||||
report_items+=("$item")
|
||||
done
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
expected="$report_dir/baron_clickhouse-stable-expected-row-counts.txt"
|
||||
actual="$report_dir/baron_clickhouse-stable-row-counts.txt"
|
||||
diff_file="$report_dir/baron_clickhouse-row-counts.diff"
|
||||
collect_clickhouse_exact_row_counts baron_clickhouse "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}" "$backup_dir/clickhouse/baron_clickhouse/tables.tsv" "$actual"
|
||||
mv "$actual" "$report_dir/baron_clickhouse-row-counts.txt"
|
||||
collect_clickhouse_native_stable_row_counts baron_clickhouse "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}" "$backup_dir/clickhouse/baron_clickhouse" "$expected"
|
||||
filter_clickhouse_stable_row_counts "$backup_dir/clickhouse/baron_clickhouse/tables.tsv" "$report_dir/baron_clickhouse-row-counts.txt" "$actual"
|
||||
item="$(compare_row_count_report "clickhouse" "$expected" "$actual" "$diff_file")"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
expected="$report_dir/ory_clickhouse-stable-expected-row-counts.txt"
|
||||
actual="$report_dir/ory_clickhouse-stable-row-counts.txt"
|
||||
diff_file="$report_dir/ory_clickhouse-row-counts.diff"
|
||||
collect_clickhouse_exact_row_counts ory_clickhouse "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}" "$backup_dir/clickhouse/ory_clickhouse/tables.tsv" "$actual"
|
||||
mv "$actual" "$report_dir/ory_clickhouse-row-counts.txt"
|
||||
collect_clickhouse_native_stable_row_counts ory_clickhouse "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}" "$backup_dir/clickhouse/ory_clickhouse" "$expected"
|
||||
filter_clickhouse_stable_row_counts "$backup_dir/clickhouse/ory_clickhouse/tables.tsv" "$report_dir/ory_clickhouse-row-counts.txt" "$actual"
|
||||
item="$(compare_row_count_report "ory-clickhouse" "$expected" "$actual" "$diff_file")"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
backup_require_path "$repo_root/config-restored"
|
||||
item="$(jq -n \
|
||||
--arg label "config" \
|
||||
--arg actual "$repo_root/config-restored" \
|
||||
--arg status "passed" \
|
||||
'{label:$label, actual:$actual, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
target_verification_reports="$(printf '%s\n' "${report_items[@]}" | jq -s '.')"
|
||||
target_verification_status="passed"
|
||||
}
|
||||
|
||||
resolve_backup_input
|
||||
|
||||
if [[ -n "${RESTORE_REPORT:-}" ]]; then
|
||||
report_path="$RESTORE_REPORT"
|
||||
elif [[ "$backup_source" == "dump_file" ]]; then
|
||||
archive_name="$(basename "$dump_file")"
|
||||
archive_name="${archive_name%.tar.zst}"
|
||||
archive_name="${archive_name%.tar.gz}"
|
||||
archive_name="${archive_name%.tgz}"
|
||||
report_path="$repo_root/reports/restore/${archive_name}-restore-report.json"
|
||||
else
|
||||
report_path="$backup_dir/reports/restore-report.json"
|
||||
fi
|
||||
|
||||
if [[ "${CONFIRM_RESTORE:-}" != "baron-sso" ]]; then
|
||||
backup_die "CONFIRM_RESTORE=baron-sso is required for restore."
|
||||
fi
|
||||
|
||||
services="$(normalize_service_filter "${RESTORE_SERVICES:-all}")"
|
||||
allow_non_empty="${ALLOW_NON_EMPTY_RESTORE:-false}"
|
||||
|
||||
if [[ "${RESTORE_TEST_NON_EMPTY:-}" == "1" && "$allow_non_empty" != "true" ]]; then
|
||||
backup_die "non-empty restore target is not allowed by default. Set ALLOW_NON_EMPTY_RESTORE=true only for an approved restore rehearsal."
|
||||
fi
|
||||
|
||||
if [[ "$dry_run" == "true" ]]; then
|
||||
backup_log "Restore plan for $backup_dir"
|
||||
backup_log "Services: $services"
|
||||
backup_log "ALLOW_NON_EMPTY_RESTORE=$allow_non_empty"
|
||||
backup_log "RESTORE_REPORT=$report_path"
|
||||
write_restore_report "planned" "restore dry-run completed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$allow_non_empty" != "true" ]]; then
|
||||
if service_enabled postgres "$services" && postgres_target_has_data baron_postgres "${DB_USER:-baron}" "${DB_PASSWORD:-password}" "${DB_NAME:-baron_sso}"; then
|
||||
backup_die "non-empty restore target is not allowed by default: baron_postgres/${DB_NAME:-baron_sso}"
|
||||
fi
|
||||
if service_enabled ory-postgres "$services" && postgres_target_has_data ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "${KRATOS_DB:-ory_kratos}"; then
|
||||
backup_die "non-empty restore target is not allowed by default: ory_postgres/${KRATOS_DB:-ory_kratos}"
|
||||
fi
|
||||
fi
|
||||
|
||||
BACKUP="$backup_dir" "$script_dir/verify-dump.sh"
|
||||
dump_checksum_status="passed"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
restore_baron_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
restore_ory_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
restore_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
restore_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
restore_config_snapshot "$backup_dir"
|
||||
fi
|
||||
|
||||
verify_restored_targets
|
||||
write_restore_report "succeeded" "restore completed and target row-count verification passed"
|
||||
|
||||
backup_log "Restore complete. Keep WORKS relay disabled until comparison dry-run passes."
|
||||
backup_log "Restore report: $report_path"
|
||||
642
scripts/backup/upload_cloud.sh
Executable file
642
scripts/backup/upload_cloud.sh
Executable file
@@ -0,0 +1,642 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
|
||||
if [[ -f "$repo_root/.env" ]]; then
|
||||
env_override_keys=(
|
||||
WORKS_DRIVE_TARGET
|
||||
WORKS_DRIVE_SHARED_DRIVE_ID
|
||||
WORKS_DRIVE_PARENT_FILE_ID
|
||||
WORKS_DRIVE_USER_ID
|
||||
WORKS_DRIVE_GROUP_ID
|
||||
WORKS_DRIVE_SHARED_FOLDER_ID
|
||||
WORKS_DRIVE_ACCESS_TOKEN
|
||||
WORKS_DRIVE_ACCESS_TOKEN_FILE
|
||||
WORKS_DRIVE_ACCESS_TOKEN_CMD
|
||||
WORKS_DRIVE_OAUTH_SCOPE
|
||||
WORKS_DRIVE_SPLIT_SIZE
|
||||
WORKS_DRIVE_MAX_SINGLE_FILE_BYTES
|
||||
WORKS_DRIVE_FORCE_SPLIT
|
||||
WORKS_DRIVE_OVERWRITE
|
||||
WORKS_DRIVE_DRY_RUN
|
||||
WORKS_DRIVE_UPLOAD_REPORTS
|
||||
WORKS_DRIVE_REPORT_FOLDER_NAME
|
||||
WORKS_DRIVE_CURL_BIN
|
||||
WORKS_DRIVE_ARCHIVE_DIR
|
||||
WORKS_DRIVE_SHAREDRIVE_ID
|
||||
WORKS_DRIVE_SHAREDRIVE_BACKUP_DIR
|
||||
WORKS_DRIVE_OAUTH_CLIENT_ID
|
||||
WORKS_DRIVE_OAUTH_CLIENT_SECRET
|
||||
WORKS_DRIVE_OAUTH_CLIENT_SERVICE_ACCOUNT
|
||||
WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY
|
||||
WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY_FILE
|
||||
WORKS_DRIVE_OAUTH_REFRESH_TOKEN
|
||||
WORKS_SHAREDRIVE_ID
|
||||
WORKS_SHAREDRIVE_BACKUP_DIR
|
||||
WORKS_ADMIN_API_BASE_URL
|
||||
WORKS_ADMIN_OAUTH_TOKEN_URL
|
||||
)
|
||||
declare -A env_override_values=()
|
||||
env_override_set=()
|
||||
for env_key in "${env_override_keys[@]}"; do
|
||||
if [[ -v "$env_key" ]]; then
|
||||
env_override_set+=("$env_key")
|
||||
env_override_values["$env_key"]="${!env_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$repo_root/.env"
|
||||
set +a
|
||||
|
||||
for env_key in "${env_override_set[@]}"; do
|
||||
printf -v "$env_key" '%s' "${env_override_values[$env_key]}"
|
||||
export "$env_key"
|
||||
done
|
||||
fi
|
||||
|
||||
backup_path="${BACKUP:-${1:-}}"
|
||||
[[ -n "$backup_path" ]] || backup_die "BACKUP is required. Example: make upload-cloud BACKUP=backups/baron-sso-backup-YYYYMMDD-HHMMSSZ"
|
||||
backup_require_path "$backup_path"
|
||||
|
||||
WORKS_DRIVE_SHARED_DRIVE_ID="${WORKS_DRIVE_SHARED_DRIVE_ID:-${WORKS_DRIVE_SHAREDRIVE_ID:-${WORKS_SHAREDRIVE_ID:-}}}"
|
||||
WORKS_DRIVE_PARENT_FILE_ID="${WORKS_DRIVE_PARENT_FILE_ID:-${WORKS_DRIVE_SHAREDRIVE_BACKUP_DIR:-${WORKS_SHAREDRIVE_BACKUP_DIR:-}}}"
|
||||
|
||||
dry_run="${WORKS_DRIVE_DRY_RUN:-false}"
|
||||
target="${WORKS_DRIVE_TARGET:-sharedrive}"
|
||||
api_base_url="${WORKS_ADMIN_API_BASE_URL:-https://www.worksapis.com}"
|
||||
curl_bin="${WORKS_DRIVE_CURL_BIN:-curl}"
|
||||
archive_dir="${WORKS_DRIVE_ARCHIVE_DIR:-/tmp/baron-sso-backup-upload}"
|
||||
split_size="${WORKS_DRIVE_SPLIT_SIZE:-9000M}"
|
||||
force_split="${WORKS_DRIVE_FORCE_SPLIT:-false}"
|
||||
max_single_file_bytes="${WORKS_DRIVE_MAX_SINGLE_FILE_BYTES:-0}"
|
||||
overwrite="${WORKS_DRIVE_OVERWRITE:-false}"
|
||||
upload_scope="${WORKS_DRIVE_OAUTH_SCOPE:-file}"
|
||||
upload_reports="${WORKS_DRIVE_UPLOAD_REPORTS:-true}"
|
||||
report_folder_name="${WORKS_DRIVE_REPORT_FOLDER_NAME:-reports}"
|
||||
report_dir="$backup_path/reports"
|
||||
|
||||
if [[ -f "$backup_path" ]]; then
|
||||
report_dir="$(dirname "$backup_path")"
|
||||
fi
|
||||
|
||||
mkdir -p "$archive_dir" "$report_dir"
|
||||
|
||||
backup_require_command jq
|
||||
|
||||
urlencode_path() {
|
||||
jq -nr --arg value "$1" '$value|@uri'
|
||||
}
|
||||
|
||||
json_string() {
|
||||
jq -nr --arg value "$1" '$value'
|
||||
}
|
||||
|
||||
bytes_from_size() {
|
||||
local raw="$1"
|
||||
local number
|
||||
local unit
|
||||
|
||||
if [[ "$raw" =~ ^([0-9]+)([KkMmGgTt]?)$ ]]; then
|
||||
number="${BASH_REMATCH[1]}"
|
||||
unit="${BASH_REMATCH[2]}"
|
||||
case "$unit" in
|
||||
K | k) printf '%s\n' $((number * 1024)) ;;
|
||||
M | m) printf '%s\n' $((number * 1024 * 1024)) ;;
|
||||
G | g) printf '%s\n' $((number * 1024 * 1024 * 1024)) ;;
|
||||
T | t) printf '%s\n' $((number * 1024 * 1024 * 1024 * 1024)) ;;
|
||||
*) printf '%s\n' "$number" ;;
|
||||
esac
|
||||
return
|
||||
fi
|
||||
|
||||
backup_die "invalid size value: $raw"
|
||||
}
|
||||
|
||||
split_curl_response() {
|
||||
local response="$1"
|
||||
local __body_var="$2"
|
||||
local __status_var="$3"
|
||||
local status
|
||||
local body
|
||||
|
||||
status="$(tail -n 1 <<<"$response")"
|
||||
if [[ "$status" =~ ^[0-9][0-9][0-9]$ ]]; then
|
||||
body="$(sed '$d' <<<"$response")"
|
||||
else
|
||||
status="200"
|
||||
body="$response"
|
||||
fi
|
||||
|
||||
printf -v "$__body_var" '%s' "$body"
|
||||
printf -v "$__status_var" '%s' "$status"
|
||||
}
|
||||
|
||||
redact_for_log() {
|
||||
sed -E 's/("(access_token|refresh_token|assertion|client_secret|Authorization)"[[:space:]]*:[[:space:]]*)"[^"]*"/\1"REDACTED"/Ig'
|
||||
}
|
||||
|
||||
resolve_target_upload_endpoint() {
|
||||
local parent_file_id="${1:-${WORKS_DRIVE_PARENT_FILE_ID:-}}"
|
||||
local encoded_parent=""
|
||||
|
||||
if [[ -n "$parent_file_id" ]]; then
|
||||
encoded_parent="$(urlencode_path "$parent_file_id")"
|
||||
fi
|
||||
|
||||
case "$target" in
|
||||
sharedrive)
|
||||
[[ -n "${WORKS_DRIVE_SHARED_DRIVE_ID:-}" ]] || backup_die "WORKS_DRIVE_SHARED_DRIVE_ID is required when WORKS_DRIVE_TARGET=sharedrive."
|
||||
local shared_drive_id
|
||||
shared_drive_id="$(urlencode_path "$WORKS_DRIVE_SHARED_DRIVE_ID")"
|
||||
if [[ -n "$encoded_parent" ]]; then
|
||||
printf '%s/v1.0/sharedrives/%s/files/%s\n' "$api_base_url" "$shared_drive_id" "$encoded_parent"
|
||||
else
|
||||
printf '%s/v1.0/sharedrives/%s/files\n' "$api_base_url" "$shared_drive_id"
|
||||
fi
|
||||
;;
|
||||
mydrive)
|
||||
local user_id="${WORKS_DRIVE_USER_ID:-me}"
|
||||
user_id="$(urlencode_path "$user_id")"
|
||||
if [[ -n "$encoded_parent" ]]; then
|
||||
printf '%s/v1.0/users/%s/drive/files/%s\n' "$api_base_url" "$user_id" "$encoded_parent"
|
||||
else
|
||||
printf '%s/v1.0/users/%s/drive/files\n' "$api_base_url" "$user_id"
|
||||
fi
|
||||
;;
|
||||
group)
|
||||
[[ -n "${WORKS_DRIVE_GROUP_ID:-}" ]] || backup_die "WORKS_DRIVE_GROUP_ID is required when WORKS_DRIVE_TARGET=group."
|
||||
local group_id
|
||||
group_id="$(urlencode_path "$WORKS_DRIVE_GROUP_ID")"
|
||||
if [[ -n "$encoded_parent" ]]; then
|
||||
printf '%s/v1.0/groups/%s/folder/files/%s\n' "$api_base_url" "$group_id" "$encoded_parent"
|
||||
else
|
||||
printf '%s/v1.0/groups/%s/folder/files\n' "$api_base_url" "$group_id"
|
||||
fi
|
||||
;;
|
||||
sharedfolder)
|
||||
[[ -n "${WORKS_DRIVE_SHARED_FOLDER_ID:-}" ]] || backup_die "WORKS_DRIVE_SHARED_FOLDER_ID is required when WORKS_DRIVE_TARGET=sharedfolder."
|
||||
local user_id="${WORKS_DRIVE_USER_ID:-me}"
|
||||
local shared_folder_id
|
||||
user_id="$(urlencode_path "$user_id")"
|
||||
shared_folder_id="$(urlencode_path "$WORKS_DRIVE_SHARED_FOLDER_ID")"
|
||||
if [[ -n "$encoded_parent" ]]; then
|
||||
printf '%s/v1.0/users/%s/drive/sharedfolders/%s/files/%s\n' "$api_base_url" "$user_id" "$shared_folder_id" "$encoded_parent"
|
||||
else
|
||||
printf '%s/v1.0/users/%s/drive/sharedfolders/%s/files\n' "$api_base_url" "$user_id" "$shared_folder_id"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
backup_die "unknown WORKS_DRIVE_TARGET: $target"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_target_children_endpoint() {
|
||||
local parent_file_id="${1:-${WORKS_DRIVE_PARENT_FILE_ID:-}}"
|
||||
local upload_endpoint
|
||||
|
||||
upload_endpoint="$(resolve_target_upload_endpoint "$parent_file_id")"
|
||||
printf '%s/children\n' "$upload_endpoint"
|
||||
}
|
||||
|
||||
resolve_target_create_folder_endpoint() {
|
||||
local parent_file_id="${1:-${WORKS_DRIVE_PARENT_FILE_ID:-}}"
|
||||
local upload_endpoint
|
||||
|
||||
upload_endpoint="$(resolve_target_upload_endpoint "$parent_file_id")"
|
||||
printf '%s/createfolder\n' "$upload_endpoint"
|
||||
}
|
||||
|
||||
base64url() {
|
||||
openssl base64 -A | tr '+/' '-_' | tr -d '='
|
||||
}
|
||||
|
||||
build_jwt_assertion() {
|
||||
backup_require_command openssl
|
||||
|
||||
local client_id="${WORKS_DRIVE_OAUTH_CLIENT_ID:-}"
|
||||
local service_account="${WORKS_DRIVE_OAUTH_CLIENT_SERVICE_ACCOUNT:-}"
|
||||
local private_key="${WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY:-}"
|
||||
local private_key_file="${WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY_FILE:-}"
|
||||
local now
|
||||
local exp
|
||||
local header
|
||||
local payload
|
||||
local signing_input
|
||||
local key_file=""
|
||||
local temp_key_file=""
|
||||
|
||||
[[ -n "$client_id" ]] || backup_die "WORKS_DRIVE_OAUTH_CLIENT_ID is required for service-account token mode."
|
||||
[[ -n "$service_account" ]] || backup_die "WORKS_DRIVE_OAUTH_CLIENT_SERVICE_ACCOUNT is required for service-account token mode."
|
||||
|
||||
if [[ -n "$private_key" ]]; then
|
||||
temp_key_file="$(mktemp /tmp/baron-sso-works-key.XXXXXX)"
|
||||
printf '%s\n' "$private_key" >"$temp_key_file"
|
||||
key_file="$temp_key_file"
|
||||
elif [[ -n "$private_key_file" ]]; then
|
||||
if [[ "$private_key_file" != /* ]]; then
|
||||
private_key_file="$repo_root/$private_key_file"
|
||||
fi
|
||||
backup_require_path "$private_key_file" || return 1
|
||||
key_file="$private_key_file"
|
||||
else
|
||||
backup_die "WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY or WORKS_DRIVE_OAUTH_CLIENT_PRIVATE_KEY_FILE is required for service-account token mode."
|
||||
fi
|
||||
|
||||
now="$(date +%s)"
|
||||
exp="$((now + 3600))"
|
||||
header="$(printf '{"alg":"RS256","typ":"JWT"}' | base64url)"
|
||||
payload="$(jq -cn \
|
||||
--arg iss "$client_id" \
|
||||
--arg sub "$service_account" \
|
||||
--argjson iat "$now" \
|
||||
--argjson exp "$exp" \
|
||||
'{iss:$iss, sub:$sub, iat:$iat, exp:$exp}' | base64url)"
|
||||
signing_input="${header}.${payload}"
|
||||
|
||||
printf '%s' "$signing_input" \
|
||||
| openssl dgst -sha256 -sign "$key_file" -binary \
|
||||
| base64url \
|
||||
| while IFS= read -r signature; do
|
||||
printf '%s.%s\n' "$signing_input" "$signature"
|
||||
done
|
||||
|
||||
if [[ -n "$temp_key_file" ]]; then
|
||||
rm -f "$temp_key_file"
|
||||
fi
|
||||
}
|
||||
|
||||
request_service_account_token() {
|
||||
local client_id="${WORKS_DRIVE_OAUTH_CLIENT_ID:-}"
|
||||
local client_secret="${WORKS_DRIVE_OAUTH_CLIENT_SECRET:-}"
|
||||
local token_url="${WORKS_ADMIN_OAUTH_TOKEN_URL:-https://auth.worksmobile.com/oauth2/v2.0/token}"
|
||||
local assertion
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
[[ -n "$client_secret" ]] || backup_die "WORKS_DRIVE_OAUTH_CLIENT_SECRET is required for service-account token mode."
|
||||
assertion="$(build_jwt_assertion)"
|
||||
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X POST \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
--data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" \
|
||||
--data-urlencode "assertion=$assertion" \
|
||||
--data-urlencode "client_id=$client_id" \
|
||||
--data-urlencode "client_secret=$client_secret" \
|
||||
--data-urlencode "scope=$upload_scope" \
|
||||
"$token_url")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS token request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
jq -er '.access_token' <<<"$response_body"
|
||||
}
|
||||
|
||||
request_refresh_access_token() {
|
||||
local client_id="${WORKS_DRIVE_OAUTH_CLIENT_ID:-}"
|
||||
local client_secret="${WORKS_DRIVE_OAUTH_CLIENT_SECRET:-}"
|
||||
local refresh_token="${WORKS_DRIVE_OAUTH_REFRESH_TOKEN:-}"
|
||||
local token_url="${WORKS_ADMIN_OAUTH_TOKEN_URL:-https://auth.worksmobile.com/oauth2/v2.0/token}"
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
[[ -n "$client_id" ]] || backup_die "WORKS_DRIVE_OAUTH_CLIENT_ID is required for refresh-token mode."
|
||||
[[ -n "$client_secret" ]] || backup_die "WORKS_DRIVE_OAUTH_CLIENT_SECRET is required for refresh-token mode."
|
||||
[[ -n "$refresh_token" ]] || backup_die "WORKS_DRIVE_OAUTH_REFRESH_TOKEN is required for refresh-token mode."
|
||||
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X POST \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
--data-urlencode "grant_type=refresh_token" \
|
||||
--data-urlencode "refresh_token=$refresh_token" \
|
||||
--data-urlencode "client_id=$client_id" \
|
||||
--data-urlencode "client_secret=$client_secret" \
|
||||
"$token_url")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS refresh token request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
jq -er '.access_token' <<<"$response_body"
|
||||
}
|
||||
|
||||
resolve_access_token() {
|
||||
if [[ -n "${WORKS_DRIVE_ACCESS_TOKEN:-}" ]]; then
|
||||
printf '%s\n' "$WORKS_DRIVE_ACCESS_TOKEN"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${WORKS_DRIVE_ACCESS_TOKEN_FILE:-}" ]]; then
|
||||
backup_require_path "$WORKS_DRIVE_ACCESS_TOKEN_FILE"
|
||||
sed -n '1p' "$WORKS_DRIVE_ACCESS_TOKEN_FILE"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${WORKS_DRIVE_ACCESS_TOKEN_CMD:-}" ]]; then
|
||||
sh -c "$WORKS_DRIVE_ACCESS_TOKEN_CMD"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${WORKS_DRIVE_OAUTH_REFRESH_TOKEN:-}" ]]; then
|
||||
request_refresh_access_token
|
||||
return
|
||||
fi
|
||||
|
||||
request_service_account_token
|
||||
}
|
||||
|
||||
package_backup_path() {
|
||||
local source_path="$1"
|
||||
local source_name
|
||||
local target_base
|
||||
|
||||
if [[ -f "$source_path" ]]; then
|
||||
[[ "$source_path" == *.zst ]] || backup_die "file BACKUP must be a .zst archive. Pass a backup directory to package it as .tar.zst automatically."
|
||||
printf '%s\n' "$source_path"
|
||||
return
|
||||
fi
|
||||
|
||||
source_name="$(basename "$source_path")"
|
||||
target_base="$archive_dir/${source_name}"
|
||||
|
||||
backup_require_command tar
|
||||
backup_require_command zstd
|
||||
tar --zstd -cf "${target_base}.tar.zst" -C "$(dirname "$source_path")" "$source_name"
|
||||
printf '%s\n' "${target_base}.tar.zst"
|
||||
}
|
||||
|
||||
build_upload_file_list() {
|
||||
local package_file="$1"
|
||||
local file_size
|
||||
local split_bytes
|
||||
local split_dir
|
||||
local split_prefix
|
||||
|
||||
backup_require_command stat
|
||||
backup_require_command split
|
||||
file_size="$(stat -c '%s' "$package_file")"
|
||||
split_bytes="$(bytes_from_size "$split_size")"
|
||||
|
||||
if [[ "$force_split" == "true" || ( "$max_single_file_bytes" != "0" && "$file_size" -gt "$max_single_file_bytes" ) ]]; then
|
||||
split_dir="$archive_dir/split-$(basename "$package_file")"
|
||||
rm -rf "$split_dir"
|
||||
mkdir -p "$split_dir"
|
||||
split_prefix="$split_dir/$(basename "$package_file").part-"
|
||||
split -b "$split_bytes" -d -a 4 "$package_file" "$split_prefix"
|
||||
find "$split_dir" -maxdepth 1 -type f -name "$(basename "$split_prefix")*" | sort
|
||||
return
|
||||
fi
|
||||
|
||||
printf '%s\n' "$package_file"
|
||||
}
|
||||
|
||||
create_upload_url() {
|
||||
local access_token="$1"
|
||||
local endpoint="$2"
|
||||
local file_path="$3"
|
||||
local file_name
|
||||
local file_size
|
||||
local payload
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
file_name="$(basename "$file_path")"
|
||||
file_size="$(stat -c '%s' "$file_path")"
|
||||
payload="$(jq -cn \
|
||||
--arg fileName "$file_name" \
|
||||
--argjson fileSize "$file_size" \
|
||||
--argjson overwrite "$overwrite" \
|
||||
'{fileName:$fileName, fileSize:$fileSize, overwrite:$overwrite}')"
|
||||
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $access_token" \
|
||||
-H "Content-Type: application/json; charset=UTF-8" \
|
||||
-d "$payload" \
|
||||
"$endpoint")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS upload URL request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
jq -er '.uploadUrl' <<<"$response_body"
|
||||
}
|
||||
|
||||
upload_file_to_url() {
|
||||
local access_token="$1"
|
||||
local upload_url="$2"
|
||||
local file_path="$3"
|
||||
local file_name
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
file_name="$(basename "$file_path")"
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $access_token" \
|
||||
-F "Filedata=@${file_path};filename=${file_name};type=application/octet-stream" \
|
||||
"$upload_url")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS file upload failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
printf '%s\n' "$response_body"
|
||||
}
|
||||
|
||||
list_child_folders() {
|
||||
local access_token="$1"
|
||||
local endpoint="$2"
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X GET \
|
||||
-H "Authorization: Bearer $access_token" \
|
||||
"$endpoint")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS folder list request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
printf '%s\n' "$response_body"
|
||||
}
|
||||
|
||||
create_child_folder() {
|
||||
local access_token="$1"
|
||||
local endpoint="$2"
|
||||
local folder_name="$3"
|
||||
local payload
|
||||
local response
|
||||
local response_body
|
||||
local http_status
|
||||
|
||||
payload="$(jq -cn --arg fileName "$folder_name" '{fileName:$fileName}')"
|
||||
response="$("$curl_bin" -sS -w $'\n%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $access_token" \
|
||||
-H "Content-Type: application/json; charset=UTF-8" \
|
||||
-d "$payload" \
|
||||
"$endpoint")"
|
||||
split_curl_response "$response" response_body http_status
|
||||
|
||||
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
|
||||
backup_die "WORKS folder create request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)"
|
||||
fi
|
||||
|
||||
jq -er '.fileId // .id' <<<"$response_body"
|
||||
}
|
||||
|
||||
resolve_report_folder_id() {
|
||||
local access_token="$1"
|
||||
local children_endpoint
|
||||
local create_folder_endpoint
|
||||
local children_json
|
||||
local folder_id
|
||||
|
||||
children_endpoint="$(resolve_target_children_endpoint)"
|
||||
create_folder_endpoint="$(resolve_target_create_folder_endpoint)"
|
||||
children_json="$(list_child_folders "$access_token" "$children_endpoint")"
|
||||
folder_id="$(jq -er --arg name "$report_folder_name" '
|
||||
[
|
||||
(.files // .children // .items // [])[]
|
||||
| select((.fileName // .name) == $name)
|
||||
| select(((.fileType // .type // "") | ascii_downcase) == "folder")
|
||||
| .fileId // .id
|
||||
][0] // empty
|
||||
' <<<"$children_json" 2>/dev/null || true)"
|
||||
|
||||
if [[ -n "$folder_id" ]]; then
|
||||
printf '%s\n' "$folder_id"
|
||||
return
|
||||
fi
|
||||
|
||||
create_child_folder "$access_token" "$create_folder_endpoint" "$report_folder_name"
|
||||
}
|
||||
|
||||
discover_markdown_reports() {
|
||||
if [[ -f "$backup_path" || ! -d "$report_dir" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
find "$report_dir" -maxdepth 1 -type f -name '*.md' | sort
|
||||
}
|
||||
|
||||
timestamp_report_file_for_upload() {
|
||||
local source_file="$1"
|
||||
local file_name
|
||||
local base_name
|
||||
local stamped_dir
|
||||
local stamped_file
|
||||
|
||||
file_name="$(basename "$source_file")"
|
||||
base_name="${file_name%.md}"
|
||||
stamped_dir="$archive_dir/reports"
|
||||
mkdir -p "$stamped_dir"
|
||||
stamped_file="$stamped_dir/${base_name}-${report_upload_timestamp}.md"
|
||||
cp "$source_file" "$stamped_file"
|
||||
printf '%s\n' "$stamped_file"
|
||||
}
|
||||
|
||||
write_upload_report() {
|
||||
local report_file="$1"
|
||||
local uploaded_json="$2"
|
||||
local uploaded_reports_json="${3:-[]}"
|
||||
|
||||
jq -n \
|
||||
--arg createdAt "$(backup_utc_now)" \
|
||||
--arg backup "$backup_path" \
|
||||
--arg target "$target" \
|
||||
--arg endpoint "$upload_endpoint" \
|
||||
--argjson files "$uploaded_json" \
|
||||
--argjson reportFiles "$uploaded_reports_json" \
|
||||
'{
|
||||
created_at: $createdAt,
|
||||
backup: $backup,
|
||||
target: $target,
|
||||
upload_endpoint: $endpoint,
|
||||
files: $files,
|
||||
report_files: $reportFiles
|
||||
}' >"$report_file"
|
||||
}
|
||||
|
||||
upload_endpoint="$(resolve_target_upload_endpoint)"
|
||||
report_upload_timestamp="$(backup_timestamp)"
|
||||
|
||||
if [[ "$dry_run" == "true" ]]; then
|
||||
backup_log "Dry run: would upload BACKUP=$backup_path to WORKS Drive target=$target endpoint=$upload_endpoint"
|
||||
if [[ "$upload_reports" == "true" ]]; then
|
||||
backup_log "Dry run: would upload markdown reports from $report_dir to WORKS Drive folder=$report_folder_name"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
backup_require_command "$curl_bin"
|
||||
|
||||
package_file="$(package_backup_path "$backup_path")"
|
||||
mapfile -t upload_files < <(build_upload_file_list "$package_file")
|
||||
access_token="$(resolve_access_token)"
|
||||
|
||||
uploaded_items="[]"
|
||||
for file_path in "${upload_files[@]}"; do
|
||||
backup_require_path "$file_path"
|
||||
backup_log "Creating WORKS Drive upload URL for $(basename "$file_path")"
|
||||
upload_url="$(create_upload_url "$access_token" "$upload_endpoint" "$file_path")"
|
||||
backup_log "Uploading $(basename "$file_path") to WORKS Drive"
|
||||
upload_response="$(upload_file_to_url "$access_token" "$upload_url" "$file_path")"
|
||||
upload_response_json="$(jq -c '.' <<<"${upload_response:-{}}" 2>/dev/null || printf '{}')"
|
||||
uploaded_items="$(jq \
|
||||
--arg fileName "$(basename "$file_path")" \
|
||||
--arg filePath "$file_path" \
|
||||
--argjson fileSize "$(stat -c '%s' "$file_path")" \
|
||||
--arg status "uploaded" \
|
||||
--arg response "$upload_response_json" \
|
||||
'. + [{file_name:$fileName, file_path:$filePath, file_size:$fileSize, status:$status, response:($response | fromjson? // {})}]' \
|
||||
<<<"$uploaded_items")"
|
||||
done
|
||||
|
||||
uploaded_report_items="[]"
|
||||
if [[ "$upload_reports" == "true" ]]; then
|
||||
mapfile -t markdown_reports < <(discover_markdown_reports)
|
||||
if [[ "${#markdown_reports[@]}" -gt 0 ]]; then
|
||||
backup_log "Resolving WORKS Drive report folder: $report_folder_name"
|
||||
report_folder_id="$(resolve_report_folder_id "$access_token")"
|
||||
report_upload_endpoint="$(resolve_target_upload_endpoint "$report_folder_id")"
|
||||
|
||||
for report_path_item in "${markdown_reports[@]}"; do
|
||||
backup_require_path "$report_path_item"
|
||||
stamped_report_path="$(timestamp_report_file_for_upload "$report_path_item")"
|
||||
backup_log "Creating WORKS Drive upload URL for $(basename "$stamped_report_path")"
|
||||
upload_url="$(create_upload_url "$access_token" "$report_upload_endpoint" "$stamped_report_path")"
|
||||
backup_log "Uploading $(basename "$stamped_report_path") to WORKS Drive reports folder"
|
||||
upload_response="$(upload_file_to_url "$access_token" "$upload_url" "$stamped_report_path")"
|
||||
upload_response_json="$(jq -c '.' <<<"${upload_response:-{}}" 2>/dev/null || printf '{}')"
|
||||
uploaded_report_items="$(jq \
|
||||
--arg fileName "$(basename "$stamped_report_path")" \
|
||||
--arg sourcePath "$report_path_item" \
|
||||
--arg filePath "$stamped_report_path" \
|
||||
--argjson fileSize "$(stat -c '%s' "$stamped_report_path")" \
|
||||
--arg status "uploaded" \
|
||||
--arg response "$upload_response_json" \
|
||||
'. + [{file_name:$fileName, source_path:$sourcePath, file_path:$filePath, file_size:$fileSize, status:$status, response:($response | fromjson? // {})}]' \
|
||||
<<<"$uploaded_report_items")"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
report_file="$report_dir/cloud-upload.json"
|
||||
write_upload_report "$report_file" "$uploaded_items" "$uploaded_report_items"
|
||||
|
||||
backup_log "Upload complete: $report_file"
|
||||
16
scripts/backup/verify-dump.sh
Executable file
16
scripts/backup/verify-dump.sh
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
|
||||
backup_dir="${BACKUP:-${1:-}}"
|
||||
[[ -n "$backup_dir" ]] || backup_die "BACKUP is required."
|
||||
backup_require_path "$backup_dir"
|
||||
backup_require_path "$backup_dir/manifest.json"
|
||||
|
||||
if ! backup_verify_checksums "$backup_dir"; then
|
||||
backup_die "checksum verification failed"
|
||||
fi
|
||||
|
||||
backup_log "Dump verification passed: $backup_dir"
|
||||
13
scripts/backup/verify-restore.sh
Executable file
13
scripts/backup/verify-restore.sh
Executable file
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
|
||||
backup_dir="${BACKUP:-${1:-}}"
|
||||
[[ -n "$backup_dir" ]] || backup_die "BACKUP is required."
|
||||
|
||||
BACKUP="$backup_dir" "$script_dir/verify-dump.sh"
|
||||
|
||||
backup_log "Restore verification policy check passed."
|
||||
backup_log "Run application smoke checks separately: super admin login, representative OIDC login, and WORKS comparison dry-run."
|
||||
Reference in New Issue
Block a user