forked from baron/baron-sso
Add personnel dataset backup filtering
This commit is contained in:
@@ -8,15 +8,17 @@ source "$script_dir/lib/postgres.sh"
|
||||
source "$script_dir/lib/clickhouse.sh"
|
||||
source "$script_dir/lib/config.sh"
|
||||
source "$script_dir/lib/report.sh"
|
||||
source "$script_dir/lib/personnel_dataset.sh"
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
services="$(normalize_service_filter "${DUMP_SERVICES:-all}")"
|
||||
dataset="$(normalize_dataset_profile "${DUMP_DATASET:-full}")"
|
||||
mode="${DUMP_MODE:-maintenance}"
|
||||
backup_root="${BACKUP_ROOT:-$repo_root/backups}"
|
||||
backup_dir="${BACKUP:-$backup_root/baron-sso-backup-$(backup_timestamp)}"
|
||||
|
||||
mkdir -p "$backup_dir/reports"
|
||||
create_manifest "$backup_dir" "$mode" "$services"
|
||||
create_manifest "$backup_dir" "$mode" "$services" "$dataset"
|
||||
service_timings_json="[]"
|
||||
|
||||
run_backup_step() {
|
||||
@@ -40,26 +42,31 @@ run_backup_step() {
|
||||
|
||||
backup_log "Creating backup at $backup_dir"
|
||||
backup_log "Backup mode: $mode"
|
||||
backup_log "Dataset: $dataset"
|
||||
backup_log "Services: $services"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
run_backup_step postgres dump_baron_postgres "$backup_dir"
|
||||
fi
|
||||
if [[ "$dataset" == "personnel" ]]; then
|
||||
run_backup_step personnel dump_personnel_dataset "$backup_dir" "$services"
|
||||
else
|
||||
if service_enabled postgres "$services"; then
|
||||
run_backup_step postgres dump_baron_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
run_backup_step ory-postgres dump_ory_postgres "$backup_dir"
|
||||
fi
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
run_backup_step ory-postgres dump_ory_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
run_backup_step clickhouse dump_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
if service_enabled clickhouse "$services"; then
|
||||
run_backup_step clickhouse dump_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
run_backup_step ory-clickhouse dump_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
run_backup_step ory-clickhouse dump_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
run_backup_step config dump_config_snapshot "$backup_dir"
|
||||
if service_enabled config "$services"; then
|
||||
run_backup_step config dump_config_snapshot "$backup_dir"
|
||||
fi
|
||||
fi
|
||||
|
||||
write_backup_markdown_report "$backup_dir" "succeeded" "$services" "$service_timings_json"
|
||||
|
||||
106
scripts/backup/filter_personnel_dump.sh
Executable file
106
scripts/backup/filter_personnel_dump.sh
Executable file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$script_dir/lib/common.sh"
|
||||
source "$script_dir/lib/manifest.sh"
|
||||
source "$script_dir/lib/postgres.sh"
|
||||
source "$script_dir/lib/config.sh"
|
||||
source "$script_dir/lib/report.sh"
|
||||
source "$script_dir/lib/personnel_dataset.sh"
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
source_backup="${BACKUP:-${RESTORE_INPUT:-${FILE_PATH:-}}}"
|
||||
services="$(normalize_service_filter "${FILTER_SERVICES:-postgres,ory-postgres}")"
|
||||
output_backup="${OUTPUT_BACKUP:-$repo_root/backups/baron-sso-personnel-filtered-$(backup_timestamp)}"
|
||||
scratch_suffix="$(date -u '+%Y%m%d%H%M%S')_$$"
|
||||
scratch_baron="baron_personnel_filter_${scratch_suffix}"
|
||||
scratch_kratos="ory_kratos_personnel_filter_${scratch_suffix}"
|
||||
scratch_keto="ory_keto_personnel_filter_${scratch_suffix}"
|
||||
|
||||
cleanup_filter_scratch() {
|
||||
if [[ -n "${scratch_baron:-}" ]]; then
|
||||
docker exec -e "PGPASSWORD=${DB_PASSWORD:-password}" baron_postgres \
|
||||
psql -U "${DB_USER:-baron}" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "drop database if exists ${scratch_baron} with (force)" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n "${scratch_kratos:-}" || -n "${scratch_keto:-}" ]]; then
|
||||
docker exec -e "PGPASSWORD=${ORY_POSTGRES_PASSWORD:-secret}" ory_postgres \
|
||||
psql -U "${ORY_POSTGRES_USER:-ory}" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "drop database if exists ${scratch_kratos} with (force)" \
|
||||
-c "drop database if exists ${scratch_keto} with (force)" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup_filter_scratch EXIT
|
||||
|
||||
quote_pg_database() {
|
||||
local raw="$1"
|
||||
printf '"%s"' "${raw//\"/\"\"}"
|
||||
}
|
||||
|
||||
create_scratch_database() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local database_ident
|
||||
|
||||
database_ident="$(quote_pg_database "$database")"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "drop database if exists ${database_ident} with (force)" \
|
||||
-c "create database ${database_ident}"
|
||||
}
|
||||
|
||||
restore_custom_dump_to_scratch() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local dump_path="$5"
|
||||
|
||||
backup_require_path "$dump_path"
|
||||
docker exec -i -e "PGPASSWORD=$password" "$container" \
|
||||
pg_restore -U "$user" -d "$database" --clean --if-exists <"$dump_path"
|
||||
}
|
||||
|
||||
[[ -n "$source_backup" ]] || backup_die "BACKUP is required. Example: make filter-personnel-dump BACKUP=backups/full OUTPUT_BACKUP=backups/personnel"
|
||||
backup_require_path "$source_backup/manifest.json"
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
backup_require_container ory_postgres
|
||||
|
||||
if [[ "$(jq -r '.dataset // "full"' "$source_backup/manifest.json")" == "personnel" ]]; then
|
||||
backup_die "source BACKUP is already a personnel dataset: $source_backup"
|
||||
fi
|
||||
|
||||
backup_log "Filtering personnel dataset from full backup: $source_backup"
|
||||
backup_log "Output backup: $output_backup"
|
||||
backup_log "Services: $services"
|
||||
|
||||
mkdir -p "$output_backup/reports"
|
||||
create_manifest "$output_backup" "filtered-from-full" "$services" "personnel"
|
||||
write_personnel_dataset_manifest "$output_backup" "$services"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
backup_log "Restoring Baron full dump to scratch DB: $scratch_baron"
|
||||
create_scratch_database baron_postgres "${DB_USER:-baron}" "${DB_PASSWORD:-password}" "$scratch_baron"
|
||||
restore_custom_dump_to_scratch baron_postgres "${DB_USER:-baron}" "${DB_PASSWORD:-password}" "$scratch_baron" "$source_backup/postgres/baron.dump"
|
||||
DB_NAME="$scratch_baron" dump_personnel_baron_postgres "$output_backup"
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
backup_log "Restoring Kratos/Keto full dumps to scratch DBs: $scratch_kratos, $scratch_keto"
|
||||
create_scratch_database ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$scratch_kratos"
|
||||
create_scratch_database ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$scratch_keto"
|
||||
restore_custom_dump_to_scratch ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$scratch_kratos" "$source_backup/postgres/${KRATOS_DB:-ory_kratos}.dump"
|
||||
restore_custom_dump_to_scratch ory_postgres "${ORY_POSTGRES_USER:-ory}" "${ORY_POSTGRES_PASSWORD:-secret}" "$scratch_keto" "$source_backup/postgres/${KETO_DB:-ory_keto}.dump"
|
||||
KRATOS_DB="$scratch_kratos" KETO_DB="$scratch_keto" dump_personnel_ory_postgres "$output_backup"
|
||||
fi
|
||||
|
||||
write_backup_markdown_report "$output_backup" "succeeded" "$services" "[]"
|
||||
backup_checksum_file "$output_backup"
|
||||
BACKUP="$output_backup" "$script_dir/verify-dump.sh"
|
||||
|
||||
backup_log "Personnel filtered backup complete: $output_backup"
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
BACKUP_SUPPORTED_SERVICES="postgres ory-postgres clickhouse ory-clickhouse config"
|
||||
BACKUP_SUPPORTED_DATASETS="full personnel"
|
||||
|
||||
backup_repo_root() {
|
||||
if [[ -n "${BACKUP_REPO_ROOT:-}" ]]; then
|
||||
@@ -65,6 +66,18 @@ normalize_service_filter() {
|
||||
printf '%s\n' "$normalized"
|
||||
}
|
||||
|
||||
normalize_dataset_profile() {
|
||||
local raw="${1:-full}"
|
||||
|
||||
[[ -n "$raw" ]] || raw="full"
|
||||
if ! grep -qw -- "$raw" <<<"$BACKUP_SUPPORTED_DATASETS"; then
|
||||
backup_die "unknown backup dataset: $raw"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$raw"
|
||||
}
|
||||
|
||||
service_enabled() {
|
||||
local service="$1"
|
||||
local services="$2"
|
||||
|
||||
@@ -4,11 +4,17 @@ create_manifest() {
|
||||
local backup_dir="$1"
|
||||
local mode="$2"
|
||||
local services="$3"
|
||||
local dataset="${4:-full}"
|
||||
local repo_root
|
||||
local created_at
|
||||
local git_commit
|
||||
local service
|
||||
local first=1
|
||||
local environment_scope="same-env-only"
|
||||
|
||||
if [[ "$dataset" == "personnel" ]]; then
|
||||
environment_scope="staging-rehearsal"
|
||||
fi
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
created_at="$(backup_utc_now)"
|
||||
@@ -20,7 +26,8 @@ create_manifest() {
|
||||
printf ' "created_at": "%s",\n' "$created_at"
|
||||
printf ' "git_commit": "%s",\n' "$git_commit"
|
||||
printf ' "mode": "%s",\n' "$mode"
|
||||
printf ' "environment_scope": "same-env-only",\n'
|
||||
printf ' "dataset": "%s",\n' "$dataset"
|
||||
printf ' "environment_scope": "%s",\n' "$environment_scope"
|
||||
printf ' "services": ['
|
||||
for service in $services; do
|
||||
if [[ "$first" -eq 1 ]]; then
|
||||
|
||||
367
scripts/backup/lib/personnel_dataset.sh
Normal file
367
scripts/backup/lib/personnel_dataset.sh
Normal file
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
personnel_dataset_dir() {
|
||||
local backup_dir="$1"
|
||||
printf '%s\n' "$backup_dir/datasets/personnel"
|
||||
}
|
||||
|
||||
write_personnel_dataset_manifest() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
local dataset_dir
|
||||
local reports_dir
|
||||
local include_kratos
|
||||
local reset_credentials
|
||||
local include_worksmobile_mapping
|
||||
local include_outbox
|
||||
local tenant_roots
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
reports_dir="$dataset_dir/reports"
|
||||
include_kratos="${PERSONNEL_INCLUDE_KRATOS_IDENTITIES:-true}"
|
||||
reset_credentials="${PERSONNEL_RESET_CREDENTIALS:-true}"
|
||||
include_worksmobile_mapping="${PERSONNEL_INCLUDE_WORKSMOBILE_MAPPING:-true}"
|
||||
include_outbox="${PERSONNEL_INCLUDE_OUTBOX:-false}"
|
||||
tenant_roots="${PERSONNEL_TENANT_ROOT_SLUGS:-}"
|
||||
|
||||
mkdir -p "$reports_dir"
|
||||
jq -n \
|
||||
--arg dataset "personnel" \
|
||||
--arg services "$services" \
|
||||
--arg tenant_roots "$tenant_roots" \
|
||||
--arg include_kratos "$include_kratos" \
|
||||
--arg reset_credentials "$reset_credentials" \
|
||||
--arg include_worksmobile_mapping "$include_worksmobile_mapping" \
|
||||
--arg include_outbox "$include_outbox" \
|
||||
'{
|
||||
dataset: $dataset,
|
||||
format_version: "1",
|
||||
services: ($services | split(" ") | map(select(length > 0))),
|
||||
scope: {
|
||||
tenant_root_slugs: (if $tenant_roots == "" then [] else ($tenant_roots | split(",") | map(gsub("^\\s+|\\s+$"; ""))) end)
|
||||
},
|
||||
included: {
|
||||
baron_postgres_tables: [
|
||||
"public.tenants",
|
||||
"public.tenant_domains",
|
||||
"public.user_groups",
|
||||
"public.users",
|
||||
"public.user_login_ids",
|
||||
"public.worksmobile_resource_mappings"
|
||||
],
|
||||
ory_kratos_tables: (if $include_kratos == "true" then ["public.identities"] else [] end),
|
||||
ory_keto_tables: ["public.keto_relation_tuples"]
|
||||
},
|
||||
excluded: {
|
||||
databases: ["ory_hydra"],
|
||||
tables: [
|
||||
"public.relying_parties",
|
||||
"public.rp_user_metadata",
|
||||
"public.client_consents",
|
||||
"public.client_secrets",
|
||||
"public.api_keys",
|
||||
"public.worksmobile_outboxes"
|
||||
],
|
||||
volatile: ["redis", "sessions", "oauth2_tokens", "audit_logs"]
|
||||
},
|
||||
restore_policy: {
|
||||
reset_credentials: ($reset_credentials == "true"),
|
||||
include_worksmobile_mapping: ($include_worksmobile_mapping == "true"),
|
||||
include_outbox: ($include_outbox == "true"),
|
||||
hydra_restore: false,
|
||||
rp_metadata_restore: false,
|
||||
default_mode: "replace-dataset-rows"
|
||||
}
|
||||
}' >"$dataset_dir/dataset-manifest.json"
|
||||
}
|
||||
|
||||
personnel_psql_jsonl() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local sql="$5"
|
||||
local output_file="$6"
|
||||
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 -c "$sql" \
|
||||
>"$output_file"
|
||||
}
|
||||
|
||||
personnel_write_count() {
|
||||
local report_file="$1"
|
||||
local key="$2"
|
||||
local file_path="$3"
|
||||
local count
|
||||
|
||||
if [[ -f "$file_path" ]]; then
|
||||
count="$(wc -l <"$file_path" | tr -d '[:space:]')"
|
||||
else
|
||||
count="0"
|
||||
fi
|
||||
printf '%s:%s\n' "$key" "$count" >>"$report_file"
|
||||
}
|
||||
|
||||
dump_personnel_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
local dataset_dir
|
||||
local output_dir
|
||||
local report_file
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
output_dir="$dataset_dir/postgres"
|
||||
report_file="$dataset_dir/reports/row-counts.txt"
|
||||
mkdir -p "$output_dir" "$dataset_dir/reports"
|
||||
: >"$report_file"
|
||||
|
||||
backup_log "Dumping personnel Baron Postgres dataset: $db_name"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.tenants t order by t.id" \
|
||||
"$output_dir/tenants.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.tenant_domains t order by t.id" \
|
||||
"$output_dir/tenant_domains.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.user_groups t order by t.id" \
|
||||
"$output_dir/user_groups.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select (to_jsonb(t) - 'password_hash' - 'relying_party_id' || jsonb_build_object('relying_party_id', null))::text from public.users t order by t.id" \
|
||||
"$output_dir/users.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.user_login_ids t order by t.id" \
|
||||
"$output_dir/user_login_ids.jsonl"
|
||||
|
||||
if [[ "${PERSONNEL_INCLUDE_WORKSMOBILE_MAPPING:-true}" == "true" ]]; then
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.worksmobile_resource_mappings t where t.baron_resource_type in ('USER', 'ORGUNIT') order by t.id" \
|
||||
"$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
else
|
||||
: >"$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
fi
|
||||
|
||||
personnel_write_count "$report_file" "public.tenants" "$output_dir/tenants.jsonl"
|
||||
personnel_write_count "$report_file" "public.tenant_domains" "$output_dir/tenant_domains.jsonl"
|
||||
personnel_write_count "$report_file" "public.user_groups" "$output_dir/user_groups.jsonl"
|
||||
personnel_write_count "$report_file" "public.users" "$output_dir/users.jsonl"
|
||||
personnel_write_count "$report_file" "public.user_login_ids" "$output_dir/user_login_ids.jsonl"
|
||||
personnel_write_count "$report_file" "public.worksmobile_resource_mappings" "$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
}
|
||||
|
||||
dump_personnel_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local dataset_dir
|
||||
local report_file
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
report_file="$dataset_dir/reports/row-counts.txt"
|
||||
mkdir -p "$dataset_dir/ory_kratos" "$dataset_dir/ory_keto" "$dataset_dir/reports"
|
||||
|
||||
if [[ "${PERSONNEL_INCLUDE_KRATOS_IDENTITIES:-true}" == "true" ]]; then
|
||||
backup_log "Dumping personnel Kratos identity subset: $kratos_db"
|
||||
personnel_psql_jsonl ory_postgres "$db_user" "$db_password" "$kratos_db" \
|
||||
"select (to_jsonb(i) - 'metadata_admin')::text from public.identities i order by i.id" \
|
||||
"$dataset_dir/ory_kratos/identities.jsonl"
|
||||
else
|
||||
: >"$dataset_dir/ory_kratos/identities.jsonl"
|
||||
fi
|
||||
jq -n '{policy:"reset_credentials", credentials:[]}' >"$dataset_dir/ory_kratos/identity_credentials.reset-plan.jsonl"
|
||||
|
||||
backup_log "Dumping personnel Keto relation tuple subset: $keto_db"
|
||||
personnel_psql_jsonl ory_postgres "$db_user" "$db_password" "$keto_db" \
|
||||
"select to_jsonb(t)::text from public.keto_relation_tuples t where t.namespace <> 'RelyingParty' and coalesce(t.subject_set_namespace, '') <> 'RelyingParty' order by t.namespace, t.object, t.relation, t.subject_id" \
|
||||
"$dataset_dir/ory_keto/relation_tuples.jsonl"
|
||||
|
||||
personnel_write_count "$report_file" "public.identities" "$dataset_dir/ory_kratos/identities.jsonl"
|
||||
personnel_write_count "$report_file" "public.keto_relation_tuples" "$dataset_dir/ory_keto/relation_tuples.jsonl"
|
||||
}
|
||||
|
||||
dump_personnel_dataset() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
|
||||
write_personnel_dataset_manifest "$backup_dir" "$services"
|
||||
if service_enabled postgres "$services"; then
|
||||
dump_personnel_baron_postgres "$backup_dir"
|
||||
fi
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
dump_personnel_ory_postgres "$backup_dir"
|
||||
fi
|
||||
if service_enabled config "$services"; then
|
||||
dump_config_snapshot "$backup_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
personnel_dataset_manifest_path() {
|
||||
local backup_dir="$1"
|
||||
printf '%s\n' "$backup_dir/datasets/personnel/dataset-manifest.json"
|
||||
}
|
||||
|
||||
restore_personnel_plan_policy_json() {
|
||||
local backup_dir="$1"
|
||||
local manifest_path
|
||||
|
||||
manifest_path="$(personnel_dataset_manifest_path "$backup_dir")"
|
||||
if [[ -f "$manifest_path" ]]; then
|
||||
jq -c '{dataset, included, excluded, restore_policy, scope}' "$manifest_path"
|
||||
else
|
||||
printf '{}\n'
|
||||
fi
|
||||
}
|
||||
|
||||
personnel_restore_jsonl_id_table() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local table="$5"
|
||||
local input_file="$6"
|
||||
local scratch
|
||||
local columns
|
||||
local updates
|
||||
|
||||
backup_require_path "$input_file"
|
||||
if [[ ! -s "$input_file" ]]; then
|
||||
backup_log "Skipping empty personnel dataset table: public.$table"
|
||||
return 0
|
||||
fi
|
||||
|
||||
scratch="_personnel_restore_${table}_json"
|
||||
columns="$(docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 \
|
||||
-c "select string_agg(quote_ident(column_name), ', ' order by ordinal_position) from information_schema.columns where table_schema = 'public' and table_name = '$table'")"
|
||||
updates="$(docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 \
|
||||
-c "select string_agg(format('%I = excluded.%I', column_name, column_name), ', ' order by ordinal_position) from information_schema.columns where table_schema = 'public' and table_name = '$table' and column_name <> 'id'")"
|
||||
|
||||
[[ -n "$columns" ]] || backup_die "cannot resolve columns for personnel restore table: public.$table"
|
||||
[[ -n "$updates" ]] || backup_die "cannot resolve update columns for personnel restore table: public.$table"
|
||||
|
||||
backup_log "Restoring personnel dataset table: public.$table"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "drop table if exists public.${scratch}" \
|
||||
-c "create table public.${scratch} (line jsonb not null)"
|
||||
docker exec -i -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "\\copy public.${scratch}(line) from stdin" <"$input_file"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "insert into public.${table} (${columns}) select ${columns} from (select (jsonb_populate_record(null::public.${table}, line)).* from public.${scratch}) r on conflict (id) do update set ${updates}" \
|
||||
-c "drop table public.${scratch}"
|
||||
}
|
||||
|
||||
restore_personnel_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
local input_dir
|
||||
|
||||
input_dir="$(personnel_dataset_dir "$backup_dir")/postgres"
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" tenants "$input_dir/tenants.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" tenant_domains "$input_dir/tenant_domains.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" user_groups "$input_dir/user_groups.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" users "$input_dir/users.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" user_login_ids "$input_dir/user_login_ids.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" worksmobile_resource_mappings "$input_dir/worksmobile_resource_mappings.jsonl"
|
||||
}
|
||||
|
||||
restore_personnel_keto_relation_tuples() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local input_file
|
||||
local scratch="_personnel_restore_keto_relation_tuples_json"
|
||||
|
||||
input_file="$(personnel_dataset_dir "$backup_dir")/ory_keto/relation_tuples.jsonl"
|
||||
backup_require_path "$input_file"
|
||||
if [[ ! -s "$input_file" ]]; then
|
||||
backup_log "Skipping empty personnel Keto relation tuple dataset"
|
||||
return 0
|
||||
fi
|
||||
|
||||
backup_log "Restoring personnel Keto relation tuple subset: $keto_db"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "drop table if exists public.${scratch}" \
|
||||
-c "create table public.${scratch} (line jsonb not null)"
|
||||
docker exec -i -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "\\copy public.${scratch}(line) from stdin" <"$input_file"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "delete from public.keto_relation_tuples where namespace <> 'RelyingParty' and coalesce(subject_set_namespace, '') <> 'RelyingParty'" \
|
||||
-c "insert into public.keto_relation_tuples select (jsonb_populate_record(null::public.keto_relation_tuples, line)).* from public.${scratch}" \
|
||||
-c "drop table public.${scratch}"
|
||||
}
|
||||
|
||||
restore_personnel_kratos_identities() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local input_file
|
||||
|
||||
input_file="$(personnel_dataset_dir "$backup_dir")/ory_kratos/identities.jsonl"
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
personnel_restore_jsonl_id_table ory_postgres "$db_user" "$db_password" "$kratos_db" identities "$input_file"
|
||||
}
|
||||
|
||||
restore_personnel_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
|
||||
restore_personnel_kratos_identities "$backup_dir"
|
||||
restore_personnel_keto_relation_tuples "$backup_dir"
|
||||
}
|
||||
|
||||
restore_personnel_dataset() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
local report_items=()
|
||||
local item
|
||||
|
||||
backup_require_path "$(personnel_dataset_manifest_path "$backup_dir")"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
restore_personnel_baron_postgres "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/postgres" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
restore_personnel_ory_postgres "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/ory-postgres" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
restore_config_snapshot "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/config" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if [[ "${#report_items[@]}" -eq 0 ]]; then
|
||||
target_verification_reports="[]"
|
||||
else
|
||||
target_verification_reports="$(printf '%s\n' "${report_items[@]}" | jq -s '.')"
|
||||
fi
|
||||
target_verification_status="passed"
|
||||
}
|
||||
@@ -130,6 +130,7 @@ write_restore_markdown_report() {
|
||||
"| 입력 유형 | \(.backup_source // "unknown") |",
|
||||
"| 백업 경로 | `\(.backup_dir // "")` |",
|
||||
"| Dump 파일 | `\(.dump_file // "")` |",
|
||||
"| Dataset | `\(.dataset // "full")` |",
|
||||
"| 서비스 | `\(services)` |",
|
||||
"",
|
||||
"## 검증",
|
||||
@@ -138,6 +139,9 @@ write_restore_markdown_report() {
|
||||
"| --- | --- |",
|
||||
"| Dump checksum | \(.verification.dump_checksum // "not_run") |",
|
||||
"| 대상 row count | \(.verification.target_row_counts // "not_run") |",
|
||||
(if (.dataset // "full") == "personnel" then
|
||||
"| Personnel exclusions | `\((.restore_policy.excluded.databases // []) | join(", "))` / `\((.restore_policy.excluded.tables // []) | join(", "))` |"
|
||||
else empty end),
|
||||
"",
|
||||
"## 대상별 검증 결과",
|
||||
"",
|
||||
|
||||
@@ -7,6 +7,7 @@ source "$script_dir/lib/postgres.sh"
|
||||
source "$script_dir/lib/clickhouse.sh"
|
||||
source "$script_dir/lib/config.sh"
|
||||
source "$script_dir/lib/report.sh"
|
||||
source "$script_dir/lib/personnel_dataset.sh"
|
||||
|
||||
dry_run=false
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
@@ -26,6 +27,7 @@ report_message=""
|
||||
dump_checksum_status="not_run"
|
||||
target_verification_status="not_run"
|
||||
target_verification_reports="[]"
|
||||
dataset="full"
|
||||
|
||||
json_array_from_words() {
|
||||
local words="$1"
|
||||
@@ -43,6 +45,7 @@ write_restore_report() {
|
||||
local finished_at
|
||||
local services_json
|
||||
local restore_policy_json="{}"
|
||||
local personnel_policy_json="{}"
|
||||
|
||||
[[ -n "$report_path" ]] || return 0
|
||||
|
||||
@@ -51,6 +54,9 @@ write_restore_report() {
|
||||
if [[ -n "${backup_dir:-}" && -f "$backup_dir/manifest.json" ]]; then
|
||||
restore_policy_json="$(jq -c '.restore_policy // {}' "$backup_dir/manifest.json")"
|
||||
fi
|
||||
if [[ "${dataset:-full}" == "personnel" ]]; then
|
||||
personnel_policy_json="$(restore_personnel_plan_policy_json "$backup_dir")"
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$report_path")"
|
||||
jq -n \
|
||||
@@ -69,6 +75,8 @@ write_restore_report() {
|
||||
--arg target_row_counts "$target_verification_status" \
|
||||
--argjson target_reports "$target_verification_reports" \
|
||||
--argjson restore_policy "$restore_policy_json" \
|
||||
--arg dataset "${dataset:-full}" \
|
||||
--argjson personnel_policy "$personnel_policy_json" \
|
||||
'{
|
||||
format_version: $format_version,
|
||||
started_at: $started_at,
|
||||
@@ -78,10 +86,11 @@ write_restore_report() {
|
||||
backup_source: $backup_source,
|
||||
backup_dir: $backup_dir,
|
||||
dump_file: (if $dump_file == "" then null else $dump_file end),
|
||||
dataset: $dataset,
|
||||
services: $services,
|
||||
allow_non_empty_restore: ($allow_non_empty_restore == "true"),
|
||||
dry_run: ($dry_run == "true"),
|
||||
restore_policy: $restore_policy,
|
||||
restore_policy: (if $dataset == "personnel" then $personnel_policy else $restore_policy end),
|
||||
verification: {
|
||||
dump_checksum: $dump_checksum,
|
||||
target_row_counts: $target_row_counts,
|
||||
@@ -439,6 +448,13 @@ if [[ "${CONFIRM_RESTORE:-}" != "baron-sso" ]]; then
|
||||
fi
|
||||
|
||||
services="$(normalize_service_filter "${RESTORE_SERVICES:-all}")"
|
||||
if [[ -n "${RESTORE_DATASET:-}" ]]; then
|
||||
dataset="$(normalize_dataset_profile "$RESTORE_DATASET")"
|
||||
elif [[ -f "$backup_dir/manifest.json" ]]; then
|
||||
dataset="$(normalize_dataset_profile "$(jq -r '.dataset // "full"' "$backup_dir/manifest.json")")"
|
||||
else
|
||||
dataset="full"
|
||||
fi
|
||||
allow_non_empty="${ALLOW_NON_EMPTY_RESTORE:-false}"
|
||||
|
||||
if [[ "${RESTORE_TEST_NON_EMPTY:-}" == "1" && "$allow_non_empty" != "true" ]]; then
|
||||
@@ -447,6 +463,7 @@ fi
|
||||
|
||||
if [[ "$dry_run" == "true" ]]; then
|
||||
backup_log "Restore plan for $backup_dir"
|
||||
backup_log "Dataset: $dataset"
|
||||
backup_log "Services: $services"
|
||||
backup_log "ALLOW_NON_EMPTY_RESTORE=$allow_non_empty"
|
||||
backup_log "RESTORE_REPORT=$report_path"
|
||||
@@ -466,27 +483,32 @@ fi
|
||||
BACKUP="$backup_dir" "$script_dir/verify-dump.sh"
|
||||
dump_checksum_status="passed"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
restore_baron_postgres "$backup_dir"
|
||||
if [[ "$dataset" == "personnel" ]]; then
|
||||
restore_personnel_dataset "$backup_dir" "$services"
|
||||
else
|
||||
if service_enabled postgres "$services"; then
|
||||
restore_baron_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
restore_ory_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
restore_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
restore_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
restore_config_snapshot "$backup_dir"
|
||||
fi
|
||||
|
||||
verify_restored_targets
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
restore_ory_postgres "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled clickhouse "$services"; then
|
||||
restore_baron_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled ory-clickhouse "$services"; then
|
||||
restore_ory_clickhouse "$backup_dir"
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
restore_config_snapshot "$backup_dir"
|
||||
fi
|
||||
|
||||
verify_restored_targets
|
||||
write_restore_report "succeeded" "restore completed and target row-count verification passed"
|
||||
|
||||
backup_log "Restore complete. Keep WORKS relay disabled until comparison dry-run passes."
|
||||
|
||||
Reference in New Issue
Block a user