forked from baron/baron-sso
Add personnel dataset backup filtering
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
BACKUP_SUPPORTED_SERVICES="postgres ory-postgres clickhouse ory-clickhouse config"
|
||||
BACKUP_SUPPORTED_DATASETS="full personnel"
|
||||
|
||||
backup_repo_root() {
|
||||
if [[ -n "${BACKUP_REPO_ROOT:-}" ]]; then
|
||||
@@ -65,6 +66,18 @@ normalize_service_filter() {
|
||||
printf '%s\n' "$normalized"
|
||||
}
|
||||
|
||||
normalize_dataset_profile() {
|
||||
local raw="${1:-full}"
|
||||
|
||||
[[ -n "$raw" ]] || raw="full"
|
||||
if ! grep -qw -- "$raw" <<<"$BACKUP_SUPPORTED_DATASETS"; then
|
||||
backup_die "unknown backup dataset: $raw"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$raw"
|
||||
}
|
||||
|
||||
service_enabled() {
|
||||
local service="$1"
|
||||
local services="$2"
|
||||
|
||||
@@ -4,11 +4,17 @@ create_manifest() {
|
||||
local backup_dir="$1"
|
||||
local mode="$2"
|
||||
local services="$3"
|
||||
local dataset="${4:-full}"
|
||||
local repo_root
|
||||
local created_at
|
||||
local git_commit
|
||||
local service
|
||||
local first=1
|
||||
local environment_scope="same-env-only"
|
||||
|
||||
if [[ "$dataset" == "personnel" ]]; then
|
||||
environment_scope="staging-rehearsal"
|
||||
fi
|
||||
|
||||
repo_root="$(backup_repo_root)"
|
||||
created_at="$(backup_utc_now)"
|
||||
@@ -20,7 +26,8 @@ create_manifest() {
|
||||
printf ' "created_at": "%s",\n' "$created_at"
|
||||
printf ' "git_commit": "%s",\n' "$git_commit"
|
||||
printf ' "mode": "%s",\n' "$mode"
|
||||
printf ' "environment_scope": "same-env-only",\n'
|
||||
printf ' "dataset": "%s",\n' "$dataset"
|
||||
printf ' "environment_scope": "%s",\n' "$environment_scope"
|
||||
printf ' "services": ['
|
||||
for service in $services; do
|
||||
if [[ "$first" -eq 1 ]]; then
|
||||
|
||||
367
scripts/backup/lib/personnel_dataset.sh
Normal file
367
scripts/backup/lib/personnel_dataset.sh
Normal file
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
personnel_dataset_dir() {
|
||||
local backup_dir="$1"
|
||||
printf '%s\n' "$backup_dir/datasets/personnel"
|
||||
}
|
||||
|
||||
write_personnel_dataset_manifest() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
local dataset_dir
|
||||
local reports_dir
|
||||
local include_kratos
|
||||
local reset_credentials
|
||||
local include_worksmobile_mapping
|
||||
local include_outbox
|
||||
local tenant_roots
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
reports_dir="$dataset_dir/reports"
|
||||
include_kratos="${PERSONNEL_INCLUDE_KRATOS_IDENTITIES:-true}"
|
||||
reset_credentials="${PERSONNEL_RESET_CREDENTIALS:-true}"
|
||||
include_worksmobile_mapping="${PERSONNEL_INCLUDE_WORKSMOBILE_MAPPING:-true}"
|
||||
include_outbox="${PERSONNEL_INCLUDE_OUTBOX:-false}"
|
||||
tenant_roots="${PERSONNEL_TENANT_ROOT_SLUGS:-}"
|
||||
|
||||
mkdir -p "$reports_dir"
|
||||
jq -n \
|
||||
--arg dataset "personnel" \
|
||||
--arg services "$services" \
|
||||
--arg tenant_roots "$tenant_roots" \
|
||||
--arg include_kratos "$include_kratos" \
|
||||
--arg reset_credentials "$reset_credentials" \
|
||||
--arg include_worksmobile_mapping "$include_worksmobile_mapping" \
|
||||
--arg include_outbox "$include_outbox" \
|
||||
'{
|
||||
dataset: $dataset,
|
||||
format_version: "1",
|
||||
services: ($services | split(" ") | map(select(length > 0))),
|
||||
scope: {
|
||||
tenant_root_slugs: (if $tenant_roots == "" then [] else ($tenant_roots | split(",") | map(gsub("^\\s+|\\s+$"; ""))) end)
|
||||
},
|
||||
included: {
|
||||
baron_postgres_tables: [
|
||||
"public.tenants",
|
||||
"public.tenant_domains",
|
||||
"public.user_groups",
|
||||
"public.users",
|
||||
"public.user_login_ids",
|
||||
"public.worksmobile_resource_mappings"
|
||||
],
|
||||
ory_kratos_tables: (if $include_kratos == "true" then ["public.identities"] else [] end),
|
||||
ory_keto_tables: ["public.keto_relation_tuples"]
|
||||
},
|
||||
excluded: {
|
||||
databases: ["ory_hydra"],
|
||||
tables: [
|
||||
"public.relying_parties",
|
||||
"public.rp_user_metadata",
|
||||
"public.client_consents",
|
||||
"public.client_secrets",
|
||||
"public.api_keys",
|
||||
"public.worksmobile_outboxes"
|
||||
],
|
||||
volatile: ["redis", "sessions", "oauth2_tokens", "audit_logs"]
|
||||
},
|
||||
restore_policy: {
|
||||
reset_credentials: ($reset_credentials == "true"),
|
||||
include_worksmobile_mapping: ($include_worksmobile_mapping == "true"),
|
||||
include_outbox: ($include_outbox == "true"),
|
||||
hydra_restore: false,
|
||||
rp_metadata_restore: false,
|
||||
default_mode: "replace-dataset-rows"
|
||||
}
|
||||
}' >"$dataset_dir/dataset-manifest.json"
|
||||
}
|
||||
|
||||
personnel_psql_jsonl() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local sql="$5"
|
||||
local output_file="$6"
|
||||
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 -c "$sql" \
|
||||
>"$output_file"
|
||||
}
|
||||
|
||||
personnel_write_count() {
|
||||
local report_file="$1"
|
||||
local key="$2"
|
||||
local file_path="$3"
|
||||
local count
|
||||
|
||||
if [[ -f "$file_path" ]]; then
|
||||
count="$(wc -l <"$file_path" | tr -d '[:space:]')"
|
||||
else
|
||||
count="0"
|
||||
fi
|
||||
printf '%s:%s\n' "$key" "$count" >>"$report_file"
|
||||
}
|
||||
|
||||
dump_personnel_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
local dataset_dir
|
||||
local output_dir
|
||||
local report_file
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
output_dir="$dataset_dir/postgres"
|
||||
report_file="$dataset_dir/reports/row-counts.txt"
|
||||
mkdir -p "$output_dir" "$dataset_dir/reports"
|
||||
: >"$report_file"
|
||||
|
||||
backup_log "Dumping personnel Baron Postgres dataset: $db_name"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.tenants t order by t.id" \
|
||||
"$output_dir/tenants.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.tenant_domains t order by t.id" \
|
||||
"$output_dir/tenant_domains.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.user_groups t order by t.id" \
|
||||
"$output_dir/user_groups.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select (to_jsonb(t) - 'password_hash' - 'relying_party_id' || jsonb_build_object('relying_party_id', null))::text from public.users t order by t.id" \
|
||||
"$output_dir/users.jsonl"
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.user_login_ids t order by t.id" \
|
||||
"$output_dir/user_login_ids.jsonl"
|
||||
|
||||
if [[ "${PERSONNEL_INCLUDE_WORKSMOBILE_MAPPING:-true}" == "true" ]]; then
|
||||
personnel_psql_jsonl baron_postgres "$db_user" "$db_password" "$db_name" \
|
||||
"select to_jsonb(t)::text from public.worksmobile_resource_mappings t where t.baron_resource_type in ('USER', 'ORGUNIT') order by t.id" \
|
||||
"$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
else
|
||||
: >"$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
fi
|
||||
|
||||
personnel_write_count "$report_file" "public.tenants" "$output_dir/tenants.jsonl"
|
||||
personnel_write_count "$report_file" "public.tenant_domains" "$output_dir/tenant_domains.jsonl"
|
||||
personnel_write_count "$report_file" "public.user_groups" "$output_dir/user_groups.jsonl"
|
||||
personnel_write_count "$report_file" "public.users" "$output_dir/users.jsonl"
|
||||
personnel_write_count "$report_file" "public.user_login_ids" "$output_dir/user_login_ids.jsonl"
|
||||
personnel_write_count "$report_file" "public.worksmobile_resource_mappings" "$output_dir/worksmobile_resource_mappings.jsonl"
|
||||
}
|
||||
|
||||
dump_personnel_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local dataset_dir
|
||||
local report_file
|
||||
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
|
||||
dataset_dir="$(personnel_dataset_dir "$backup_dir")"
|
||||
report_file="$dataset_dir/reports/row-counts.txt"
|
||||
mkdir -p "$dataset_dir/ory_kratos" "$dataset_dir/ory_keto" "$dataset_dir/reports"
|
||||
|
||||
if [[ "${PERSONNEL_INCLUDE_KRATOS_IDENTITIES:-true}" == "true" ]]; then
|
||||
backup_log "Dumping personnel Kratos identity subset: $kratos_db"
|
||||
personnel_psql_jsonl ory_postgres "$db_user" "$db_password" "$kratos_db" \
|
||||
"select (to_jsonb(i) - 'metadata_admin')::text from public.identities i order by i.id" \
|
||||
"$dataset_dir/ory_kratos/identities.jsonl"
|
||||
else
|
||||
: >"$dataset_dir/ory_kratos/identities.jsonl"
|
||||
fi
|
||||
jq -n '{policy:"reset_credentials", credentials:[]}' >"$dataset_dir/ory_kratos/identity_credentials.reset-plan.jsonl"
|
||||
|
||||
backup_log "Dumping personnel Keto relation tuple subset: $keto_db"
|
||||
personnel_psql_jsonl ory_postgres "$db_user" "$db_password" "$keto_db" \
|
||||
"select to_jsonb(t)::text from public.keto_relation_tuples t where t.namespace <> 'RelyingParty' and coalesce(t.subject_set_namespace, '') <> 'RelyingParty' order by t.namespace, t.object, t.relation, t.subject_id" \
|
||||
"$dataset_dir/ory_keto/relation_tuples.jsonl"
|
||||
|
||||
personnel_write_count "$report_file" "public.identities" "$dataset_dir/ory_kratos/identities.jsonl"
|
||||
personnel_write_count "$report_file" "public.keto_relation_tuples" "$dataset_dir/ory_keto/relation_tuples.jsonl"
|
||||
}
|
||||
|
||||
dump_personnel_dataset() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
|
||||
write_personnel_dataset_manifest "$backup_dir" "$services"
|
||||
if service_enabled postgres "$services"; then
|
||||
dump_personnel_baron_postgres "$backup_dir"
|
||||
fi
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
dump_personnel_ory_postgres "$backup_dir"
|
||||
fi
|
||||
if service_enabled config "$services"; then
|
||||
dump_config_snapshot "$backup_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
personnel_dataset_manifest_path() {
|
||||
local backup_dir="$1"
|
||||
printf '%s\n' "$backup_dir/datasets/personnel/dataset-manifest.json"
|
||||
}
|
||||
|
||||
restore_personnel_plan_policy_json() {
|
||||
local backup_dir="$1"
|
||||
local manifest_path
|
||||
|
||||
manifest_path="$(personnel_dataset_manifest_path "$backup_dir")"
|
||||
if [[ -f "$manifest_path" ]]; then
|
||||
jq -c '{dataset, included, excluded, restore_policy, scope}' "$manifest_path"
|
||||
else
|
||||
printf '{}\n'
|
||||
fi
|
||||
}
|
||||
|
||||
personnel_restore_jsonl_id_table() {
|
||||
local container="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local database="$4"
|
||||
local table="$5"
|
||||
local input_file="$6"
|
||||
local scratch
|
||||
local columns
|
||||
local updates
|
||||
|
||||
backup_require_path "$input_file"
|
||||
if [[ ! -s "$input_file" ]]; then
|
||||
backup_log "Skipping empty personnel dataset table: public.$table"
|
||||
return 0
|
||||
fi
|
||||
|
||||
scratch="_personnel_restore_${table}_json"
|
||||
columns="$(docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 \
|
||||
-c "select string_agg(quote_ident(column_name), ', ' order by ordinal_position) from information_schema.columns where table_schema = 'public' and table_name = '$table'")"
|
||||
updates="$(docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -At -v ON_ERROR_STOP=1 \
|
||||
-c "select string_agg(format('%I = excluded.%I', column_name, column_name), ', ' order by ordinal_position) from information_schema.columns where table_schema = 'public' and table_name = '$table' and column_name <> 'id'")"
|
||||
|
||||
[[ -n "$columns" ]] || backup_die "cannot resolve columns for personnel restore table: public.$table"
|
||||
[[ -n "$updates" ]] || backup_die "cannot resolve update columns for personnel restore table: public.$table"
|
||||
|
||||
backup_log "Restoring personnel dataset table: public.$table"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "drop table if exists public.${scratch}" \
|
||||
-c "create table public.${scratch} (line jsonb not null)"
|
||||
docker exec -i -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "\\copy public.${scratch}(line) from stdin" <"$input_file"
|
||||
docker exec -e "PGPASSWORD=$password" "$container" \
|
||||
psql -U "$user" -d "$database" -v ON_ERROR_STOP=1 \
|
||||
-c "insert into public.${table} (${columns}) select ${columns} from (select (jsonb_populate_record(null::public.${table}, line)).* from public.${scratch}) r on conflict (id) do update set ${updates}" \
|
||||
-c "drop table public.${scratch}"
|
||||
}
|
||||
|
||||
restore_personnel_baron_postgres() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${DB_USER:-baron}"
|
||||
local db_password="${DB_PASSWORD:-password}"
|
||||
local db_name="${DB_NAME:-baron_sso}"
|
||||
local input_dir
|
||||
|
||||
input_dir="$(personnel_dataset_dir "$backup_dir")/postgres"
|
||||
backup_require_command docker
|
||||
backup_require_container baron_postgres
|
||||
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" tenants "$input_dir/tenants.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" tenant_domains "$input_dir/tenant_domains.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" user_groups "$input_dir/user_groups.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" users "$input_dir/users.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" user_login_ids "$input_dir/user_login_ids.jsonl"
|
||||
personnel_restore_jsonl_id_table baron_postgres "$db_user" "$db_password" "$db_name" worksmobile_resource_mappings "$input_dir/worksmobile_resource_mappings.jsonl"
|
||||
}
|
||||
|
||||
restore_personnel_keto_relation_tuples() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local keto_db="${KETO_DB:-ory_keto}"
|
||||
local input_file
|
||||
local scratch="_personnel_restore_keto_relation_tuples_json"
|
||||
|
||||
input_file="$(personnel_dataset_dir "$backup_dir")/ory_keto/relation_tuples.jsonl"
|
||||
backup_require_path "$input_file"
|
||||
if [[ ! -s "$input_file" ]]; then
|
||||
backup_log "Skipping empty personnel Keto relation tuple dataset"
|
||||
return 0
|
||||
fi
|
||||
|
||||
backup_log "Restoring personnel Keto relation tuple subset: $keto_db"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "drop table if exists public.${scratch}" \
|
||||
-c "create table public.${scratch} (line jsonb not null)"
|
||||
docker exec -i -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "\\copy public.${scratch}(line) from stdin" <"$input_file"
|
||||
docker exec -e "PGPASSWORD=$db_password" ory_postgres \
|
||||
psql -U "$db_user" -d "$keto_db" -v ON_ERROR_STOP=1 \
|
||||
-c "delete from public.keto_relation_tuples where namespace <> 'RelyingParty' and coalesce(subject_set_namespace, '') <> 'RelyingParty'" \
|
||||
-c "insert into public.keto_relation_tuples select (jsonb_populate_record(null::public.keto_relation_tuples, line)).* from public.${scratch}" \
|
||||
-c "drop table public.${scratch}"
|
||||
}
|
||||
|
||||
restore_personnel_kratos_identities() {
|
||||
local backup_dir="$1"
|
||||
local db_user="${ORY_POSTGRES_USER:-ory}"
|
||||
local db_password="${ORY_POSTGRES_PASSWORD:-secret}"
|
||||
local kratos_db="${KRATOS_DB:-ory_kratos}"
|
||||
local input_file
|
||||
|
||||
input_file="$(personnel_dataset_dir "$backup_dir")/ory_kratos/identities.jsonl"
|
||||
backup_require_command docker
|
||||
backup_require_container ory_postgres
|
||||
personnel_restore_jsonl_id_table ory_postgres "$db_user" "$db_password" "$kratos_db" identities "$input_file"
|
||||
}
|
||||
|
||||
restore_personnel_ory_postgres() {
|
||||
local backup_dir="$1"
|
||||
|
||||
restore_personnel_kratos_identities "$backup_dir"
|
||||
restore_personnel_keto_relation_tuples "$backup_dir"
|
||||
}
|
||||
|
||||
restore_personnel_dataset() {
|
||||
local backup_dir="$1"
|
||||
local services="$2"
|
||||
local report_items=()
|
||||
local item
|
||||
|
||||
backup_require_path "$(personnel_dataset_manifest_path "$backup_dir")"
|
||||
|
||||
if service_enabled postgres "$services"; then
|
||||
restore_personnel_baron_postgres "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/postgres" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled ory-postgres "$services"; then
|
||||
restore_personnel_ory_postgres "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/ory-postgres" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if service_enabled config "$services"; then
|
||||
restore_config_snapshot "$backup_dir"
|
||||
item="$(jq -n --arg label "personnel/config" --arg status "passed" '{label:$label, status:$status}')"
|
||||
report_items+=("$item")
|
||||
fi
|
||||
|
||||
if [[ "${#report_items[@]}" -eq 0 ]]; then
|
||||
target_verification_reports="[]"
|
||||
else
|
||||
target_verification_reports="$(printf '%s\n' "${report_items[@]}" | jq -s '.')"
|
||||
fi
|
||||
target_verification_status="passed"
|
||||
}
|
||||
@@ -130,6 +130,7 @@ write_restore_markdown_report() {
|
||||
"| 입력 유형 | \(.backup_source // "unknown") |",
|
||||
"| 백업 경로 | `\(.backup_dir // "")` |",
|
||||
"| Dump 파일 | `\(.dump_file // "")` |",
|
||||
"| Dataset | `\(.dataset // "full")` |",
|
||||
"| 서비스 | `\(services)` |",
|
||||
"",
|
||||
"## 검증",
|
||||
@@ -138,6 +139,9 @@ write_restore_markdown_report() {
|
||||
"| --- | --- |",
|
||||
"| Dump checksum | \(.verification.dump_checksum // "not_run") |",
|
||||
"| 대상 row count | \(.verification.target_row_counts // "not_run") |",
|
||||
(if (.dataset // "full") == "personnel" then
|
||||
"| Personnel exclusions | `\((.restore_policy.excluded.databases // []) | join(", "))` / `\((.restore_policy.excluded.tables // []) | join(", "))` |"
|
||||
else empty end),
|
||||
"",
|
||||
"## 대상별 검증 결과",
|
||||
"",
|
||||
|
||||
Reference in New Issue
Block a user