백업/복구로직 변경, 깜빡임 버그 해결

2026-06-05 12:26:51 +09:00
parent 4bae1dd00d
commit 29038254dd
43 changed files with 3695 additions and 75 deletions
--- a/scripts/backup/lib/clickhouse.sh
+++ b/scripts/backup/lib/clickhouse.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+clickhouse_query() {
+  local container="$1"
+  local user="$2"
+  local password="$3"
+  local query="$4"
+
+  docker exec "$container" clickhouse-client --user "$user" --password "$password" --query "$query"
+}
+
+render_clickhouse_schema() {
+  local schema_file="$1"
+
+  if head -n 1 "$schema_file" | grep -q '\\n'; then
+    perl -0pe 's{\\n}{\n}g; s{\\t}{\t}g; s{\\\x27}{\x27}g' "$schema_file"
+    return
+  fi
+
+  cat "$schema_file"
+}
+
+dump_clickhouse_container() {
+  local backup_dir="$1"
+  local service_name="$2"
+  local container="$3"
+  local user="$4"
+  local password="$5"
+  local output_dir="$backup_dir/clickhouse/$service_name"
+  local table_list="$output_dir/tables.tsv"
+  local database
+  local table
+  local engine
+  local safe_name
+
+  backup_require_command docker
+  backup_require_container "$container"
+  mkdir -p "$output_dir/schema" "$output_dir/data" "$backup_dir/reports"
+
+  backup_log "Dumping ClickHouse metadata and Native data: $container"
+  clickhouse_query "$container" "$user" "$password" \
+    "select database, name, engine from system.tables where database not in ('INFORMATION_SCHEMA','information_schema','system') order by database, if(positionCaseInsensitive(engine, 'View') > 0, 1, 0), name format TSV" \
+    >"$table_list"
+
+  while IFS=$'\t' read -r database table engine; do
+    [[ -n "$database" && -n "$table" ]] || continue
+    safe_name="${database}__${table}"
+    clickhouse_query "$container" "$user" "$password" "show create table \`${database}\`.\`${table}\` FORMAT RawBLOB" >"$output_dir/schema/${safe_name}.sql"
+    if [[ "$engine" != *View* ]]; then
+      clickhouse_query "$container" "$user" "$password" "select * from \`${database}\`.\`${table}\` format Native" >"$output_dir/data/${safe_name}.native"
+    fi
+    clickhouse_query "$container" "$user" "$password" "select '${database}.${table}:' || toString(count()) from \`${database}\`.\`${table}\`" \
+      >>"$backup_dir/reports/${service_name}-row-counts.txt"
+  done <"$table_list"
+}
+
+restore_clickhouse_container() {
+  local backup_dir="$1"
+  local service_name="$2"
+  local container="$3"
+  local user="$4"
+  local password="$5"
+  local input_dir="$backup_dir/clickhouse/$service_name"
+  local table_list="$input_dir/tables.tsv"
+  local database
+  local table
+  local engine
+  local safe_name
+  local restored_databases=""
+
+  backup_require_command docker
+  backup_require_container "$container"
+  backup_require_path "$table_list"
+
+  backup_log "Restoring ClickHouse tables: $container"
+  while IFS=$'\t' read -r database table engine; do
+    [[ -n "$database" && -n "$table" ]] || continue
+    if ! grep -qw -- "$database" <<<"$restored_databases"; then
+      docker exec "$container" clickhouse-client --user "$user" --password "$password" \
+        --query "drop database if exists \`${database}\`"
+      docker exec "$container" clickhouse-client --user "$user" --password "$password" \
+        --query "create database if not exists \`${database}\`"
+      restored_databases="${restored_databases:+$restored_databases }$database"
+    fi
+  done <"$table_list"
+
+  while IFS=$'\t' read -r database table engine; do
+    [[ -n "$database" && -n "$table" ]] || continue
+    safe_name="${database}__${table}"
+    backup_require_path "$input_dir/schema/${safe_name}.sql"
+    render_clickhouse_schema "$input_dir/schema/${safe_name}.sql" \
+      | docker exec -i "$container" clickhouse-client --user "$user" --password "$password" --multiquery
+    if [[ "$engine" != *View* ]]; then
+      backup_require_path "$input_dir/data/${safe_name}.native"
+      docker exec -i "$container" clickhouse-client --user "$user" --password "$password" \
+        --query "insert into \`${database}\`.\`${table}\` format Native" <"$input_dir/data/${safe_name}.native"
+    fi
+  done <"$table_list"
+}
+
+dump_baron_clickhouse() {
+  dump_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
+}
+
+dump_ory_clickhouse() {
+  dump_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
+}
+
+restore_baron_clickhouse() {
+  restore_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
+}
+
+restore_ory_clickhouse() {
+  restore_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
+}