forked from baron/baron-sso
116 lines
4.4 KiB
Bash
116 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
clickhouse_query() {
|
|
local container="$1"
|
|
local user="$2"
|
|
local password="$3"
|
|
local query="$4"
|
|
|
|
docker exec "$container" clickhouse-client --user "$user" --password "$password" --query "$query"
|
|
}
|
|
|
|
render_clickhouse_schema() {
|
|
local schema_file="$1"
|
|
|
|
if head -n 1 "$schema_file" | grep -q '\\n'; then
|
|
perl -0pe 's{\\n}{\n}g; s{\\t}{\t}g; s{\\\x27}{\x27}g' "$schema_file"
|
|
return
|
|
fi
|
|
|
|
cat "$schema_file"
|
|
}
|
|
|
|
dump_clickhouse_container() {
|
|
local backup_dir="$1"
|
|
local service_name="$2"
|
|
local container="$3"
|
|
local user="$4"
|
|
local password="$5"
|
|
local output_dir="$backup_dir/clickhouse/$service_name"
|
|
local table_list="$output_dir/tables.tsv"
|
|
local database
|
|
local table
|
|
local engine
|
|
local safe_name
|
|
|
|
backup_require_command docker
|
|
backup_require_container "$container"
|
|
mkdir -p "$output_dir/schema" "$output_dir/data" "$backup_dir/reports"
|
|
|
|
backup_log "Dumping ClickHouse metadata and Native data: $container"
|
|
clickhouse_query "$container" "$user" "$password" \
|
|
"select database, name, engine from system.tables where database not in ('INFORMATION_SCHEMA','information_schema','system') order by database, if(positionCaseInsensitive(engine, 'View') > 0, 1, 0), name format TSV" \
|
|
>"$table_list"
|
|
|
|
while IFS=$'\t' read -r database table engine; do
|
|
[[ -n "$database" && -n "$table" ]] || continue
|
|
safe_name="${database}__${table}"
|
|
clickhouse_query "$container" "$user" "$password" "show create table \`${database}\`.\`${table}\` FORMAT RawBLOB" >"$output_dir/schema/${safe_name}.sql"
|
|
if [[ "$engine" != *View* ]]; then
|
|
clickhouse_query "$container" "$user" "$password" "select * from \`${database}\`.\`${table}\` format Native" >"$output_dir/data/${safe_name}.native"
|
|
fi
|
|
clickhouse_query "$container" "$user" "$password" "select '${database}.${table}:' || toString(count()) from \`${database}\`.\`${table}\`" \
|
|
>>"$backup_dir/reports/${service_name}-row-counts.txt"
|
|
done <"$table_list"
|
|
}
|
|
|
|
restore_clickhouse_container() {
|
|
local backup_dir="$1"
|
|
local service_name="$2"
|
|
local container="$3"
|
|
local user="$4"
|
|
local password="$5"
|
|
local input_dir="$backup_dir/clickhouse/$service_name"
|
|
local table_list="$input_dir/tables.tsv"
|
|
local database
|
|
local table
|
|
local engine
|
|
local safe_name
|
|
local restored_databases=""
|
|
|
|
backup_require_command docker
|
|
backup_require_container "$container"
|
|
backup_require_path "$table_list"
|
|
|
|
backup_log "Restoring ClickHouse tables: $container"
|
|
while IFS=$'\t' read -r database table engine; do
|
|
[[ -n "$database" && -n "$table" ]] || continue
|
|
if ! grep -qw -- "$database" <<<"$restored_databases"; then
|
|
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
|
--query "drop database if exists \`${database}\`"
|
|
docker exec "$container" clickhouse-client --user "$user" --password "$password" \
|
|
--query "create database if not exists \`${database}\`"
|
|
restored_databases="${restored_databases:+$restored_databases }$database"
|
|
fi
|
|
done <"$table_list"
|
|
|
|
while IFS=$'\t' read -r database table engine; do
|
|
[[ -n "$database" && -n "$table" ]] || continue
|
|
safe_name="${database}__${table}"
|
|
backup_require_path "$input_dir/schema/${safe_name}.sql"
|
|
render_clickhouse_schema "$input_dir/schema/${safe_name}.sql" \
|
|
| docker exec -i "$container" clickhouse-client --user "$user" --password "$password" --multiquery
|
|
if [[ "$engine" != *View* ]]; then
|
|
backup_require_path "$input_dir/data/${safe_name}.native"
|
|
docker exec -i "$container" clickhouse-client --user "$user" --password "$password" \
|
|
--query "insert into \`${database}\`.\`${table}\` format Native" <"$input_dir/data/${safe_name}.native"
|
|
fi
|
|
done <"$table_list"
|
|
}
|
|
|
|
dump_baron_clickhouse() {
|
|
dump_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
|
|
}
|
|
|
|
dump_ory_clickhouse() {
|
|
dump_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
|
|
}
|
|
|
|
restore_baron_clickhouse() {
|
|
restore_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}"
|
|
}
|
|
|
|
restore_ory_clickhouse() {
|
|
restore_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}"
|
|
}
|