#!/usr/bin/env bash clickhouse_query() { local container="$1" local user="$2" local password="$3" local query="$4" docker exec "$container" clickhouse-client --user "$user" --password "$password" --query "$query" } render_clickhouse_schema() { local schema_file="$1" if head -n 1 "$schema_file" | grep -q '\\n'; then perl -0pe 's{\\n}{\n}g; s{\\t}{\t}g; s{\\\x27}{\x27}g' "$schema_file" return fi cat "$schema_file" } dump_clickhouse_container() { local backup_dir="$1" local service_name="$2" local container="$3" local user="$4" local password="$5" local output_dir="$backup_dir/clickhouse/$service_name" local table_list="$output_dir/tables.tsv" local database local table local engine local safe_name backup_require_command docker backup_require_container "$container" mkdir -p "$output_dir/schema" "$output_dir/data" "$backup_dir/reports" backup_log "Dumping ClickHouse metadata and Native data: $container" clickhouse_query "$container" "$user" "$password" \ "select database, name, engine from system.tables where database not in ('INFORMATION_SCHEMA','information_schema','system') order by database, if(positionCaseInsensitive(engine, 'View') > 0, 1, 0), name format TSV" \ >"$table_list" while IFS=$'\t' read -r database table engine; do [[ -n "$database" && -n "$table" ]] || continue safe_name="${database}__${table}" clickhouse_query "$container" "$user" "$password" "show create table \`${database}\`.\`${table}\` FORMAT RawBLOB" >"$output_dir/schema/${safe_name}.sql" if [[ "$engine" != *View* ]]; then clickhouse_query "$container" "$user" "$password" "select * from \`${database}\`.\`${table}\` format Native" >"$output_dir/data/${safe_name}.native" fi clickhouse_query "$container" "$user" "$password" "select '${database}.${table}:' || toString(count()) from \`${database}\`.\`${table}\`" \ >>"$backup_dir/reports/${service_name}-row-counts.txt" done <"$table_list" } restore_clickhouse_container() { local backup_dir="$1" local service_name="$2" local container="$3" local user="$4" local password="$5" local input_dir="$backup_dir/clickhouse/$service_name" local table_list="$input_dir/tables.tsv" local database local table local engine local safe_name local restored_databases="" backup_require_command docker backup_require_container "$container" backup_require_path "$table_list" backup_log "Restoring ClickHouse tables: $container" while IFS=$'\t' read -r database table engine; do [[ -n "$database" && -n "$table" ]] || continue if ! grep -qw -- "$database" <<<"$restored_databases"; then docker exec "$container" clickhouse-client --user "$user" --password "$password" \ --query "drop database if exists \`${database}\`" docker exec "$container" clickhouse-client --user "$user" --password "$password" \ --query "create database if not exists \`${database}\`" restored_databases="${restored_databases:+$restored_databases }$database" fi done <"$table_list" while IFS=$'\t' read -r database table engine; do [[ -n "$database" && -n "$table" ]] || continue safe_name="${database}__${table}" backup_require_path "$input_dir/schema/${safe_name}.sql" render_clickhouse_schema "$input_dir/schema/${safe_name}.sql" \ | docker exec -i "$container" clickhouse-client --user "$user" --password "$password" --multiquery if [[ "$engine" != *View* ]]; then backup_require_path "$input_dir/data/${safe_name}.native" docker exec -i "$container" clickhouse-client --user "$user" --password "$password" \ --query "insert into \`${database}\`.\`${table}\` format Native" <"$input_dir/data/${safe_name}.native" fi done <"$table_list" } dump_baron_clickhouse() { dump_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}" } dump_ory_clickhouse() { dump_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}" } restore_baron_clickhouse() { restore_clickhouse_container "$1" "baron_clickhouse" "baron_clickhouse" "${CLICKHOUSE_USER:-baron}" "${CLICKHOUSE_PASSWORD:-password}" } restore_ory_clickhouse() { restore_clickhouse_container "$1" "ory_clickhouse" "ory_clickhouse" "${ORY_CLICKHOUSE_USER:-ory}" "${ORY_CLICKHOUSE_PASSWORD:-orypass}" }