tdc114plus/scripts/startup.sh

#!/usr/bin/env bash
# scripts/startup.sh
# Startup helper for tdc114plus development environment
# Usage: ./scripts/startup.sh [--dry-run] [--auto] [--wait=N] [--skip-android-check]

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
BARON_DIR="${BARON_SSO_WORKTREE:-/home/ubuntu/workspace/baron-sso-tdc114plus-api}"
LOG_BASE_DIR="${TDC114_LOG_BASE:-$ROOT_DIR/logs}"
LOG_DIR="$LOG_BASE_DIR/$(date +%F)"
LOG_FILE="$LOG_DIR/startup.log"
CHECK_SCRIPT="${TDC114_CHECK_SCRIPT:-$ROOT_DIR/scripts/check-baron-api-env.sh}"
API_SMOKE_SCRIPT="${TDC114_API_SMOKE_SCRIPT:-$ROOT_DIR/scripts/api-smoke.sh}"
DOCKER_BIN="${DOCKER_BIN:-docker}"
MAKE_BIN="${MAKE_BIN:-make}"
CHMOD_BIN="${CHMOD_BIN:-chmod}"
REQUIRE_AUTH_SMOKE="${TDC114_REQUIRE_AUTH_SMOKE:-false}"
ANDROID_PRECHECK_SCRIPT="${TDC114_ANDROID_PRECHECK_SCRIPT:-$ROOT_DIR/scripts/check-android-device-env.sh}"
AUTH_SERVER_SCRIPT="${TDC114_AUTH_SERVER_SCRIPT:-$ROOT_DIR/scripts/start-auth-server.sh}"
START_AUTH_SERVER="${TDC114_START_AUTH_SERVER:-true}"
DAILY_HANDOFF_DIR="${TDC114_DAILY_HANDOFF_DIR:-$ROOT_DIR/docs/daily-issues}"
REQUIRE_DAILY_HANDOFF="${TDC114_REQUIRE_DAILY_HANDOFF:-true}"
WSL_MAINTENANCE_SCRIPT="${TDC114_WSL_MAINTENANCE_SCRIPT:-$ROOT_DIR/scripts/check-wsl-maintenance.sh}"
WSL_MAINTENANCE_ENABLED="${TDC114_WSL_MAINTENANCE_ENABLED:-true}"

DRY_RUN=true
AUTO=false
WAIT=30
RETRIES=6
SLEEP_BETWEEN=10
SKIP_ANDROID_CHECK=false

for arg in "$@"; do
  case "$arg" in
    --dry-run) DRY_RUN=true ;;
    --auto) DRY_RUN=false; AUTO=true ;;
    --wait=*) WAIT=${arg#*=} ;;
    --skip-android-check|--skip-device-check) SKIP_ANDROID_CHECK=true ;;
    --help|-h)
      cat <<EOF
Usage: $0 [--dry-run] [--auto] [--wait=N] [--skip-android-check]

--dry-run  : show commands and write non-destructive logs (default)
--auto     : actually perform startup actions
--wait=N   : seconds to wait for services to stabilize (default 30)
--skip-android-check : skip Android target preflight (not recommended)
--help     : show this message
EOF
      exit 0
      ;;
    --wait) echo "Please use --wait=N" >&2; exit 2 ;;
    *) echo "Unknown arg: $arg" >&2; exit 2 ;;
  esac
done

case "$WAIT" in
  ''|*[!0-9]*)
    echo "--wait must be a non-negative integer" >&2
    exit 2
    ;;
esac

mkdir -p "$LOG_DIR"

log() {
  echo "$*" | tee -a "$LOG_FILE"
}

run_cmd() {
  log "+ $*"
  if [ "$DRY_RUN" = false ]; then
    set +e
    bash -lc "$*" 2>&1 | tee -a "$LOG_FILE"
    local rc=${PIPESTATUS[0]}
    set -e
    return "$rc"
  fi
}

run_cmd_capture() {
  local output_file="$1"
  shift

  log "+ $*"
  if [ "$DRY_RUN" = false ]; then
    set +e
    bash -lc "$*" 2>&1 | tee "$output_file" | tee -a "$LOG_FILE"
    local rc=${PIPESTATUS[0]}
    set -e
    return "$rc"
  fi
}

compose_cmd() {
  printf "%s compose -f docker-compose.yaml -f compose.infra.yaml -f compose.ory.yaml %s" \
    "$DOCKER_BIN" "$1"
}

ensure_required_paths() {
  if [ ! -d "$BARON_DIR" ]; then
    log "Startup aborted: Baron worktree not found at $BARON_DIR"
    exit 1
  fi
  if [ ! -x "$CHECK_SCRIPT" ]; then
    log "Startup aborted: check script is missing or not executable at $CHECK_SCRIPT"
    exit 1
  fi
  if [ ! -x "$API_SMOKE_SCRIPT" ]; then
    log "Startup aborted: api smoke script is missing or not executable at $API_SMOKE_SCRIPT"
    exit 1
  fi
  if [ "$SKIP_ANDROID_CHECK" = false ] && [ ! -x "$ANDROID_PRECHECK_SCRIPT" ]; then
    log "Startup aborted: Android precheck script is missing or not executable at $ANDROID_PRECHECK_SCRIPT"
    exit 1
  fi
  if [ "$START_AUTH_SERVER" = true ] && [ ! -x "$AUTH_SERVER_SCRIPT" ]; then
    log "Startup aborted: auth server script is missing or not executable at $AUTH_SERVER_SCRIPT"
    exit 1
  fi
  if [ ! -d "$DAILY_HANDOFF_DIR" ]; then
    log "Startup aborted: daily handoff directory is missing at $DAILY_HANDOFF_DIR"
    exit 1
  fi
  if [ "$WSL_MAINTENANCE_ENABLED" = true ] && [ ! -x "$WSL_MAINTENANCE_SCRIPT" ]; then
    log "Startup aborted: WSL maintenance script is missing or not executable at $WSL_MAINTENANCE_SCRIPT"
    exit 1
  fi
}

review_daily_handoff() {
  local today
  local handoff_file

  today="$(date +%F)"
  handoff_file="$(find "$DAILY_HANDOFF_DIR" -maxdepth 1 -type f -name '20*.md' ! -name "$today*" | sort | tail -n 1 || true)"

  if [ -z "$handoff_file" ]; then
    log "Daily handoff review: no previous handoff file found in $DAILY_HANDOFF_DIR"
    if [ "$REQUIRE_DAILY_HANDOFF" = true ]; then
      log "Startup aborted: previous daily handoff is required. Set TDC114_REQUIRE_DAILY_HANDOFF=false only for exceptional recovery."
      exit 1
    fi
    return 0
  fi

  log "Daily handoff review required before startup"
  log "Daily handoff file: $handoff_file"
  log "----- daily handoff excerpt start -----"
  sed -n '1,160p' "$handoff_file" | tee -a "$LOG_FILE"
  log "----- daily handoff excerpt end -----"
}

review_wsl_maintenance() {
  if [ "$WSL_MAINTENANCE_ENABLED" != true ]; then
    log "WSL maintenance reminder skipped by TDC114_WSL_MAINTENANCE_ENABLED=$WSL_MAINTENANCE_ENABLED"
    return 0
  fi

  log "WSL maintenance review started: $WSL_MAINTENANCE_SCRIPT status"
  set +e
  (
    cd "$ROOT_DIR"
    "$WSL_MAINTENANCE_SCRIPT" status
  ) 2>&1 | tee -a "$LOG_FILE"
  local rc=${PIPESTATUS[0]}
  set -e

  if [ "$rc" -ne 0 ]; then
    log "Startup aborted: WSL maintenance review script failed"
    exit 1
  fi
}

run_android_precheck() {
  if [ "$SKIP_ANDROID_CHECK" = true ]; then
    log "Android target preflight skipped by option"
    return 0
  fi

  log "Android target preflight started: $ANDROID_PRECHECK_SCRIPT"
  set +e
  (
    cd "$ROOT_DIR"
    "$ANDROID_PRECHECK_SCRIPT"
  ) 2>&1 | tee -a "$LOG_FILE"
  local rc=${PIPESTATUS[0]}
  set -e

  if [ "$rc" -ne 0 ]; then
    log "Startup paused: Android target preflight did not pass"
    log "Resolve the physical-device or ADB steps above, then rerun startup."
    exit 1
  fi

  log "Android target preflight passed"
}

ensure_auth_server() {
  if [ "$START_AUTH_SERVER" != true ]; then
    log "tdc114plus-auth startup skipped by TDC114_START_AUTH_SERVER=$START_AUTH_SERVER"
    return 0
  fi

  log "tdc114plus-auth startup started: $AUTH_SERVER_SCRIPT --restart"
  set +e
  (
    cd "$ROOT_DIR"
    "$AUTH_SERVER_SCRIPT" --restart
  ) 2>&1 | tee -a "$LOG_FILE"
  local rc=${PIPESTATUS[0]}
  set -e

  if [ "$rc" -ne 0 ]; then
    log "Startup aborted: tdc114plus-auth did not start"
    exit 1
  fi

  log "tdc114plus-auth ready"
}

ensure_generated_config_writable() {
  local generated_dir="$BARON_DIR/config/.generated"

  run_cmd "mkdir -p '$generated_dir'"
  run_cmd "cd '$BARON_DIR' && $CHMOD_BIN -R u+w config/.generated 2>/dev/null || true"

  if [ "$DRY_RUN" = false ] && [ ! -w "$generated_dir" ]; then
    log "Startup aborted: $generated_dir is not writable by $(id -un). Fix ownership/permissions before startup."
    exit 1
  fi
}

sanitize_rendered_kratos_config() {
  local kratos_config="$BARON_DIR/config/.generated/ory/kratos/kratos.yml"

  if [ ! -f "$kratos_config" ]; then
    log "Startup aborted: rendered Kratos config is missing at $kratos_config"
    exit 1
  fi

  if grep -Fq "        - []" "$kratos_config"; then
    log "Rendered Kratos config contains invalid empty allowed_return_urls entry; removing it automatically"
    set +e
    perl -0pi -e 's/^[ ]*-\s*\[\]\n//mg' "$kratos_config"
    local rc=$?
    set -e
    if [ "$rc" -ne 0 ]; then
      log "Startup aborted: failed to sanitize $kratos_config"
      exit 1
    fi
  fi
}

ensure_rendered_ory_files() {
  local oathkeeper_entrypoint="$BARON_DIR/config/.generated/ory/oathkeeper/entrypoint.sh"

  sanitize_rendered_kratos_config

  if [ ! -x "$oathkeeper_entrypoint" ]; then
    log "Startup aborted: rendered Oathkeeper entrypoint is missing or not executable at $oathkeeper_entrypoint"
    exit 1
  fi
}

capture_failure_diagnostics() {
  local prefix="${1:-runtime}"

  if [ "$DRY_RUN" = true ]; then
    return 0
  fi

  run_cmd "mkdir -p '$LOG_DIR'"
  run_cmd "cd '$BARON_DIR' && $(compose_cmd "ps") > '$LOG_DIR/${prefix}-compose-ps.log' 2>&1 || true"
  run_cmd "$DOCKER_BIN ps -a --format '{{.Names}} {{.Status}}' | grep -E 'baron|ory' > '$LOG_DIR/${prefix}-docker-ps.log' 2>&1 || true"
  run_cmd "$DOCKER_BIN logs --tail 120 ory_kratos > '$LOG_DIR/${prefix}-ory_kratos.log' 2>&1 || true"
  run_cmd "$DOCKER_BIN logs --tail 120 ory_stack_check > '$LOG_DIR/${prefix}-ory_stack_check.log' 2>&1 || true"
  run_cmd "$DOCKER_BIN logs --tail 120 oathkeeper > '$LOG_DIR/${prefix}-oathkeeper.log' 2>&1 || true"
  run_cmd "$DOCKER_BIN logs --tail 120 baron_backend > '$LOG_DIR/${prefix}-baron_backend.log' 2>&1 || true"
}

run_check_script() {
  local attempt="$1"
  local output_file="$LOG_DIR/check-baron-api-env.attempt-${attempt}.log"

  set +e
  (
    cd "$ROOT_DIR"
    "$CHECK_SCRIPT"
  ) 2>&1 | tee "$output_file" | tee -a "$LOG_FILE"
  local rc=${PIPESTATUS[0]}
  set -e

  if [ "$rc" -ne 0 ]; then
    log "check-baron-api-env.sh failed on attempt $attempt"
    return 1
  fi

  if ! grep -Fq "RESULT ready-ish: 0 failure(s), 0 warning(s)" "$output_file"; then
    log "check-baron-api-env.sh reported warnings on attempt $attempt"
    return 1
  fi

  log "check-baron-api-env.sh passed with zero warnings"
  return 0
}

run_api_smoke_script() {
  local attempt="$1"
  local output_file="$LOG_DIR/api-smoke.attempt-${attempt}.log"
  local fallback_output_file="$LOG_DIR/api-smoke.base-only.attempt-${attempt}.log"

  set +e
  (
    cd "$ROOT_DIR"
    "$API_SMOKE_SCRIPT"
  ) 2>&1 | tee "$output_file" | tee -a "$LOG_FILE"
  local rc=${PIPESTATUS[0]}
  set -e

  if [ "$rc" -ne 0 ]; then
    if [ "$REQUIRE_AUTH_SMOKE" != "true" ] && grep -Fq '"code":"login_failed"' "$output_file"; then
      log "api-smoke authenticated login failed; retrying in base-smoke-only mode because TDC114_REQUIRE_AUTH_SMOKE is false"
      set +e
      (
        cd "$ROOT_DIR"
        TDC114_SKIP_AUTH_SMOKE=true "$API_SMOKE_SCRIPT"
      ) 2>&1 | tee "$fallback_output_file" | tee -a "$LOG_FILE"
      rc=${PIPESTATUS[0]}
      set -e

      if [ "$rc" -eq 0 ]; then
        log "api-smoke base-smoke-only mode passed"
        return 0
      fi
    fi

    log "api-smoke.sh failed on attempt $attempt"
    return 1
  fi

  log "api-smoke.sh passed"
  return 0
}

start_stack() {
  local down_cmd="cd '$BARON_DIR' && $(compose_cmd "down")"
  local up_cmd="cd '$BARON_DIR' && $(compose_cmd "up -d")"
  local output_file="$LOG_DIR/compose-up.initial.log"

  run_cmd_capture "$LOG_DIR/compose-down.initial.log" "$down_cmd" || true

  if run_cmd_capture "$output_file" "$up_cmd"; then
    return 0
  fi

  if grep -Eq "name conflict|is already in use by container" "$output_file"; then
    log "compose up failed due to stale or conflicting containers; attempting automatic cleanup"
    run_cmd "cd '$BARON_DIR' && $(compose_cmd "down") >/dev/null 2>&1 || true"
    run_cmd "$DOCKER_BIN ps -a --format '{{.Names}}' | grep -iE 'baron|ory' | xargs -r $DOCKER_BIN rm -f || true"
    run_cmd_capture "$LOG_DIR/compose-up.retry.log" "$up_cmd"
    return $?
  fi

  if grep -Eq "dependency failed to start|didn't complete successfully|is unhealthy" "$output_file"; then
    log "compose up reported dependency or health issues; continuing to stabilization checks before declaring failure"
    return 0
  fi

  return 1
}

ensure_required_paths

log "Startup started: $(date)"

# 0) Always review the previous daily handoff before starting runtime work.
review_daily_handoff
review_wsl_maintenance

# 1) Verify Android target readiness before backend startup work
run_android_precheck

# 2) Ensure networks and config rendered (if Makefile target exists)
if [ -f "$BARON_DIR/Makefile" ]; then
  ensure_generated_config_writable
  run_cmd "cd '$BARON_DIR' && $MAKE_BIN ensure-networks || true"
  if ! run_cmd "cd '$BARON_DIR' && $MAKE_BIN render-ory-config"; then
    log "Startup aborted: make render-ory-config failed"
    exit 1
  fi
  ensure_rendered_ory_files
fi

# 3) Start all necessary compose stacks, retrying once after conflict cleanup
start_stack

# 4) Wait for services to stabilize
run_cmd "sleep $WAIT"

# 5) Show container status
run_cmd "$DOCKER_BIN ps --format '{{.Names}} {{.Status}}' | grep -E 'baron|ory' || true"

# 5-1) Start local auth broker used by the app.
ensure_auth_server

# 6) Run health checks and smoke tests with retries
startup_ok=false
i=1
while [ "$i" -le "$RETRIES" ]; do
  log "Health check attempt $i/$RETRIES"
  if [ "$DRY_RUN" = false ]; then
    if run_check_script "$i" && run_api_smoke_script "$i"; then
      startup_ok=true
      break
    fi
  else
    log "(dry-run) would run check-baron-api-env.sh and api-smoke.sh"
    startup_ok=true
    break
  fi
  i=$((i + 1))
  if [ "$i" -le "$RETRIES" ]; then
    run_cmd "sleep $SLEEP_BETWEEN"
  fi
done

# 7) Save logs
run_cmd "cd '$BARON_DIR' && $(compose_cmd "logs --no-color") > '$LOG_DIR/baron-compose.log' 2>&1 || true"

if [ "$startup_ok" = false ]; then
  capture_failure_diagnostics "startup-failure"
  log "Startup failed: health checks did not pass after $RETRIES attempt(s)"
  exit 1
fi

log "Startup finished: $(date)"

if [ "$DRY_RUN" = true ]; then
  log "Dry-run mode: no actions were actually performed. Use --auto to run for real."
fi

exit 0