From 737703683d7808a93bd15281f28bd05d8b9781a9 Mon Sep 17 00:00:00 2001 From: Lectom Date: Mon, 22 Jun 2026 13:19:01 +0900 Subject: [PATCH] Fix WORKS Drive image upload recovery --- .gitea/workflows/image_publish.yml | 48 +++++++++-- scripts/docker-image/upload_works_drive.sh | 83 ++++++++++++++----- .../production_image_workflows_policy_test.sh | 8 ++ ...s_drive_docker_image_upload_policy_test.sh | 72 ++++++++++++++++ 4 files changed, 184 insertions(+), 27 deletions(-) diff --git a/.gitea/workflows/image_publish.yml b/.gitea/workflows/image_publish.yml index 772fc223..5fa9c392 100644 --- a/.gitea/workflows/image_publish.yml +++ b/.gitea/workflows/image_publish.yml @@ -154,6 +154,19 @@ jobs: provenance: false sbom: false + - name: Verify built Docker images before WORKS upload + env: + IMAGE_TAG: ${{ steps.version.outputs.image_tag }} + run: | + set -euo pipefail + + for image in backend userfront adminfront devfront orgfront; do + image_ref="baron_sso/${image}:${IMAGE_TAG}" + echo "Checking built Docker image before WORKS upload: ${image_ref}" + docker image inspect "${image_ref}" >/dev/null + docker image ls "${image_ref}" + done + - name: Resolve WORKS Drive access token env: WORKS_DRIVE_ACCESS_TOKEN_INPUT: ${{ secrets.WORKS_DRIVE_ACCESS_TOKEN }} @@ -233,12 +246,33 @@ jobs: fi done - for image in backend userfront adminfront devfront orgfront; do + images="backend userfront adminfront devfront orgfront" + image_total=5 + image_index=0 + uploaded_images="" + + for image in ${images}; do + image_index=$((image_index + 1)) image_ref="baron_sso/${image}:${IMAGE_TAG}" - DOCKER_IMAGE_REF="${image_ref}" \ - WORKS_DRIVE_DOCKER_IMAGE_DIR="${WORKS_DRIVE_DOCKER_IMAGE_DIR}" \ - WORKS_DRIVE_SHARED_DRIVE_ID="${WORKS_DRIVE_DOCKER_IMAGE_DRIVE_ID}" \ - WORKS_DRIVE_PARENT_FILE_ID="${WORKS_DRIVE_DOCKER_IMAGE_PARENT_FILE_ID:-}" \ - WORKS_DOCKER_IMAGE_ARCHIVE_DIR="${RUNNER_TEMP}/baron-sso-docker-image-upload" \ - scripts/docker-image/upload_works_drive.sh + echo "WORKS image upload ${image_index}/${image_total}: ${image_ref}" + docker image inspect "${image_ref}" >/dev/null + if DOCKER_IMAGE_REF="${image_ref}" \ + WORKS_DRIVE_DOCKER_IMAGE_DIR="${WORKS_DRIVE_DOCKER_IMAGE_DIR}" \ + WORKS_DRIVE_SHARED_DRIVE_ID="${WORKS_DRIVE_DOCKER_IMAGE_DRIVE_ID}" \ + WORKS_DRIVE_PARENT_FILE_ID="${WORKS_DRIVE_DOCKER_IMAGE_PARENT_FILE_ID:-}" \ + WORKS_DOCKER_IMAGE_ARCHIVE_DIR="${RUNNER_TEMP}/baron-sso-docker-image-upload" \ + scripts/docker-image/upload_works_drive.sh; then + uploaded_images="${uploaded_images}${uploaded_images:+ }${image_ref}" + echo "WORKS image upload completed: ${image_ref}" + else + upload_status="$?" + echo "::error::WORKS image upload failed at ${image_index}/${image_total}: ${image_ref}" + echo "Already uploaded images: ${uploaded_images:-none}" + exit "${upload_status}" + fi + done + + echo "Uploaded WORKS image archives:" + for image_ref in ${uploaded_images}; do + echo " - ${image_ref}" done diff --git a/scripts/docker-image/upload_works_drive.sh b/scripts/docker-image/upload_works_drive.sh index 13601b97..be9e7fc1 100755 --- a/scripts/docker-image/upload_works_drive.sh +++ b/scripts/docker-image/upload_works_drive.sh @@ -365,6 +365,27 @@ list_child_folders() { printf '%s\n' "$response_body" } +find_folder_id_in_listing() { + local listing_json="$1" + local folder_name="$2" + local strict_type="${3:-true}" + + jq -er --arg name "$folder_name" --arg strictType "$strict_type" ' + [ + (.files // .children // .items // .data // .contents // [])[] + | select((.fileName // .name // .displayName // .title) == $name) + | select( + $strictType != "true" + or ( + ((.fileType // .type // .resourceType // "") | ascii_downcase) as $type + | ($type == "" or $type == "folder" or $type == "dir" or $type == "directory") + ) + ) + | .fileId // .id + ][0] // empty + ' <<<"$listing_json" 2>/dev/null || true +} + create_child_folder() { local access_token="$1" local endpoint="$2" @@ -382,6 +403,11 @@ create_child_folder() { "$endpoint")" split_curl_response "$response" response_body http_status + if [[ "$http_status" -eq 409 ]]; then + printf 'WORKS_CONFLICT\n' + return 2 + fi + if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then backup_die "WORKS folder create request failed (HTTP $http_status): $(printf '%s' "$response_body" | redact_for_log)" fi @@ -396,35 +422,48 @@ ensure_child_folder() { local children_endpoint local create_folder_endpoint local children_json + local refreshed_children_json local folder_id + local create_status - if [[ -n "$parent_file_id" ]]; then + children_endpoint="$(resolve_target_children_endpoint "$parent_file_id")" + create_folder_endpoint="$(resolve_target_create_folder_endpoint "$parent_file_id")" + backup_log "Checking WORKS folder: parent=${parent_file_id:-root} name=$folder_name" >&2 + if ! children_json="$(list_child_folders "$access_token" "$children_endpoint")"; then + return 1 + fi + folder_id="$(find_folder_id_in_listing "$children_json" "$folder_name" "true")" + + if [[ -n "$folder_id" ]]; then + backup_log "Found existing WORKS folder: $folder_name -> $folder_id" >&2 + printf '%s\n' "$folder_id" + return + fi + + backup_log "Creating WORKS folder: parent=${parent_file_id:-root} name=$folder_name" >&2 + if folder_id="$(create_child_folder "$access_token" "$create_folder_endpoint" "$folder_name")"; then + backup_log "Created WORKS folder: $folder_name -> $folder_id" >&2 + printf '%s\n' "$folder_id" + return + else + create_status="$?" + fi + + if [[ "$create_status" -eq 2 ]]; then + backup_log "WORKS folder already exists, resolving existing folder id: $folder_name" >&2 children_endpoint="$(resolve_target_children_endpoint "$parent_file_id")" - create_folder_endpoint="$(resolve_target_create_folder_endpoint "$parent_file_id")" - if ! children_json="$(list_child_folders "$access_token" "$children_endpoint")"; then + if ! refreshed_children_json="$(list_child_folders "$access_token" "$children_endpoint")"; then return 1 fi - folder_id="$(jq -er --arg name "$folder_name" ' - [ - (.files // .children // .items // [])[] - | select((.fileName // .name) == $name) - | select(((.fileType // .type // "") | ascii_downcase) == "folder") - | .fileId // .id - ][0] // empty - ' <<<"$children_json" 2>/dev/null || true)" - + folder_id="$(find_folder_id_in_listing "$refreshed_children_json" "$folder_name" "false")" if [[ -n "$folder_id" ]]; then + backup_log "Resolved existing WORKS folder after conflict: $folder_name -> $folder_id" >&2 printf '%s\n' "$folder_id" return fi - else - create_folder_endpoint="$(resolve_target_create_folder_endpoint "$parent_file_id")" + backup_die "WORKS folder already exists but its fileId could not be resolved: $folder_name" fi - - if ! folder_id="$(create_child_folder "$access_token" "$create_folder_endpoint" "$folder_name")"; then - return 1 - fi - printf '%s\n' "$folder_id" + return 1 } ensure_folder_path() { @@ -441,9 +480,11 @@ ensure_folder_path() { accumulated_path="${accumulated_path:+$accumulated_path/}$component" cached_folder_id="$(read_cached_folder_id "$accumulated_path")" if [[ -n "$cached_folder_id" ]]; then + backup_log "Using cached WORKS folder: $accumulated_path -> $cached_folder_id" >&2 parent_file_id="$cached_folder_id" continue fi + backup_log "Resolving WORKS folder component: $accumulated_path" >&2 if ! parent_file_id="$(ensure_child_folder "$access_token" "$parent_file_id" "$component")"; then return 1 fi @@ -569,7 +610,9 @@ checksum_file="$artifact_dir/${image_name}.${image_tag}.sha256" manifest_file="$artifact_dir/manifest.${image_tag}.json" upload_report_file="$artifact_dir/works-upload.json" -rm -f "$tar_file" "$archive_file" "$checksum_file" "$manifest_file" "$upload_report_file" +rm -f "$tar_file" "$archive_file" "$checksum_file" "$upload_report_file" + +backup_log "Docker image archive context: image_ref=$image_ref remote_path=$remote_path artifact_dir=$artifact_dir" if [[ -n "$commit_container" ]]; then backup_log "Committing container $commit_container to $image_ref" diff --git a/test/production_image_workflows_policy_test.sh b/test/production_image_workflows_policy_test.sh index ef1868bc..cefb7158 100644 --- a/test/production_image_workflows_policy_test.sh +++ b/test/production_image_workflows_policy_test.sh @@ -44,6 +44,8 @@ grep -Fq "steps.version.outputs.image_tag" "$publish_workflow" \ || fail "publish workflow must use the computed image tag for built image archives." grep -Fq "Upload built images to WORKS Drive archive" "$publish_workflow" \ || fail "publish workflow must archive locally built images to WORKS Drive." +grep -Fq "Verify built Docker images before WORKS upload" "$publish_workflow" \ + || fail "publish workflow must verify all built Docker images before WORKS upload." grep -Fq "scripts/docker-image/upload_works_drive.sh" "$publish_workflow" \ || fail "publish workflow must use the shared WORKS Drive image archive script." grep -Fq "docker/build-push-action@v5" "$publish_workflow" \ @@ -54,6 +56,12 @@ for image in backend userfront adminfront devfront orgfront; do grep -Fq "baron_sso/${image}:" "$publish_workflow" \ || fail "publish workflow must build ${image} image." done +grep -Fq 'docker image inspect "${image_ref}"' "$publish_workflow" \ + || fail "publish workflow must inspect each built Docker image before upload." +grep -Fq 'WORKS image upload ${image_index}/${image_total}: ${image_ref}' "$publish_workflow" \ + || fail "publish workflow must log each WORKS image upload with index and image ref." +grep -Fq 'uploaded_images' "$publish_workflow" \ + || fail "publish workflow must track successfully uploaded image refs for failure diagnostics." grep -Fq "WORKS_DRIVE_ACCESS_TOKEN_INPUT: \${{ secrets.WORKS_DRIVE_ACCESS_TOKEN }}" "$publish_workflow" \ || fail "publish workflow must support direct WORKS Drive access token auth." grep -Fq "WORKS_DRIVE_OAUTH_CLIENT_SECRET: \${{ secrets.WORKS_OAUTH_CLIENT_SECRET }}" "$publish_workflow" \ diff --git a/test/works_drive_docker_image_upload_policy_test.sh b/test/works_drive_docker_image_upload_policy_test.sh index 0d180841..e7e1f564 100755 --- a/test/works_drive_docker_image_upload_policy_test.sh +++ b/test/works_drive_docker_image_upload_policy_test.sh @@ -264,9 +264,81 @@ for image in backend userfront; do "$script" >"$tmp_dir/root-${image}.out" done +root_artifact_dir="$root_archive_dir/baron-sso/v1.2606.ab12" +[[ -f "$root_artifact_dir/backend.v1.2606.ab12.tar.zst" ]] \ + || fail "script must keep the backend image archive after follow-up image uploads." +[[ -f "$root_artifact_dir/userfront.v1.2606.ab12.tar.zst" ]] \ + || fail "script must keep the userfront image archive after follow-up image uploads." +jq -e \ + '.images.backend.archive.file_name == "backend.v1.2606.ab12.tar.zst" + and .images.userfront.archive.file_name == "userfront.v1.2606.ab12.tar.zst"' \ + "$root_artifact_dir/manifest.v1.2606.ab12.json" >/dev/null \ + || fail "manifest must accumulate all uploaded images for the same tag." + root_create_count="$(cat "${root_curl_log}.root-create-count")" [[ "$root_create_count" == "1" ]] || fail "script must reuse the cached root archive folder id across image uploads in the same run." grep -Fq "sharedrives/root-drive/files/root-tag-id" "$root_curl_log" \ || fail "script must upload follow-up images into the cached tag folder." +conflict_curl_log="$tmp_dir/conflict-curl.log" +conflict_fake_curl="$tmp_dir/conflict-fake-curl.sh" +cat >"$conflict_fake_curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${FAKE_CURL_LOG}" +last_arg="${!#}" + +case "$last_arg" in + https://www.worksapis.com/v1.0/sharedrives/conflict-drive/files) + list_count_file="${FAKE_CURL_LOG}.root-list-count" + list_count=0 + [[ -f "$list_count_file" ]] && list_count="$(cat "$list_count_file")" + list_count=$((list_count + 1)) + printf '%s' "$list_count" >"$list_count_file" + if [[ "$list_count" -eq 1 ]]; then + printf '{"files":[]}\n200' + else + printf '{"files":[{"fileId":"conflict-baron-sso-id","fileName":"baron-sso","fileType":"FILE"}]}\n200' + fi + ;; + https://www.worksapis.com/v1.0/sharedrives/conflict-drive/files/createfolder) + printf '{"code":"RESOURCE_ALREADY_EXIST","description":"Resource already exists."}\n409' + ;; + https://www.worksapis.com/v1.0/sharedrives/conflict-drive/files/conflict-baron-sso-id/children) + printf '{"files":[]}\n200' + ;; + https://www.worksapis.com/v1.0/sharedrives/conflict-drive/files/conflict-baron-sso-id/createfolder) + printf '{"fileId":"conflict-tag-id","fileName":"v1.2606.ab12","fileType":"FOLDER"}\n200' + ;; + https://www.worksapis.com/v1.0/sharedrives/conflict-drive/files/conflict-tag-id) + printf '{"uploadUrl":"https://upload.example.test/conflict-docker-image"}\n200' + ;; + https://upload.example.test/conflict-docker-image) + printf '{"fileId":"uploaded-conflict-file-id"}\n200' + ;; + *) + echo "unexpected conflict curl URL: $last_arg" >&2 + exit 2 + ;; +esac +EOF +chmod +x "$conflict_fake_curl" + +FAKE_DOCKER_LOG="$docker_log" \ +FAKE_CURL_LOG="$conflict_curl_log" \ +PATH="$fake_bin:$PATH" \ +WORKS_DRIVE_ACCESS_TOKEN="test-access-token" \ +WORKS_DRIVE_TARGET="sharedrive" \ +WORKS_DRIVE_SHARED_DRIVE_ID="conflict-drive" \ +WORKS_DRIVE_PARENT_FILE_ID="" \ +WORKS_DRIVE_CURL_BIN="$conflict_fake_curl" \ +WORKS_DOCKER_IMAGE_ARCHIVE_DIR="$tmp_dir/conflict-archive" \ +DOCKER_IMAGE_REF="baron_sso/backend:v1.2606.ab12" \ +"$script" >"$tmp_dir/conflict.out" 2>&1 + +grep -Fq "WORKS folder already exists, resolving existing folder id: baron-sso" "$tmp_dir/conflict.out" \ + || fail "script must recover an existing folder id after WORKS createfolder returns 409." +grep -Fq "sharedrives/conflict-drive/files/conflict-tag-id" "$conflict_curl_log" \ + || fail "script must upload into the resolved folder after a create conflict." + echo "OK: WORKS Drive Docker image archive upload flow commits, packages, and uploads image artifacts"