feat: improve Worksmobile tenant sync handling

backend/internal/service/worksmobile_relay_worker.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"log/slog"
+	"sort"
 	"strings"
 	"time"
 )
@@ -53,6 +54,7 @@ func (w *WorksmobileRelayWorker) ProcessOnce(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	jobs = sortWorksmobileReadyJobs(jobs)
 	for _, job := range jobs {
 		if err := w.processJob(ctx, job); err != nil {
 			slog.Warn("Worksmobile relay job failed", "jobID", job.ID, "resourceType", job.ResourceType, "resourceID", job.ResourceID, "error", err)
@@ -62,11 +64,15 @@ func (w *WorksmobileRelayWorker) ProcessOnce(ctx context.Context) error {
 }
 
 func (w *WorksmobileRelayWorker) processJob(ctx context.Context, job domain.WorksmobileOutbox) error {
-	if err := w.repo.MarkProcessing(ctx, job.ID); err != nil {
+	claimed, err := w.repo.MarkProcessing(ctx, job.ID)
+	if err != nil {
 		return err
 	}
+	if !claimed {
+		return nil
+	}
 
-	err := w.dispatch(ctx, job)
+	err = w.dispatch(ctx, job)
 	if err != nil {
 		nextAttempt := time.Now().Add(worksmobileRetryDelay(job.RetryCount))
 		_ = w.repo.MarkFailed(ctx, job.ID, err.Error(), nextAttempt)
@@ -136,6 +142,91 @@ func (w *WorksmobileRelayWorker) dispatch(ctx context.Context, job domain.Worksm
 	}
 }
 
+func sortWorksmobileReadyJobs(jobs []domain.WorksmobileOutbox) []domain.WorksmobileOutbox {
+	sorted := append([]domain.WorksmobileOutbox(nil), jobs...)
+	depthByID := worksmobileOrgUnitDepths(sorted)
+	sort.SliceStable(sorted, func(i, j int) bool {
+		leftClass := worksmobileRelayOrderClass(sorted[i])
+		rightClass := worksmobileRelayOrderClass(sorted[j])
+		if leftClass != rightClass {
+			return leftClass < rightClass
+		}
+		leftDepth := depthByID[sorted[i].ID]
+		rightDepth := depthByID[sorted[j].ID]
+		if leftDepth != rightDepth {
+			return leftDepth < rightDepth
+		}
+		return sorted[i].CreatedAt.Before(sorted[j].CreatedAt)
+	})
+	return sorted
+}
+
+func worksmobileRelayOrderClass(job domain.WorksmobileOutbox) int {
+	if job.ResourceType == domain.WorksmobileResourceOrgUnit && job.Action == domain.WorksmobileActionUpsert {
+		return 0
+	}
+	if job.ResourceType == domain.WorksmobileResourceUser {
+		return 1
+	}
+	return 2
+}
+
+func worksmobileOrgUnitDepths(jobs []domain.WorksmobileOutbox) map[string]int {
+	type orgUnitJob struct {
+		jobID     string
+		parentKey string
+	}
+	byExternalKey := map[string]orgUnitJob{}
+	for _, job := range jobs {
+		externalKey, parentKey := worksmobileOrgUnitExternalKeys(job)
+		if externalKey == "" {
+			continue
+		}
+		byExternalKey[externalKey] = orgUnitJob{jobID: job.ID, parentKey: parentKey}
+	}
+
+	depthByExternalKey := map[string]int{}
+	var depth func(externalKey string, seen map[string]bool) int
+	depth = func(externalKey string, seen map[string]bool) int {
+		if value, ok := depthByExternalKey[externalKey]; ok {
+			return value
+		}
+		job, ok := byExternalKey[externalKey]
+		if !ok || job.parentKey == "" || seen[externalKey] {
+			depthByExternalKey[externalKey] = 0
+			return 0
+		}
+		seen[externalKey] = true
+		value := depth(job.parentKey, seen) + 1
+		delete(seen, externalKey)
+		depthByExternalKey[externalKey] = value
+		return value
+	}
+
+	depthByJobID := map[string]int{}
+	for externalKey, job := range byExternalKey {
+		depthByJobID[job.jobID] = depth(externalKey, map[string]bool{})
+	}
+	return depthByJobID
+}
+
+func worksmobileOrgUnitExternalKeys(job domain.WorksmobileOutbox) (string, string) {
+	if job.ResourceType != domain.WorksmobileResourceOrgUnit || job.Action != domain.WorksmobileActionUpsert {
+		return "", ""
+	}
+	var payload WorksmobileOrgUnitPayload
+	if err := decodeWorksmobileRequest(job.Payload, &payload); err != nil {
+		return "", ""
+	}
+	parentKey := strings.TrimSpace(payload.ParentOrgUnitID)
+	if strings.HasPrefix(parentKey, "externalKey:") {
+		parentKey = strings.TrimSpace(strings.TrimPrefix(parentKey, "externalKey:"))
+	} else {
+		parentKey = ""
+	}
+	return strings.TrimSpace(payload.OrgUnitExternalKey), parentKey
+}
+
 func worksmobileOutboxUserIdentifier(job domain.WorksmobileOutbox) string {
 	userID := stringValue(job.Payload["loginEmail"])
 	if userID == "" {