119 lines
3.0 KiB
Go
119 lines
3.0 KiB
Go
package userprogram
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"geoip-rest/internal/importer"
|
|
)
|
|
|
|
type SyncConfig struct {
|
|
MySQL MySQLConfig
|
|
DatabaseURL string
|
|
InitialCSV string
|
|
UpdateDir string
|
|
LogDir string
|
|
Schema string
|
|
Logger *log.Logger
|
|
}
|
|
|
|
func (c *SyncConfig) defaults() {
|
|
if c.InitialCSV == "" {
|
|
c.InitialCSV = DefaultInitialCSV
|
|
}
|
|
if c.UpdateDir == "" {
|
|
c.UpdateDir = DefaultUpdateDir
|
|
}
|
|
if c.LogDir == "" {
|
|
c.LogDir = DefaultLogDir
|
|
}
|
|
if c.Schema == "" {
|
|
c.Schema = DefaultSchema
|
|
}
|
|
if c.Logger == nil {
|
|
c.Logger = log.Default()
|
|
}
|
|
}
|
|
|
|
// Sync ensures replica table exists and imports initial data, then dumps and imports
|
|
// updates using the primary key high-water mark up to yesterday (KST).
|
|
func Sync(ctx context.Context, cfg SyncConfig) error {
|
|
cfg.defaults()
|
|
|
|
dumper, err := NewDumper(cfg.MySQL, cfg.UpdateDir)
|
|
if err != nil {
|
|
return fmt.Errorf("init dumper: %w", err)
|
|
}
|
|
defer dumper.Close()
|
|
|
|
conn, err := pgx.Connect(ctx, cfg.DatabaseURL)
|
|
if err != nil {
|
|
return fmt.Errorf("connect postgres: %w", err)
|
|
}
|
|
defer conn.Close(context.Background())
|
|
|
|
if err := importer.EnsureUserProgramReplica(ctx, conn, cfg.InitialCSV, cfg.Schema, cfg.LogDir); err != nil {
|
|
return fmt.Errorf("ensure replica: %w", err)
|
|
}
|
|
|
|
lastID, err := importer.LatestID(ctx, conn, cfg.Schema, importer.ReplicaTable)
|
|
if err != nil {
|
|
return fmt.Errorf("read latest id: %w", err)
|
|
}
|
|
|
|
endDate := yesterdayKST()
|
|
upperID, err := dumper.MaxIDUntil(ctx, endDate)
|
|
if err != nil {
|
|
return fmt.Errorf("read upstream max id: %w", err)
|
|
}
|
|
|
|
if upperID <= lastID {
|
|
cfg.Logger.Printf("no dump needed (last_id=%d upstream_max=%d)", lastID, upperID)
|
|
return nil
|
|
}
|
|
|
|
cfg.Logger.Printf("dumping ids (%d, %d] to %s", lastID, upperID, cfg.UpdateDir)
|
|
csvPath, err := dumper.DumpRange(ctx, lastID, upperID, endDate)
|
|
if err != nil {
|
|
return fmt.Errorf("dump range: %w", err)
|
|
}
|
|
if csvPath == "" {
|
|
cfg.Logger.Printf("no rows dumped (last_id=%d upstream_max=%d)", lastID, upperID)
|
|
return nil
|
|
}
|
|
|
|
if err := importer.ImportUserProgramUpdates(ctx, conn, csvPath, cfg.Schema, cfg.LogDir); err != nil {
|
|
return fmt.Errorf("import updates: %w", err)
|
|
}
|
|
|
|
cfg.Logger.Printf("sync complete (last_id=%d -> %d)", lastID, upperID)
|
|
|
|
if err := verifyCounts(ctx, cfg, dumper, conn, upperID); err != nil {
|
|
cfg.Logger.Printf("sync verification warning: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func toKST(t time.Time) time.Time {
|
|
return t.In(kst())
|
|
}
|
|
|
|
func verifyCounts(ctx context.Context, cfg SyncConfig, dumper *Dumper, conn *pgx.Conn, upperID int64) error {
|
|
sourceCount, err := dumper.CountUpToID(ctx, upperID)
|
|
if err != nil {
|
|
return fmt.Errorf("source count: %w", err)
|
|
}
|
|
targetCount, err := importer.CountUpToID(ctx, conn, cfg.Schema, importer.ReplicaTable, upperID)
|
|
if err != nil {
|
|
return fmt.Errorf("target count: %w", err)
|
|
}
|
|
if targetCount != sourceCount {
|
|
return fmt.Errorf("count mismatch up to id %d (source=%d target=%d)", upperID, sourceCount, targetCount)
|
|
}
|
|
return nil
|
|
}
|