diff --git a/AGENTS.md b/AGENTS.md index 0d6bf7f..6b6d1fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ - Keep `cmd/server` thin; place new logic in `internal/` with clear boundaries. ## Build, Test, and Development Commands -- `PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker. +- `SERVICE_PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker. - `docker compose up --build` builds and starts the containerized service (mounts the local database). - `curl "http://localhost:8080/lookup?ip=1.1.1.1"` exercises the lookup endpoint; omit `ip` to use the caller’s address. - `go build ./...` validates compilation before pushing changes. diff --git a/Dockerfile b/Dockerfile index 1055fae..e801839 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,14 +10,12 @@ RUN go mod download COPY . . RUN CGO_ENABLED=0 go build -o /bin/geoip ./cmd/server && \ CGO_ENABLED=0 go build -o /bin/geoip-loader ./cmd/loader && \ - CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import + CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import && \ + CGO_ENABLED=0 go build -o /bin/user-program-dump ./cmd/user_program_dump && \ + CGO_ENABLED=0 go build -o /bin/user-program-sync ./cmd/user_program_sync FROM debian:trixie-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends mysql-client && \ - rm -rf /var/lib/apt/lists/* - RUN useradd --create-home --shell /usr/sbin/nologin appuser WORKDIR /app @@ -25,11 +23,10 @@ WORKDIR /app COPY --from=builder /bin/geoip /usr/local/bin/geoip COPY --from=builder /bin/geoip-loader /usr/local/bin/geoip-loader COPY --from=builder /bin/user-program-import /usr/local/bin/user-program-import +COPY --from=builder /bin/user-program-dump /usr/local/bin/user-program-dump +COPY --from=builder /bin/user-program-sync /usr/local/bin/user-program-sync COPY initial_data /app/initial_data -COPY scripts /app/scripts -RUN mkdir -p /app/update_data /app/log && \ - chmod 0755 /app/scripts/dump_and_import.sh && \ - chmod -R 0755 /app/scripts +RUN mkdir -p /app/update_data ENV GEOIP_DB_PATH=/app/initial_data/GeoLite2-City.mmdb USER appuser diff --git a/cmd/server/main.go b/cmd/server/main.go index da4858a..596af90 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -6,6 +6,7 @@ import ( "log" "net/url" "os" + "strings" "time" "github.com/gofiber/fiber/v2" @@ -17,8 +18,8 @@ import ( const ( defaultPort = "8080" defaultDBPath = "/initial_data/GeoLite2-City.mmdb" - defaultCron = "" - defaultScript = "./scripts/dump_and_import.sh" + defaultCron = "5 0 * * *" // 매일 00:05 KST + defaultJob = "user-program-sync" ) func main() { @@ -26,7 +27,7 @@ func main() { dbPath := env("GEOIP_DB_PATH", defaultDBPath) dbURL := os.Getenv("DATABASE_URL") lookupQuery := os.Getenv("GEOIP_LOOKUP_QUERY") - port := env("PORT", defaultPort) + port := env("SERVICE_PORT", defaultPort) resolver, err := geo.NewResolver(geo.Config{ Backend: backend, @@ -112,6 +113,21 @@ func env(key, fallback string) string { return fallback } +func envBool(key string, fallback bool) bool { + val := os.Getenv(key) + if val == "" { + return fallback + } + switch strings.ToLower(val) { + case "1", "t", "true", "y", "yes", "on": + return true + case "0", "f", "false", "n", "no", "off": + return false + default: + return fallback + } +} + func sanitizeDBURL(raw string) string { u, err := url.Parse(raw) if err != nil { @@ -121,15 +137,16 @@ func sanitizeDBURL(raw string) string { } func maybeStartScheduler() func() context.Context { - cronExpr := env("USER_PROGRAM_CRON", defaultCron) - if cronExpr == "" { + enabled := envBool("USER_PROGRAM_CRON_ENABLE", false) + if !enabled { return nil } - script := env("USER_PROGRAM_SCRIPT", defaultScript) + cronExpr := defaultCron + command := defaultJob sched, err := schedule.Start(schedule.Config{ CronExpr: cronExpr, - ScriptPath: script, + Command: command, Logger: log.Default(), }) if err != nil { diff --git a/cmd/user_program_dump/main.go b/cmd/user_program_dump/main.go new file mode 100644 index 0000000..eb60a2d --- /dev/null +++ b/cmd/user_program_dump/main.go @@ -0,0 +1,295 @@ +package main + +import ( + "context" + "database/sql" + "encoding/csv" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "github.com/go-sql-driver/mysql" +) + +const ( + defaultUpdateDir = "./update_data" + defaultTable = "user_program_info" + defaultDB = "user_program_info" + defaultDumpTimeout = 5 * time.Minute +) + +type config struct { + host string + port int + user string + password string + database string + table string + updateDir string + target time.Time +} + +func main() { + logger := log.New(os.Stdout, "[dump] ", log.LstdFlags) + + cfg, err := loadConfig() + if err != nil { + log.Fatalf("config error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultDumpTimeout) + defer cancel() + + dsn := (&mysql.Config{ + User: cfg.user, + Passwd: cfg.password, + Net: "tcp", + Addr: netAddr(cfg.host, cfg.port), + DBName: cfg.database, + Params: map[string]string{"parseTime": "true", "loc": "UTC", "charset": "utf8mb4"}, + AllowNativePasswords: true, + }).FormatDSN() + + db, err := sql.Open("mysql", dsn) + if err != nil { + log.Fatalf("failed to open mysql connection: %v", err) + } + defer db.Close() + + if err := db.PingContext(ctx); err != nil { + log.Fatalf("failed to ping mysql: %v", err) + } + + if _, err := db.ExecContext(ctx, "SET time_zone = '+00:00'"); err != nil { + log.Fatalf("failed to set timezone: %v", err) + } + + outPath, err := dumpToCSV(ctx, db, cfg) + if err != nil { + log.Fatalf("dump failed: %v", err) + } + + logger.Printf("dumped %s to %s", cfg.target.Format("2006-01-02"), outPath) +} + +func loadConfig() (config, error) { + port, err := strconv.Atoi(env("USER_PROGRAM_INFO_PORT", "3306")) + if err != nil { + return config{}, fmt.Errorf("invalid USER_PROGRAM_INFO_PORT: %w", err) + } + + target, err := resolveTargetDate(env("USER_PROGRAM_TARGET_DATE", "")) + if err != nil { + return config{}, err + } + + table := env("USER_PROGRAM_INFO_TABLE", defaultTable) + if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) { + return config{}, fmt.Errorf("invalid table name: %s", table) + } + + updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir) + if updateDir == "" { + updateDir = defaultUpdateDir + } + if err := os.MkdirAll(updateDir, 0o755); err != nil { + return config{}, fmt.Errorf("creating update dir: %w", err) + } + + return config{ + host: envRequired("USER_PROGRAM_INFO_HOST"), + port: port, + user: envRequired("USER_PROGRAM_INFO_USERNAME"), + password: envRequired("USER_PROGRAM_INFO_PASSWORD"), + database: env("USER_PROGRAM_INFO_DB", defaultDB), + table: table, + updateDir: updateDir, + target: target, + }, nil +} + +func dumpToCSV(ctx context.Context, db *sql.DB, cfg config) (string, error) { + query := fmt.Sprintf(` +SELECT + id, + product_name, + login_id, + user_employee_id, + login_version, + login_public_ip, + login_local_ip, + user_company, + user_department, + user_position, + user_login_time, + created_at, + user_family_flag +FROM %s +WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = ?;`, cfg.table) + + rows, err := db.QueryContext(ctx, query, cfg.target.Format("2006-01-02")) + if err != nil { + return "", err + } + defer rows.Close() + + filename := fmt.Sprintf("user_program_info_%s.csv", cfg.target.Format("20060102")) + outPath := filepath.Join(cfg.updateDir, filename) + tmpPath := outPath + ".tmp" + + f, err := os.Create(tmpPath) + if err != nil { + return "", err + } + defer f.Close() + + writer := csv.NewWriter(f) + defer writer.Flush() + + header := []string{ + "id", + "product_name", + "login_id", + "user_employee_id", + "login_version", + "login_public_ip", + "login_local_ip", + "user_company", + "user_department", + "user_position", + "user_login_time", + "created_at", + "user_family_flag", + } + if err := writer.Write(header); err != nil { + return "", err + } + + for rows.Next() { + record, err := scanRow(rows) + if err != nil { + return "", err + } + if err := writer.Write(record); err != nil { + return "", err + } + } + if err := rows.Err(); err != nil { + return "", err + } + writer.Flush() + if err := writer.Error(); err != nil { + return "", err + } + + if err := os.Rename(tmpPath, outPath); err != nil { + return "", err + } + return outPath, nil +} + +func scanRow(rows *sql.Rows) ([]string, error) { + var ( + id sql.NullInt64 + productName sql.NullString + loginID sql.NullString + employeeID sql.NullString + loginVersion sql.NullString + loginPublicIP sql.NullString + loginLocalIP sql.NullString + userCompany sql.NullString + userDepartment sql.NullString + userPosition sql.NullString + userLoginTime sql.NullString + createdAt sql.NullString + userFamilyFlag sql.NullString + ) + + if err := rows.Scan( + &id, + &productName, + &loginID, + &employeeID, + &loginVersion, + &loginPublicIP, + &loginLocalIP, + &userCompany, + &userDepartment, + &userPosition, + &userLoginTime, + &createdAt, + &userFamilyFlag, + ); err != nil { + return nil, err + } + if !id.Valid { + return nil, fmt.Errorf("row missing id") + } + + return []string{ + strconv.FormatInt(id.Int64, 10), + nullToString(productName), + nullToString(loginID), + nullToString(employeeID), + nullToString(loginVersion), + nullToString(loginPublicIP), + nullToString(loginLocalIP), + nullToString(userCompany), + nullToString(userDepartment), + nullToString(userPosition), + nullToString(userLoginTime), + nullToString(createdAt), + nullToString(userFamilyFlag), + }, nil +} + +func resolveTargetDate(raw string) (time.Time, error) { + if raw == "" { + now := time.Now().In(kst()) + yesterday := now.AddDate(0, 0, -1) + return time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, kst()), nil + } + t, err := time.ParseInLocation("2006-01-02", raw, kst()) + if err != nil { + return time.Time{}, fmt.Errorf("invalid USER_PROGRAM_TARGET_DATE: %w", err) + } + return t, nil +} + +func kst() *time.Location { + loc, err := time.LoadLocation("Asia/Seoul") + if err != nil { + return time.FixedZone("KST", 9*60*60) + } + return loc +} + +func env(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envRequired(key string) string { + v := os.Getenv(key) + if v == "" { + log.Fatalf("%s is required", key) + } + return v +} + +func netAddr(host string, port int) string { + return fmt.Sprintf("%s:%d", host, port) +} + +func nullToString(v sql.NullString) string { + if v.Valid { + return v.String + } + return "" +} diff --git a/cmd/user_program_sync/main.go b/cmd/user_program_sync/main.go new file mode 100644 index 0000000..bcdc509 --- /dev/null +++ b/cmd/user_program_sync/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "log" + "os" + "os/exec" + "path/filepath" + "time" +) + +const ( + defaultUpdateDir = "/app/update_data" + defaultLogDir = "/app/log" + defaultSchema = "public" + defaultTimeout = 15 * time.Minute +) + +func main() { + logger := log.New(os.Stdout, "[sync] ", log.LstdFlags) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + + updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir) + logDir := env("USER_PROGRAM_IMPORT_LOG_DIR", defaultLogDir) + schema := env("USER_PROGRAM_INFO_SCHEMA", defaultSchema) + + ensureDir(updateDir, logger) + ensureDir(logDir, logger) + + if err := runCmd(ctx, logger, "user-program-dump", map[string]string{ + "USER_PROGRAM_UPDATE_DIR": updateDir, + }); err != nil { + logger.Fatalf("dump failed: %v", err) + } + + if err := runCmd(ctx, logger, "user-program-import", map[string]string{ + "USER_PROGRAM_UPDATE_DIR": updateDir, + "USER_PROGRAM_IMPORT_LOG_DIR": logDir, + "USER_PROGRAM_INFO_SCHEMA": schema, + }); err != nil { + logger.Fatalf("import failed: %v", err) + } + + logger.Printf("sync completed (update_dir=%s, log_dir=%s, schema=%s)", updateDir, logDir, schema) +} + +func runCmd(ctx context.Context, logger *log.Logger, command string, extraEnv map[string]string) error { + cmd := exec.CommandContext(ctx, command) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + for k, v := range extraEnv { + if v == "" { + continue + } + cmd.Env = append(cmd.Env, k+"="+v) + } + logger.Printf("running %s", filepath.Base(command)) + return cmd.Run() +} + +func ensureDir(path string, logger *log.Logger) { + if err := os.MkdirAll(path, 0o755); err != nil { + logger.Fatalf("failed to create dir %s: %v", path, err) + } +} + +func env(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} diff --git a/docker-compose.yml b/docker-compose.yml index a389340..17a2c50 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,9 @@ services: db: condition: service_healthy ports: - - "${PORT:-8080}:8080" + - "${SERVICE_PORT:-8080}:8080" environment: - - PORT=${PORT:-8080} + - SERVICE_PORT=${SERVICE_PORT:-8080} - GEOIP_DB_PATH=${GEOIP_DB_PATH:-/app/initial_data/GeoLite2-City.mmdb} - GEOIP_BACKEND=${GEOIP_BACKEND:-mmdb} - GEOIP_LOADER_TIMEOUT=${GEOIP_LOADER_TIMEOUT:-30m} diff --git a/go.mod b/go.mod index db4758c..6cb14bc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module geoip-rest go 1.25 require ( + github.com/go-sql-driver/mysql v1.8.1 github.com/gofiber/fiber/v2 v2.52.8 github.com/jackc/pgx/v5 v5.7.6 github.com/oschwald/geoip2-golang v1.9.0 @@ -11,6 +12,7 @@ require ( ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7320d0b..89363a5 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,12 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4= github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/internal/schedule/scheduler.go b/internal/schedule/scheduler.go index d011b67..dcfe49c 100644 --- a/internal/schedule/scheduler.go +++ b/internal/schedule/scheduler.go @@ -12,9 +12,10 @@ import ( ) type Config struct { - CronExpr string - ScriptPath string - Logger *log.Logger + CronExpr string + Command string + Args []string + Logger *log.Logger } type Scheduler struct { @@ -29,18 +30,14 @@ func Start(cfg Config) (*Scheduler, error) { if cfg.CronExpr == "" { return nil, errors.New("CronExpr is required") } - if cfg.ScriptPath == "" { - return nil, errors.New("ScriptPath is required") + if cfg.Command == "" { + return nil, errors.New("Command is required") } if cfg.Logger == nil { cfg.Logger = log.Default() } - if _, err := os.Stat(cfg.ScriptPath); err != nil { - return nil, err - } - kst, err := time.LoadLocation("Asia/Seoul") if err != nil { kst = time.FixedZone("KST", 9*60*60) @@ -54,12 +51,12 @@ func Start(cfg Config) (*Scheduler, error) { c := cron.New(cron.WithLocation(kst), cron.WithParser(parser)) c.Schedule(spec, cron.FuncJob(func() { - runScript(cfg.Logger, cfg.ScriptPath) + runCommand(cfg.Logger, cfg.Command, cfg.Args...) })) c.Start() - cfg.Logger.Printf("scheduler started with cron=%s script=%s tz=%s", cfg.CronExpr, cfg.ScriptPath, kst) + cfg.Logger.Printf("scheduler started with cron=%s command=%s args=%v tz=%s", cfg.CronExpr, cfg.Command, cfg.Args, kst) return &Scheduler{ cron: c, @@ -75,11 +72,11 @@ func (s *Scheduler) Stop() context.Context { return s.cron.Stop() } -func runScript(logger *log.Logger, script string) { +func runCommand(logger *log.Logger, command string, args ...string) { start := time.Now() - logger.Printf("scheduler: running %s", script) + logger.Printf("scheduler: running %s %v", command, args) - cmd := exec.Command("/bin/bash", script) + cmd := exec.Command(command, args...) cmd.Env = os.Environ() out, err := cmd.CombinedOutput() duration := time.Since(start) @@ -88,8 +85,8 @@ func runScript(logger *log.Logger, script string) { logger.Printf("scheduler: output:\n%s", string(out)) } if err != nil { - logger.Printf("scheduler: %s failed after %s: %v", script, duration, err) + logger.Printf("scheduler: %s failed after %s: %v", command, duration, err) return } - logger.Printf("scheduler: %s completed in %s", script, duration) + logger.Printf("scheduler: %s completed in %s", command, duration) } diff --git a/scripts/dump_and_import.sh b/scripts/dump_and_import.sh deleted file mode 100644 index d82b647..0000000 --- a/scripts/dump_and_import.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -LOG_DIR="${USER_PROGRAM_IMPORT_LOG_DIR:-/app/log}" -UPDATE_DIR="${USER_PROGRAM_UPDATE_DIR:-/app/update_data}" -SCHEMA="${USER_PROGRAM_INFO_SCHEMA:-public}" -CSV_DIR="${USER_PROGRAM_INFO_CSV_DIR:-/app/initial_data}" - -MYSQL_HOST="${USER_PROGRAM_INFO_HOST:?USER_PROGRAM_INFO_HOST is required}" -MYSQL_PORT="${USER_PROGRAM_INFO_PORT:-3306}" -MYSQL_USER="${USER_PROGRAM_INFO_USERNAME:?USER_PROGRAM_INFO_USERNAME is required}" -MYSQL_PASS="${USER_PROGRAM_INFO_PASSWORD:?USER_PROGRAM_INFO_PASSWORD is required}" -MYSQL_DB="${USER_PROGRAM_INFO_DB:-user_program_info}" -MYSQL_TABLE="${USER_PROGRAM_INFO_TABLE:-user_program_info}" - -mkdir -p "${LOG_DIR}" "${UPDATE_DIR}" - -# Target date: yesterday in KST unless USER_PROGRAM_TARGET_DATE=YYYY-MM-DD is provided. -TARGET_DATE="${USER_PROGRAM_TARGET_DATE:-$(TZ=Asia/Seoul date -d 'yesterday' +%Y-%m-%d)}" -TARGET_DATE_COMPACT="${TARGET_DATE//-/}" -OUT_FILE="${UPDATE_DIR}/user_program_info_${TARGET_DATE_COMPACT}.csv" -TMP_FILE="${OUT_FILE}.tmp" - -QUERY=$(cat <