크론구조 개선
This commit is contained in:
@@ -7,7 +7,7 @@
|
|||||||
- Keep `cmd/server` thin; place new logic in `internal/<domain>` with clear boundaries.
|
- Keep `cmd/server` thin; place new logic in `internal/<domain>` with clear boundaries.
|
||||||
|
|
||||||
## Build, Test, and Development Commands
|
## Build, Test, and Development Commands
|
||||||
- `PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker.
|
- `SERVICE_PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker.
|
||||||
- `docker compose up --build` builds and starts the containerized service (mounts the local database).
|
- `docker compose up --build` builds and starts the containerized service (mounts the local database).
|
||||||
- `curl "http://localhost:8080/lookup?ip=1.1.1.1"` exercises the lookup endpoint; omit `ip` to use the caller’s address.
|
- `curl "http://localhost:8080/lookup?ip=1.1.1.1"` exercises the lookup endpoint; omit `ip` to use the caller’s address.
|
||||||
- `go build ./...` validates compilation before pushing changes.
|
- `go build ./...` validates compilation before pushing changes.
|
||||||
|
|||||||
15
Dockerfile
15
Dockerfile
@@ -10,14 +10,12 @@ RUN go mod download
|
|||||||
COPY . .
|
COPY . .
|
||||||
RUN CGO_ENABLED=0 go build -o /bin/geoip ./cmd/server && \
|
RUN CGO_ENABLED=0 go build -o /bin/geoip ./cmd/server && \
|
||||||
CGO_ENABLED=0 go build -o /bin/geoip-loader ./cmd/loader && \
|
CGO_ENABLED=0 go build -o /bin/geoip-loader ./cmd/loader && \
|
||||||
CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import
|
CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import && \
|
||||||
|
CGO_ENABLED=0 go build -o /bin/user-program-dump ./cmd/user_program_dump && \
|
||||||
|
CGO_ENABLED=0 go build -o /bin/user-program-sync ./cmd/user_program_sync
|
||||||
|
|
||||||
FROM debian:trixie-slim
|
FROM debian:trixie-slim
|
||||||
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install -y --no-install-recommends mysql-client && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
@@ -25,11 +23,10 @@ WORKDIR /app
|
|||||||
COPY --from=builder /bin/geoip /usr/local/bin/geoip
|
COPY --from=builder /bin/geoip /usr/local/bin/geoip
|
||||||
COPY --from=builder /bin/geoip-loader /usr/local/bin/geoip-loader
|
COPY --from=builder /bin/geoip-loader /usr/local/bin/geoip-loader
|
||||||
COPY --from=builder /bin/user-program-import /usr/local/bin/user-program-import
|
COPY --from=builder /bin/user-program-import /usr/local/bin/user-program-import
|
||||||
|
COPY --from=builder /bin/user-program-dump /usr/local/bin/user-program-dump
|
||||||
|
COPY --from=builder /bin/user-program-sync /usr/local/bin/user-program-sync
|
||||||
COPY initial_data /app/initial_data
|
COPY initial_data /app/initial_data
|
||||||
COPY scripts /app/scripts
|
RUN mkdir -p /app/update_data
|
||||||
RUN mkdir -p /app/update_data /app/log && \
|
|
||||||
chmod 0755 /app/scripts/dump_and_import.sh && \
|
|
||||||
chmod -R 0755 /app/scripts
|
|
||||||
|
|
||||||
ENV GEOIP_DB_PATH=/app/initial_data/GeoLite2-City.mmdb
|
ENV GEOIP_DB_PATH=/app/initial_data/GeoLite2-City.mmdb
|
||||||
USER appuser
|
USER appuser
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gofiber/fiber/v2"
|
"github.com/gofiber/fiber/v2"
|
||||||
@@ -17,8 +18,8 @@ import (
|
|||||||
const (
|
const (
|
||||||
defaultPort = "8080"
|
defaultPort = "8080"
|
||||||
defaultDBPath = "/initial_data/GeoLite2-City.mmdb"
|
defaultDBPath = "/initial_data/GeoLite2-City.mmdb"
|
||||||
defaultCron = ""
|
defaultCron = "5 0 * * *" // 매일 00:05 KST
|
||||||
defaultScript = "./scripts/dump_and_import.sh"
|
defaultJob = "user-program-sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -26,7 +27,7 @@ func main() {
|
|||||||
dbPath := env("GEOIP_DB_PATH", defaultDBPath)
|
dbPath := env("GEOIP_DB_PATH", defaultDBPath)
|
||||||
dbURL := os.Getenv("DATABASE_URL")
|
dbURL := os.Getenv("DATABASE_URL")
|
||||||
lookupQuery := os.Getenv("GEOIP_LOOKUP_QUERY")
|
lookupQuery := os.Getenv("GEOIP_LOOKUP_QUERY")
|
||||||
port := env("PORT", defaultPort)
|
port := env("SERVICE_PORT", defaultPort)
|
||||||
|
|
||||||
resolver, err := geo.NewResolver(geo.Config{
|
resolver, err := geo.NewResolver(geo.Config{
|
||||||
Backend: backend,
|
Backend: backend,
|
||||||
@@ -112,6 +113,21 @@ func env(key, fallback string) string {
|
|||||||
return fallback
|
return fallback
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func envBool(key string, fallback bool) bool {
|
||||||
|
val := os.Getenv(key)
|
||||||
|
if val == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
switch strings.ToLower(val) {
|
||||||
|
case "1", "t", "true", "y", "yes", "on":
|
||||||
|
return true
|
||||||
|
case "0", "f", "false", "n", "no", "off":
|
||||||
|
return false
|
||||||
|
default:
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func sanitizeDBURL(raw string) string {
|
func sanitizeDBURL(raw string) string {
|
||||||
u, err := url.Parse(raw)
|
u, err := url.Parse(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -121,15 +137,16 @@ func sanitizeDBURL(raw string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func maybeStartScheduler() func() context.Context {
|
func maybeStartScheduler() func() context.Context {
|
||||||
cronExpr := env("USER_PROGRAM_CRON", defaultCron)
|
enabled := envBool("USER_PROGRAM_CRON_ENABLE", false)
|
||||||
if cronExpr == "" {
|
if !enabled {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
script := env("USER_PROGRAM_SCRIPT", defaultScript)
|
cronExpr := defaultCron
|
||||||
|
command := defaultJob
|
||||||
|
|
||||||
sched, err := schedule.Start(schedule.Config{
|
sched, err := schedule.Start(schedule.Config{
|
||||||
CronExpr: cronExpr,
|
CronExpr: cronExpr,
|
||||||
ScriptPath: script,
|
Command: command,
|
||||||
Logger: log.Default(),
|
Logger: log.Default(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
295
cmd/user_program_dump/main.go
Normal file
295
cmd/user_program_dump/main.go
Normal file
@@ -0,0 +1,295 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/csv"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-sql-driver/mysql"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultUpdateDir = "./update_data"
|
||||||
|
defaultTable = "user_program_info"
|
||||||
|
defaultDB = "user_program_info"
|
||||||
|
defaultDumpTimeout = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
host string
|
||||||
|
port int
|
||||||
|
user string
|
||||||
|
password string
|
||||||
|
database string
|
||||||
|
table string
|
||||||
|
updateDir string
|
||||||
|
target time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := log.New(os.Stdout, "[dump] ", log.LstdFlags)
|
||||||
|
|
||||||
|
cfg, err := loadConfig()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("config error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), defaultDumpTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
dsn := (&mysql.Config{
|
||||||
|
User: cfg.user,
|
||||||
|
Passwd: cfg.password,
|
||||||
|
Net: "tcp",
|
||||||
|
Addr: netAddr(cfg.host, cfg.port),
|
||||||
|
DBName: cfg.database,
|
||||||
|
Params: map[string]string{"parseTime": "true", "loc": "UTC", "charset": "utf8mb4"},
|
||||||
|
AllowNativePasswords: true,
|
||||||
|
}).FormatDSN()
|
||||||
|
|
||||||
|
db, err := sql.Open("mysql", dsn)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("failed to open mysql connection: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
if err := db.PingContext(ctx); err != nil {
|
||||||
|
log.Fatalf("failed to ping mysql: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := db.ExecContext(ctx, "SET time_zone = '+00:00'"); err != nil {
|
||||||
|
log.Fatalf("failed to set timezone: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outPath, err := dumpToCSV(ctx, db, cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("dump failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Printf("dumped %s to %s", cfg.target.Format("2006-01-02"), outPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadConfig() (config, error) {
|
||||||
|
port, err := strconv.Atoi(env("USER_PROGRAM_INFO_PORT", "3306"))
|
||||||
|
if err != nil {
|
||||||
|
return config{}, fmt.Errorf("invalid USER_PROGRAM_INFO_PORT: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
target, err := resolveTargetDate(env("USER_PROGRAM_TARGET_DATE", ""))
|
||||||
|
if err != nil {
|
||||||
|
return config{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
table := env("USER_PROGRAM_INFO_TABLE", defaultTable)
|
||||||
|
if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) {
|
||||||
|
return config{}, fmt.Errorf("invalid table name: %s", table)
|
||||||
|
}
|
||||||
|
|
||||||
|
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
|
||||||
|
if updateDir == "" {
|
||||||
|
updateDir = defaultUpdateDir
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(updateDir, 0o755); err != nil {
|
||||||
|
return config{}, fmt.Errorf("creating update dir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return config{
|
||||||
|
host: envRequired("USER_PROGRAM_INFO_HOST"),
|
||||||
|
port: port,
|
||||||
|
user: envRequired("USER_PROGRAM_INFO_USERNAME"),
|
||||||
|
password: envRequired("USER_PROGRAM_INFO_PASSWORD"),
|
||||||
|
database: env("USER_PROGRAM_INFO_DB", defaultDB),
|
||||||
|
table: table,
|
||||||
|
updateDir: updateDir,
|
||||||
|
target: target,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dumpToCSV(ctx context.Context, db *sql.DB, cfg config) (string, error) {
|
||||||
|
query := fmt.Sprintf(`
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
product_name,
|
||||||
|
login_id,
|
||||||
|
user_employee_id,
|
||||||
|
login_version,
|
||||||
|
login_public_ip,
|
||||||
|
login_local_ip,
|
||||||
|
user_company,
|
||||||
|
user_department,
|
||||||
|
user_position,
|
||||||
|
user_login_time,
|
||||||
|
created_at,
|
||||||
|
user_family_flag
|
||||||
|
FROM %s
|
||||||
|
WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = ?;`, cfg.table)
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(ctx, query, cfg.target.Format("2006-01-02"))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("user_program_info_%s.csv", cfg.target.Format("20060102"))
|
||||||
|
outPath := filepath.Join(cfg.updateDir, filename)
|
||||||
|
tmpPath := outPath + ".tmp"
|
||||||
|
|
||||||
|
f, err := os.Create(tmpPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
writer := csv.NewWriter(f)
|
||||||
|
defer writer.Flush()
|
||||||
|
|
||||||
|
header := []string{
|
||||||
|
"id",
|
||||||
|
"product_name",
|
||||||
|
"login_id",
|
||||||
|
"user_employee_id",
|
||||||
|
"login_version",
|
||||||
|
"login_public_ip",
|
||||||
|
"login_local_ip",
|
||||||
|
"user_company",
|
||||||
|
"user_department",
|
||||||
|
"user_position",
|
||||||
|
"user_login_time",
|
||||||
|
"created_at",
|
||||||
|
"user_family_flag",
|
||||||
|
}
|
||||||
|
if err := writer.Write(header); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
record, err := scanRow(rows)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := writer.Write(record); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
writer.Flush()
|
||||||
|
if err := writer.Error(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(tmpPath, outPath); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return outPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanRow(rows *sql.Rows) ([]string, error) {
|
||||||
|
var (
|
||||||
|
id sql.NullInt64
|
||||||
|
productName sql.NullString
|
||||||
|
loginID sql.NullString
|
||||||
|
employeeID sql.NullString
|
||||||
|
loginVersion sql.NullString
|
||||||
|
loginPublicIP sql.NullString
|
||||||
|
loginLocalIP sql.NullString
|
||||||
|
userCompany sql.NullString
|
||||||
|
userDepartment sql.NullString
|
||||||
|
userPosition sql.NullString
|
||||||
|
userLoginTime sql.NullString
|
||||||
|
createdAt sql.NullString
|
||||||
|
userFamilyFlag sql.NullString
|
||||||
|
)
|
||||||
|
|
||||||
|
if err := rows.Scan(
|
||||||
|
&id,
|
||||||
|
&productName,
|
||||||
|
&loginID,
|
||||||
|
&employeeID,
|
||||||
|
&loginVersion,
|
||||||
|
&loginPublicIP,
|
||||||
|
&loginLocalIP,
|
||||||
|
&userCompany,
|
||||||
|
&userDepartment,
|
||||||
|
&userPosition,
|
||||||
|
&userLoginTime,
|
||||||
|
&createdAt,
|
||||||
|
&userFamilyFlag,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !id.Valid {
|
||||||
|
return nil, fmt.Errorf("row missing id")
|
||||||
|
}
|
||||||
|
|
||||||
|
return []string{
|
||||||
|
strconv.FormatInt(id.Int64, 10),
|
||||||
|
nullToString(productName),
|
||||||
|
nullToString(loginID),
|
||||||
|
nullToString(employeeID),
|
||||||
|
nullToString(loginVersion),
|
||||||
|
nullToString(loginPublicIP),
|
||||||
|
nullToString(loginLocalIP),
|
||||||
|
nullToString(userCompany),
|
||||||
|
nullToString(userDepartment),
|
||||||
|
nullToString(userPosition),
|
||||||
|
nullToString(userLoginTime),
|
||||||
|
nullToString(createdAt),
|
||||||
|
nullToString(userFamilyFlag),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveTargetDate(raw string) (time.Time, error) {
|
||||||
|
if raw == "" {
|
||||||
|
now := time.Now().In(kst())
|
||||||
|
yesterday := now.AddDate(0, 0, -1)
|
||||||
|
return time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, kst()), nil
|
||||||
|
}
|
||||||
|
t, err := time.ParseInLocation("2006-01-02", raw, kst())
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("invalid USER_PROGRAM_TARGET_DATE: %w", err)
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kst() *time.Location {
|
||||||
|
loc, err := time.LoadLocation("Asia/Seoul")
|
||||||
|
if err != nil {
|
||||||
|
return time.FixedZone("KST", 9*60*60)
|
||||||
|
}
|
||||||
|
return loc
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key, fallback string) string {
|
||||||
|
if v := os.Getenv(key); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func envRequired(key string) string {
|
||||||
|
v := os.Getenv(key)
|
||||||
|
if v == "" {
|
||||||
|
log.Fatalf("%s is required", key)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func netAddr(host string, port int) string {
|
||||||
|
return fmt.Sprintf("%s:%d", host, port)
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullToString(v sql.NullString) string {
|
||||||
|
if v.Valid {
|
||||||
|
return v.String
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
75
cmd/user_program_sync/main.go
Normal file
75
cmd/user_program_sync/main.go
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultUpdateDir = "/app/update_data"
|
||||||
|
defaultLogDir = "/app/log"
|
||||||
|
defaultSchema = "public"
|
||||||
|
defaultTimeout = 15 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := log.New(os.Stdout, "[sync] ", log.LstdFlags)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
|
||||||
|
logDir := env("USER_PROGRAM_IMPORT_LOG_DIR", defaultLogDir)
|
||||||
|
schema := env("USER_PROGRAM_INFO_SCHEMA", defaultSchema)
|
||||||
|
|
||||||
|
ensureDir(updateDir, logger)
|
||||||
|
ensureDir(logDir, logger)
|
||||||
|
|
||||||
|
if err := runCmd(ctx, logger, "user-program-dump", map[string]string{
|
||||||
|
"USER_PROGRAM_UPDATE_DIR": updateDir,
|
||||||
|
}); err != nil {
|
||||||
|
logger.Fatalf("dump failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := runCmd(ctx, logger, "user-program-import", map[string]string{
|
||||||
|
"USER_PROGRAM_UPDATE_DIR": updateDir,
|
||||||
|
"USER_PROGRAM_IMPORT_LOG_DIR": logDir,
|
||||||
|
"USER_PROGRAM_INFO_SCHEMA": schema,
|
||||||
|
}); err != nil {
|
||||||
|
logger.Fatalf("import failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Printf("sync completed (update_dir=%s, log_dir=%s, schema=%s)", updateDir, logDir, schema)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCmd(ctx context.Context, logger *log.Logger, command string, extraEnv map[string]string) error {
|
||||||
|
cmd := exec.CommandContext(ctx, command)
|
||||||
|
cmd.Stdout = os.Stdout
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
cmd.Env = os.Environ()
|
||||||
|
for k, v := range extraEnv {
|
||||||
|
if v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cmd.Env = append(cmd.Env, k+"="+v)
|
||||||
|
}
|
||||||
|
logger.Printf("running %s", filepath.Base(command))
|
||||||
|
return cmd.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureDir(path string, logger *log.Logger) {
|
||||||
|
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||||
|
logger.Fatalf("failed to create dir %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key, fallback string) string {
|
||||||
|
if v := os.Getenv(key); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
@@ -7,9 +7,9 @@ services:
|
|||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
ports:
|
ports:
|
||||||
- "${PORT:-8080}:8080"
|
- "${SERVICE_PORT:-8080}:8080"
|
||||||
environment:
|
environment:
|
||||||
- PORT=${PORT:-8080}
|
- SERVICE_PORT=${SERVICE_PORT:-8080}
|
||||||
- GEOIP_DB_PATH=${GEOIP_DB_PATH:-/app/initial_data/GeoLite2-City.mmdb}
|
- GEOIP_DB_PATH=${GEOIP_DB_PATH:-/app/initial_data/GeoLite2-City.mmdb}
|
||||||
- GEOIP_BACKEND=${GEOIP_BACKEND:-mmdb}
|
- GEOIP_BACKEND=${GEOIP_BACKEND:-mmdb}
|
||||||
- GEOIP_LOADER_TIMEOUT=${GEOIP_LOADER_TIMEOUT:-30m}
|
- GEOIP_LOADER_TIMEOUT=${GEOIP_LOADER_TIMEOUT:-30m}
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -3,6 +3,7 @@ module geoip-rest
|
|||||||
go 1.25
|
go 1.25
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/go-sql-driver/mysql v1.8.1
|
||||||
github.com/gofiber/fiber/v2 v2.52.8
|
github.com/gofiber/fiber/v2 v2.52.8
|
||||||
github.com/jackc/pgx/v5 v5.7.6
|
github.com/jackc/pgx/v5 v5.7.6
|
||||||
github.com/oschwald/geoip2-golang v1.9.0
|
github.com/oschwald/geoip2-golang v1.9.0
|
||||||
@@ -11,6 +12,7 @@ require (
|
|||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
filippo.io/edwards25519 v1.1.0 // indirect
|
||||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,8 +1,12 @@
|
|||||||
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||||
|
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||||
github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
|
github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
|
||||||
github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
|||||||
@@ -12,9 +12,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
CronExpr string
|
CronExpr string
|
||||||
ScriptPath string
|
Command string
|
||||||
Logger *log.Logger
|
Args []string
|
||||||
|
Logger *log.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
type Scheduler struct {
|
type Scheduler struct {
|
||||||
@@ -29,18 +30,14 @@ func Start(cfg Config) (*Scheduler, error) {
|
|||||||
if cfg.CronExpr == "" {
|
if cfg.CronExpr == "" {
|
||||||
return nil, errors.New("CronExpr is required")
|
return nil, errors.New("CronExpr is required")
|
||||||
}
|
}
|
||||||
if cfg.ScriptPath == "" {
|
if cfg.Command == "" {
|
||||||
return nil, errors.New("ScriptPath is required")
|
return nil, errors.New("Command is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Logger == nil {
|
if cfg.Logger == nil {
|
||||||
cfg.Logger = log.Default()
|
cfg.Logger = log.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := os.Stat(cfg.ScriptPath); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
kst, err := time.LoadLocation("Asia/Seoul")
|
kst, err := time.LoadLocation("Asia/Seoul")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
kst = time.FixedZone("KST", 9*60*60)
|
kst = time.FixedZone("KST", 9*60*60)
|
||||||
@@ -54,12 +51,12 @@ func Start(cfg Config) (*Scheduler, error) {
|
|||||||
|
|
||||||
c := cron.New(cron.WithLocation(kst), cron.WithParser(parser))
|
c := cron.New(cron.WithLocation(kst), cron.WithParser(parser))
|
||||||
c.Schedule(spec, cron.FuncJob(func() {
|
c.Schedule(spec, cron.FuncJob(func() {
|
||||||
runScript(cfg.Logger, cfg.ScriptPath)
|
runCommand(cfg.Logger, cfg.Command, cfg.Args...)
|
||||||
}))
|
}))
|
||||||
|
|
||||||
c.Start()
|
c.Start()
|
||||||
|
|
||||||
cfg.Logger.Printf("scheduler started with cron=%s script=%s tz=%s", cfg.CronExpr, cfg.ScriptPath, kst)
|
cfg.Logger.Printf("scheduler started with cron=%s command=%s args=%v tz=%s", cfg.CronExpr, cfg.Command, cfg.Args, kst)
|
||||||
|
|
||||||
return &Scheduler{
|
return &Scheduler{
|
||||||
cron: c,
|
cron: c,
|
||||||
@@ -75,11 +72,11 @@ func (s *Scheduler) Stop() context.Context {
|
|||||||
return s.cron.Stop()
|
return s.cron.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
func runScript(logger *log.Logger, script string) {
|
func runCommand(logger *log.Logger, command string, args ...string) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
logger.Printf("scheduler: running %s", script)
|
logger.Printf("scheduler: running %s %v", command, args)
|
||||||
|
|
||||||
cmd := exec.Command("/bin/bash", script)
|
cmd := exec.Command(command, args...)
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
duration := time.Since(start)
|
duration := time.Since(start)
|
||||||
@@ -88,8 +85,8 @@ func runScript(logger *log.Logger, script string) {
|
|||||||
logger.Printf("scheduler: output:\n%s", string(out))
|
logger.Printf("scheduler: output:\n%s", string(out))
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Printf("scheduler: %s failed after %s: %v", script, duration, err)
|
logger.Printf("scheduler: %s failed after %s: %v", command, duration, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
logger.Printf("scheduler: %s completed in %s", script, duration)
|
logger.Printf("scheduler: %s completed in %s", command, duration)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
LOG_DIR="${USER_PROGRAM_IMPORT_LOG_DIR:-/app/log}"
|
|
||||||
UPDATE_DIR="${USER_PROGRAM_UPDATE_DIR:-/app/update_data}"
|
|
||||||
SCHEMA="${USER_PROGRAM_INFO_SCHEMA:-public}"
|
|
||||||
CSV_DIR="${USER_PROGRAM_INFO_CSV_DIR:-/app/initial_data}"
|
|
||||||
|
|
||||||
MYSQL_HOST="${USER_PROGRAM_INFO_HOST:?USER_PROGRAM_INFO_HOST is required}"
|
|
||||||
MYSQL_PORT="${USER_PROGRAM_INFO_PORT:-3306}"
|
|
||||||
MYSQL_USER="${USER_PROGRAM_INFO_USERNAME:?USER_PROGRAM_INFO_USERNAME is required}"
|
|
||||||
MYSQL_PASS="${USER_PROGRAM_INFO_PASSWORD:?USER_PROGRAM_INFO_PASSWORD is required}"
|
|
||||||
MYSQL_DB="${USER_PROGRAM_INFO_DB:-user_program_info}"
|
|
||||||
MYSQL_TABLE="${USER_PROGRAM_INFO_TABLE:-user_program_info}"
|
|
||||||
|
|
||||||
mkdir -p "${LOG_DIR}" "${UPDATE_DIR}"
|
|
||||||
|
|
||||||
# Target date: yesterday in KST unless USER_PROGRAM_TARGET_DATE=YYYY-MM-DD is provided.
|
|
||||||
TARGET_DATE="${USER_PROGRAM_TARGET_DATE:-$(TZ=Asia/Seoul date -d 'yesterday' +%Y-%m-%d)}"
|
|
||||||
TARGET_DATE_COMPACT="${TARGET_DATE//-/}"
|
|
||||||
OUT_FILE="${UPDATE_DIR}/user_program_info_${TARGET_DATE_COMPACT}.csv"
|
|
||||||
TMP_FILE="${OUT_FILE}.tmp"
|
|
||||||
|
|
||||||
QUERY=$(cat <<SQL
|
|
||||||
SET time_zone = '+00:00';
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
product_name,
|
|
||||||
login_id,
|
|
||||||
user_employee_id,
|
|
||||||
login_version,
|
|
||||||
login_public_ip,
|
|
||||||
login_local_ip,
|
|
||||||
user_company,
|
|
||||||
user_department,
|
|
||||||
user_position,
|
|
||||||
user_login_time,
|
|
||||||
created_at,
|
|
||||||
user_family_flag
|
|
||||||
FROM ${MYSQL_TABLE}
|
|
||||||
WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = '${TARGET_DATE}';
|
|
||||||
SQL
|
|
||||||
)
|
|
||||||
|
|
||||||
echo "[scheduler] dumping data for ${TARGET_DATE} to ${OUT_FILE}"
|
|
||||||
|
|
||||||
mysql --host="${MYSQL_HOST}" --port="${MYSQL_PORT}" --user="${MYSQL_USER}" --password="${MYSQL_PASS}" \
|
|
||||||
--database="${MYSQL_DB}" --batch --raw --silent --skip-column-names -e "${QUERY}" \
|
|
||||||
| python - <<'PY'
|
|
||||||
import csv, sys, os
|
|
||||||
out_path = os.environ["TMP_FILE"]
|
|
||||||
writer = csv.writer(open(out_path, "w", newline=""))
|
|
||||||
writer.writerow([
|
|
||||||
"id","product_name","login_id","user_employee_id","login_version",
|
|
||||||
"login_public_ip","login_local_ip","user_company","user_department",
|
|
||||||
"user_position","user_login_time","created_at","user_family_flag",
|
|
||||||
])
|
|
||||||
for line in sys.stdin:
|
|
||||||
row = line.rstrip("\n").split("\t")
|
|
||||||
writer.writerow(row)
|
|
||||||
PY
|
|
||||||
|
|
||||||
mv "${TMP_FILE}" "${OUT_FILE}"
|
|
||||||
|
|
||||||
echo "[scheduler] running import for ${OUT_FILE}"
|
|
||||||
DATABASE_URL="${DATABASE_URL:?DATABASE_URL is required}" USER_PROGRAM_UPDATE_DIR="${UPDATE_DIR}" USER_PROGRAM_IMPORT_LOG_DIR="${LOG_DIR}" \
|
|
||||||
user-program-import
|
|
||||||
4
to-do.md
4
to-do.md
@@ -13,6 +13,10 @@
|
|||||||
- [x] `user_program_info_replica` DDL/CSV 임포터 추가 (`id bigint`, 텍스트 컬럼, timestamp KST 파싱, bool 플래그) 완료: 2025-12-09 18:32 KST
|
- [x] `user_program_info_replica` DDL/CSV 임포터 추가 (`id bigint`, 텍스트 컬럼, timestamp KST 파싱, bool 플래그) 완료: 2025-12-09 18:32 KST
|
||||||
- [x] 초기/일간 CSV 디렉토리 기반 임포트 + 로그 파일 기록(`log/`), upsert 로직 업데이트 완료: 2025-12-09 19:06 KST
|
- [x] 초기/일간 CSV 디렉토리 기반 임포트 + 로그 파일 기록(`log/`), upsert 로직 업데이트 완료: 2025-12-09 19:06 KST
|
||||||
- [x] Fiber 프로세스 내 cron 스케줄러 추가(전일 덤프 스크립트 실행 + update_data 적용, KST cron 지원) 완료: 2025-12-09 19:28 KST
|
- [x] Fiber 프로세스 내 cron 스케줄러 추가(전일 덤프 스크립트 실행 + update_data 적용, KST cron 지원) 완료: 2025-12-09 19:28 KST
|
||||||
|
- [x] MySQL CLI 의존성 제거, Go 기반 덤퍼(`cmd/user_program_dump`) 추가 및 `scripts/dump_and_import.sh`에서 사용하도록 변경 완료: 2025-12-10 09:34 KST
|
||||||
|
- [x] 스케줄러 토글 env(`USER_PROGRAM_CRON_ENABLE`) 추가, true일 때만 크론 구동하도록 변경 완료: 2025-12-10 09:45 KST
|
||||||
|
- [x] 크론 표현식 env(`USER_PROGRAM_CRON`) 제거, 코드에 KST 00:05 고정 스케줄 적용 완료: 2025-12-10 09:56 KST
|
||||||
|
- [x] bash 스크립트 의존 없이 Go CLI(`user-program-sync`)로 덤프+임포트 수행, 스케줄러가 해당 CLI를 직접 호출하도록 변경 완료: 2025-12-10 09:50 KST
|
||||||
|
|
||||||
## 진행 예정
|
## 진행 예정
|
||||||
- [x] PostgreSQL 전용 Docker 이미지(또는 build 단계)에서 `maxminddb_fdw` 설치 후 `GeoLite2-City.mmdb` 볼륨을 `/data`로 마운트하는 `postgres` 서비스 추가 및 5432 외부 노출
|
- [x] PostgreSQL 전용 Docker 이미지(또는 build 단계)에서 `maxminddb_fdw` 설치 후 `GeoLite2-City.mmdb` 볼륨을 `/data`로 마운트하는 `postgres` 서비스 추가 및 5432 외부 노출
|
||||||
|
|||||||
Reference in New Issue
Block a user