경로 통일
This commit is contained in:
Lectom C Han
@@ -2,294 +2,62 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"geoip-rest/internal/userprogram"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUpdateDir = "./update_data"
|
||||
defaultTable = "user_program_info"
|
||||
defaultDB = "user_program_info"
|
||||
defaultDumpTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
type config struct {
|
||||
host string
|
||||
port int
|
||||
user string
|
||||
password string
|
||||
database string
|
||||
table string
|
||||
updateDir string
|
||||
target time.Time
|
||||
}
|
||||
const defaultDumpTimeout = 5 * time.Minute
|
||||
|
||||
func main() {
|
||||
logger := log.New(os.Stdout, "[dump] ", log.LstdFlags)
|
||||
|
||||
cfg, err := loadConfig()
|
||||
mysqlCfg, err := userprogram.NewMySQLConfigFromEnv()
|
||||
if err != nil {
|
||||
log.Fatalf("config error: %v", err)
|
||||
}
|
||||
updateDir := userprogram.DefaultUpdateDir
|
||||
if val := os.Getenv("USER_PROGRAM_UPDATE_DIR"); val != "" {
|
||||
updateDir = val
|
||||
}
|
||||
target, err := userprogram.ParseTargetDate(os.Getenv("USER_PROGRAM_TARGET_DATE"))
|
||||
if err != nil {
|
||||
log.Fatalf("target date error: %v", err)
|
||||
}
|
||||
startID := int64(0)
|
||||
if val := os.Getenv("USER_PROGRAM_START_ID"); val != "" {
|
||||
parsed, parseErr := strconv.ParseInt(val, 10, 64)
|
||||
if parseErr != nil {
|
||||
log.Fatalf("invalid USER_PROGRAM_START_ID: %v", parseErr)
|
||||
}
|
||||
startID = parsed
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultDumpTimeout)
|
||||
defer cancel()
|
||||
|
||||
dsn := (&mysql.Config{
|
||||
User: cfg.user,
|
||||
Passwd: cfg.password,
|
||||
Net: "tcp",
|
||||
Addr: netAddr(cfg.host, cfg.port),
|
||||
DBName: cfg.database,
|
||||
Params: map[string]string{"parseTime": "true", "loc": "UTC", "charset": "utf8mb4"},
|
||||
AllowNativePasswords: true,
|
||||
}).FormatDSN()
|
||||
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
dumper, err := userprogram.NewDumper(mysqlCfg, updateDir)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to open mysql connection: %v", err)
|
||||
log.Fatalf("init dumper failed: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
defer dumper.Close()
|
||||
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
log.Fatalf("failed to ping mysql: %v", err)
|
||||
endID, err := dumper.MaxIDUntil(ctx, target)
|
||||
if err != nil {
|
||||
log.Fatalf("determine end id failed: %v", err)
|
||||
}
|
||||
if endID <= startID {
|
||||
logger.Printf("no rows to dump (start_id=%d end_id=%d)", startID, endID)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(ctx, "SET time_zone = '+00:00'"); err != nil {
|
||||
log.Fatalf("failed to set timezone: %v", err)
|
||||
}
|
||||
|
||||
outPath, err := dumpToCSV(ctx, db, cfg)
|
||||
outPath, err := dumper.DumpRange(ctx, startID, endID, target)
|
||||
if err != nil {
|
||||
log.Fatalf("dump failed: %v", err)
|
||||
}
|
||||
|
||||
logger.Printf("dumped %s to %s", cfg.target.Format("2006-01-02"), outPath)
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
port, err := strconv.Atoi(env("USER_PROGRAM_INFO_PORT", "3306"))
|
||||
if err != nil {
|
||||
return config{}, fmt.Errorf("invalid USER_PROGRAM_INFO_PORT: %w", err)
|
||||
}
|
||||
|
||||
target, err := resolveTargetDate(env("USER_PROGRAM_TARGET_DATE", ""))
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
|
||||
table := env("USER_PROGRAM_INFO_TABLE", defaultTable)
|
||||
if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) {
|
||||
return config{}, fmt.Errorf("invalid table name: %s", table)
|
||||
}
|
||||
|
||||
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
|
||||
if updateDir == "" {
|
||||
updateDir = defaultUpdateDir
|
||||
}
|
||||
if err := os.MkdirAll(updateDir, 0o755); err != nil {
|
||||
return config{}, fmt.Errorf("creating update dir: %w", err)
|
||||
}
|
||||
|
||||
return config{
|
||||
host: envRequired("USER_PROGRAM_INFO_HOST"),
|
||||
port: port,
|
||||
user: envRequired("USER_PROGRAM_INFO_USERNAME"),
|
||||
password: envRequired("USER_PROGRAM_INFO_PASSWORD"),
|
||||
database: env("USER_PROGRAM_INFO_DB", defaultDB),
|
||||
table: table,
|
||||
updateDir: updateDir,
|
||||
target: target,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func dumpToCSV(ctx context.Context, db *sql.DB, cfg config) (string, error) {
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
id,
|
||||
product_name,
|
||||
login_id,
|
||||
user_employee_id,
|
||||
login_version,
|
||||
login_public_ip,
|
||||
login_local_ip,
|
||||
user_company,
|
||||
user_department,
|
||||
user_position,
|
||||
user_login_time,
|
||||
created_at,
|
||||
user_family_flag
|
||||
FROM %s
|
||||
WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = ?;`, cfg.table)
|
||||
|
||||
rows, err := db.QueryContext(ctx, query, cfg.target.Format("2006-01-02"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
filename := fmt.Sprintf("user_program_info_%s.csv", cfg.target.Format("20060102"))
|
||||
outPath := filepath.Join(cfg.updateDir, filename)
|
||||
tmpPath := outPath + ".tmp"
|
||||
|
||||
f, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
writer := csv.NewWriter(f)
|
||||
defer writer.Flush()
|
||||
|
||||
header := []string{
|
||||
"id",
|
||||
"product_name",
|
||||
"login_id",
|
||||
"user_employee_id",
|
||||
"login_version",
|
||||
"login_public_ip",
|
||||
"login_local_ip",
|
||||
"user_company",
|
||||
"user_department",
|
||||
"user_position",
|
||||
"user_login_time",
|
||||
"created_at",
|
||||
"user_family_flag",
|
||||
}
|
||||
if err := writer.Write(header); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
record, err := scanRow(rows)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writer.Write(record); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
writer.Flush()
|
||||
if err := writer.Error(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpPath, outPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
func scanRow(rows *sql.Rows) ([]string, error) {
|
||||
var (
|
||||
id sql.NullInt64
|
||||
productName sql.NullString
|
||||
loginID sql.NullString
|
||||
employeeID sql.NullString
|
||||
loginVersion sql.NullString
|
||||
loginPublicIP sql.NullString
|
||||
loginLocalIP sql.NullString
|
||||
userCompany sql.NullString
|
||||
userDepartment sql.NullString
|
||||
userPosition sql.NullString
|
||||
userLoginTime sql.NullString
|
||||
createdAt sql.NullString
|
||||
userFamilyFlag sql.NullString
|
||||
)
|
||||
|
||||
if err := rows.Scan(
|
||||
&id,
|
||||
&productName,
|
||||
&loginID,
|
||||
&employeeID,
|
||||
&loginVersion,
|
||||
&loginPublicIP,
|
||||
&loginLocalIP,
|
||||
&userCompany,
|
||||
&userDepartment,
|
||||
&userPosition,
|
||||
&userLoginTime,
|
||||
&createdAt,
|
||||
&userFamilyFlag,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !id.Valid {
|
||||
return nil, fmt.Errorf("row missing id")
|
||||
}
|
||||
|
||||
return []string{
|
||||
strconv.FormatInt(id.Int64, 10),
|
||||
nullToString(productName),
|
||||
nullToString(loginID),
|
||||
nullToString(employeeID),
|
||||
nullToString(loginVersion),
|
||||
nullToString(loginPublicIP),
|
||||
nullToString(loginLocalIP),
|
||||
nullToString(userCompany),
|
||||
nullToString(userDepartment),
|
||||
nullToString(userPosition),
|
||||
nullToString(userLoginTime),
|
||||
nullToString(createdAt),
|
||||
nullToString(userFamilyFlag),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveTargetDate(raw string) (time.Time, error) {
|
||||
if raw == "" {
|
||||
now := time.Now().In(kst())
|
||||
yesterday := now.AddDate(0, 0, -1)
|
||||
return time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, kst()), nil
|
||||
}
|
||||
t, err := time.ParseInLocation("2006-01-02", raw, kst())
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("invalid USER_PROGRAM_TARGET_DATE: %w", err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func kst() *time.Location {
|
||||
loc, err := time.LoadLocation("Asia/Seoul")
|
||||
if err != nil {
|
||||
return time.FixedZone("KST", 9*60*60)
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envRequired(key string) string {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
log.Fatalf("%s is required", key)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func netAddr(host string, port int) string {
|
||||
return fmt.Sprintf("%s:%d", host, port)
|
||||
}
|
||||
|
||||
func nullToString(v sql.NullString) string {
|
||||
if v.Valid {
|
||||
return v.String
|
||||
}
|
||||
return ""
|
||||
logger.Printf("dumped ids (%d, %d] to %s", startID, endID, outPath)
|
||||
}
|
||||
|
||||
@@ -4,72 +4,42 @@ import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"geoip-rest/internal/userprogram"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUpdateDir = "/app/update_data"
|
||||
defaultLogDir = "/app/log"
|
||||
defaultSchema = "public"
|
||||
defaultTimeout = 15 * time.Minute
|
||||
)
|
||||
const defaultTimeout = 30 * time.Minute
|
||||
|
||||
func main() {
|
||||
logger := log.New(os.Stdout, "[sync] ", log.LstdFlags)
|
||||
|
||||
dbURL := os.Getenv("DATABASE_URL")
|
||||
if dbURL == "" {
|
||||
logger.Fatal("DATABASE_URL is required")
|
||||
}
|
||||
|
||||
mysqlCfg, err := userprogram.NewMySQLConfigFromEnv()
|
||||
if err != nil {
|
||||
logger.Fatalf("mysql config: %v", err)
|
||||
}
|
||||
paths, err := userprogram.NewPathsFromEnv()
|
||||
if err != nil {
|
||||
logger.Fatalf("paths config: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
|
||||
logDir := env("USER_PROGRAM_IMPORT_LOG_DIR", defaultLogDir)
|
||||
schema := env("USER_PROGRAM_INFO_SCHEMA", defaultSchema)
|
||||
|
||||
ensureDir(updateDir, logger)
|
||||
ensureDir(logDir, logger)
|
||||
|
||||
if err := runCmd(ctx, logger, "user-program-dump", map[string]string{
|
||||
"USER_PROGRAM_UPDATE_DIR": updateDir,
|
||||
if err := userprogram.Sync(ctx, userprogram.SyncConfig{
|
||||
MySQL: mysqlCfg,
|
||||
DatabaseURL: dbURL,
|
||||
InitialCSV: paths.InitialCSV,
|
||||
UpdateDir: paths.UpdateDir,
|
||||
LogDir: paths.LogDir,
|
||||
Schema: paths.Schema,
|
||||
Logger: logger,
|
||||
}); err != nil {
|
||||
logger.Fatalf("dump failed: %v", err)
|
||||
}
|
||||
|
||||
if err := runCmd(ctx, logger, "user-program-import", map[string]string{
|
||||
"USER_PROGRAM_UPDATE_DIR": updateDir,
|
||||
"USER_PROGRAM_IMPORT_LOG_DIR": logDir,
|
||||
"USER_PROGRAM_INFO_SCHEMA": schema,
|
||||
}); err != nil {
|
||||
logger.Fatalf("import failed: %v", err)
|
||||
}
|
||||
|
||||
logger.Printf("sync completed (update_dir=%s, log_dir=%s, schema=%s)", updateDir, logDir, schema)
|
||||
}
|
||||
|
||||
func runCmd(ctx context.Context, logger *log.Logger, command string, extraEnv map[string]string) error {
|
||||
cmd := exec.CommandContext(ctx, command)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Env = os.Environ()
|
||||
for k, v := range extraEnv {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
cmd.Env = append(cmd.Env, k+"="+v)
|
||||
}
|
||||
logger.Printf("running %s", filepath.Base(command))
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func ensureDir(path string, logger *log.Logger) {
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
logger.Fatalf("failed to create dir %s: %v", path, err)
|
||||
logger.Fatalf("sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user