Last active
September 12, 2025 21:29
-
-
Save dixonwhitmire/53c17a6a2b3b9de03e077bd4e9ea1c9b to your computer and use it in GitHub Desktop.
A Go timer example which does a bit of database comparisons for a project at work. Names of things have been changed.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "errors" | |
| "flag" | |
| "fmt" | |
| "log" | |
| "os" | |
| "os/exec" | |
| "os/signal" | |
| "strings" | |
| "syscall" | |
| "time" | |
| "github.com/jackc/pgx/v5" | |
| ) | |
| // environment variable names and settings | |
| const ( | |
| v2DbUriEnvName = "DB_V2_URI" | |
| v3DbUriEnvName = "DB_V3_URI" | |
| unoDbUriEnvName = "DB_URI" | |
| unoMountPoint = "/mnt/SOME_PATH" | |
| ) | |
| // application constants | |
| const ( | |
| processingThreshold = 0.5 | |
| studyRecordDeltaSize = 25_000 | |
| studyRecordSize = 625_000 | |
| origin = "origin" | |
| ) | |
| // timerDuration specifies how often "uno ketchup" runs. | |
| var timerDuration time.Duration | |
| // studyLocations is the "set" of locations from a Study Hall V2 or V3 database. | |
| type studyLocations map[string]struct{} | |
| // parseUnoPath returns the path to the UNO executable from the provided "flag" args. | |
| // flag.Args() returns the remaining positional arguments not consumed by flag parameters. | |
| func parseUnoPath(flagArgs []string) (string, error) { | |
| if len(flagArgs) < 1 { | |
| return "", errors.New("uno path is missing from os args") | |
| } | |
| _, err := os.Stat(flagArgs[0]) | |
| if err != nil { | |
| return "", err | |
| } | |
| return flagArgs[0], nil | |
| } | |
| // parseRequiredEnv parses an env key from the environment, returning an error if the key does not exist. | |
| func parseRequiredEnv(keyName string) (string, error) { | |
| keyValue := os.Getenv(keyName) | |
| if keyValue == "" { | |
| return "", errors.New(keyName + " is missing") | |
| } | |
| return keyValue, nil | |
| } | |
| // fetchStudyLocations returns the study locations from the specified database. | |
| func fetchStudyLocations(ctx context.Context, dbUri, dbSchema string) (studyLocations, error) { | |
| results := make(studyLocations, studyRecordSize) | |
| log.Printf("Fetching study locations from %s.that_table", dbSchema) | |
| dbConnection, err := pgx.Connect(ctx, dbUri) | |
| if err != nil { | |
| return nil, err | |
| } | |
| defer dbConnection.Close(ctx) | |
| selectSql := fmt.Sprintf("SELECT study_location FROM %s.that_table WHERE origin = $1", dbSchema) | |
| rows, err := dbConnection.Query(ctx, selectSql, origin) | |
| if err != nil { | |
| return nil, err | |
| } | |
| defer rows.Close() | |
| for rows.Next() { | |
| var studyLocation string | |
| if err := rows.Scan(&studyLocation); err != nil { | |
| return nil, err | |
| } | |
| // V2 has "/" while V3 has "//" let's clean that up | |
| studyLocation = "/" + strings.TrimLeft(studyLocation, "/") | |
| results[studyLocation] = struct{}{} | |
| } | |
| return results, nil | |
| } | |
| // diffStudyLocations returns a new studyLocations value containing the v2 locations which are not yet in v3. | |
| func diffStudyLocations(v2Locations, v3Locations studyLocations) studyLocations { | |
| diffs := make(studyLocations, studyRecordDeltaSize) | |
| for v2Location, _ := range v2Locations { | |
| if _, ok := v3Locations[v2Location]; !ok { | |
| diffs[v2Location] = struct{}{} | |
| } | |
| } | |
| return diffs | |
| } | |
| // queryV3StudyCount returns the number of studies in the V3 database. | |
| func queryV3StudyCount(ctx context.Context, dbConn *pgx.Conn) (int, error) { | |
| row := dbConn.QueryRow(ctx, "SELECT COUNT(id) FROM swell_schema.that_table WHERE origin = $1", origin) | |
| var studyCount int | |
| if err := row.Scan(&studyCount); err != nil { | |
| return 0, err | |
| } | |
| return studyCount, nil | |
| } | |
| // addToV3 adds Offload studyLocations to the V3 database | |
| func addToV3(ctx context.Context, studyLocations studyLocations, dbUri, unoPath string) error { | |
| if err := os.Setenv(unoDbUriEnvName, dbUri); err != nil { | |
| return err | |
| } | |
| dbConnection, err := pgx.Connect(ctx, dbUri) | |
| if err != nil { | |
| return err | |
| } | |
| defer dbConnection.Close(ctx) | |
| studyCount, err := queryV3StudyCount(ctx, dbConnection) | |
| if err != nil { | |
| return err | |
| } | |
| log.Printf("V3 study count prior to update: %d\n", studyCount) | |
| errorCount := 0 | |
| threshold := int(float64(len(studyLocations)) * processingThreshold) | |
| for studyLocation, _ := range studyLocations { | |
| unoCmd := exec.Command(unoPath, "-force", "-mount", unoMountPoint, "-directory", studyLocation) | |
| _, err := unoCmd.Output() | |
| if err != nil { | |
| log.Printf("error adding directory %q error: %v", studyLocation, err) | |
| errorCount++ | |
| // bail if we hit our threshold | |
| if errorCount > threshold { | |
| log.Printf("shutting down process. error count: %d exceeds threshold: %d", errorCount, threshold) | |
| break | |
| } | |
| } | |
| } | |
| studyCount, err = queryV3StudyCount(ctx, dbConnection) | |
| if err != nil { | |
| return err | |
| } | |
| log.Printf("V3 study count following update: %d\n", studyCount) | |
| return nil | |
| } | |
| func ketchup(ctx context.Context) { | |
| // parse configuration | |
| unoPath, err := parseUnoPath(flag.Args()) | |
| if err != nil { | |
| log.Fatalf("Usage: %s <path to uno exec>\n", os.Args[0]) | |
| } | |
| v2Uri, err := parseRequiredEnv(v2DbUriEnvName) | |
| if err != nil { | |
| log.Fatalf("%q is not set\n", v2DbUriEnvName) | |
| } | |
| v3Uri, err := parseRequiredEnv(v3DbUriEnvName) | |
| if err != nil { | |
| log.Fatalf("%q is not set\n", v3DbUriEnvName) | |
| } | |
| // diff studies between V2 and V3 | |
| v2Locations, err := fetchStudyLocations(context.Background(), v2Uri, "study_catalog") | |
| if err != nil { | |
| log.Fatalf("Error fetching v2 studies %v\n", err) | |
| } | |
| log.Printf("Fetched %d v2 study locations\n", len(v2Locations)) | |
| v3Locations, err := fetchStudyLocations(context.Background(), v3Uri, "study_hall") | |
| if err != nil { | |
| log.Fatalf("Error fetching v3 studies %v\n", err) | |
| } | |
| log.Printf("Fetched %d v3 study locations\n", len(v3Locations)) | |
| diffs := diffStudyLocations(v2Locations, v3Locations) | |
| log.Printf("Found %d study locations to add to V3\n", len(diffs)) | |
| if err := addToV3(context.Background(), diffs, v3Uri, unoPath); err != nil { | |
| log.Fatalf("Error adding to v3: %v\n", err) | |
| } | |
| } | |
| func main() { | |
| // support term/interrupt signals | |
| ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) | |
| defer stop() | |
| // our command line flag may accept any valid duration string | |
| // 300ms | |
| // 10s | |
| // 1.5h | |
| // 2h30m | |
| flag.DurationVar(&timerDuration, "duration", 3*time.Hour, "The duration or interval between runs") | |
| flag.Parse() | |
| ticker := time.NewTicker(timerDuration) | |
| defer ticker.Stop() | |
| // run "uno" ketchup | |
| ketchup(ctx) | |
| for { | |
| select { | |
| // run "uno" ketchup at the next interval | |
| case <-ticker.C: | |
| log.Println("timer has expired . . . starting unoketchup") | |
| ketchup(ctx) | |
| // shutdown | |
| case <-ctx.Done(): | |
| log.Println("Shutting down gracefully...") | |
| return | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment