Split backend service into checker and generator
Both are linked through a unix socket.
This commit is contained in:
parent
f755d7c998
commit
ed091e761c
34 changed files with 657 additions and 205 deletions
1
generator/.gitignore
vendored
Normal file
1
generator/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
generator
|
||||
280
generator/generation.go
Normal file
280
generator/generation.go
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
)
|
||||
|
||||
var parallelJobs = runtime.NumCPU()
|
||||
var genQueue chan fic.GenStruct
|
||||
var inQueueMutex sync.RWMutex
|
||||
var inGenQueue map[fic.GenerateType]bool
|
||||
|
||||
func init() {
|
||||
genQueue = make(chan fic.GenStruct)
|
||||
inGenQueue = map[fic.GenerateType]bool{}
|
||||
}
|
||||
|
||||
func launchWorkers() {
|
||||
log.Println("Running with", parallelJobs, "worker(s)")
|
||||
for i := parallelJobs; i > 0; i-- {
|
||||
go consumer()
|
||||
}
|
||||
}
|
||||
|
||||
func enqueueHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var gs fic.GenStruct
|
||||
|
||||
dec := json.NewDecoder(r.Body)
|
||||
err := dec.Decode(&gs)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("POST /enqueue | %v", gs)
|
||||
|
||||
appendGenQueue(gs)
|
||||
http.Error(w, "OK", http.StatusOK)
|
||||
}
|
||||
|
||||
func appendGenQueue(gs fic.GenStruct) {
|
||||
if gs.Type == fic.GenTeam || gs.Type == fic.GenTeamIssues {
|
||||
genQueue <- gs
|
||||
return
|
||||
}
|
||||
|
||||
// Append only if not already in queue
|
||||
inQueueMutex.RLock()
|
||||
if v, ok := inGenQueue[gs.Type]; !ok || !v {
|
||||
inQueueMutex.RUnlock()
|
||||
|
||||
inQueueMutex.Lock()
|
||||
inGenQueue[gs.Type] = true
|
||||
inQueueMutex.Unlock()
|
||||
|
||||
genQueue <- gs
|
||||
} else {
|
||||
inQueueMutex.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
func consumer() {
|
||||
var id string
|
||||
var err error
|
||||
|
||||
for {
|
||||
gs := <-genQueue
|
||||
id = gs.Id
|
||||
|
||||
inQueueMutex.Lock()
|
||||
inGenQueue[gs.Type] = false
|
||||
inQueueMutex.Unlock()
|
||||
|
||||
switch gs.Type {
|
||||
case fic.GenPublic:
|
||||
err = genMyPublicFile()
|
||||
case fic.GenEvents:
|
||||
err = genEventsFile()
|
||||
case fic.GenTeam:
|
||||
err = genTeamMyFile(gs.TeamId)
|
||||
case fic.GenTeams:
|
||||
err = genTeamsFile()
|
||||
case fic.GenThemes:
|
||||
err = genThemesFile()
|
||||
case fic.GenTeamIssues:
|
||||
err = genTeamIssuesFile(gs.TeamId)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Println(id, "[ERR] Unable to generate:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate issues.json for a given team
|
||||
func genTeamIssuesFile(teamid int64) error {
|
||||
team, err := fic.GetTeam(teamid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to GetTeam: %w", err)
|
||||
}
|
||||
|
||||
dirPath := path.Join(TeamsDir, fmt.Sprintf("%d", team.Id))
|
||||
|
||||
my, err := team.MyIssueFile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if my == nil {
|
||||
if _, err := os.Stat(path.Join(dirPath, "issues.json")); !os.IsNotExist(err) {
|
||||
err = os.Remove(path.Join(dirPath, "issues.json"))
|
||||
if err != nil {
|
||||
log.Printf("Unable to remove empty issues file: %s", path.Join(dirPath, "issues.json"))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if s, err := os.Stat(dirPath); os.IsNotExist(err) {
|
||||
os.MkdirAll(dirPath, 0777)
|
||||
} else if !s.IsDir() {
|
||||
return fmt.Errorf("%s is not a directory", dirPath)
|
||||
}
|
||||
|
||||
if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "issues.json"), j, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate my.json, wait.json and scores.json for a given team
|
||||
func genTeamMyFile(teamid int64) error {
|
||||
team, err := fic.GetTeam(teamid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to GetTeam: %w", err)
|
||||
}
|
||||
|
||||
dirPath := path.Join(TeamsDir, fmt.Sprintf("%d", team.Id))
|
||||
|
||||
if s, err := os.Stat(dirPath); os.IsNotExist(err) {
|
||||
os.MkdirAll(dirPath, 0777)
|
||||
} else if !s.IsDir() {
|
||||
return fmt.Errorf("%s is not a directory", dirPath)
|
||||
}
|
||||
|
||||
if my, err := fic.MyJSONTeam(team, true); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "my.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Speed up generation when challenge is started
|
||||
if !ChStarted {
|
||||
if my, err := fic.MyJSONTeam(team, false); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "wait.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if scores, err := team.ScoreGrid(); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(scores); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "scores.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate public my.json file
|
||||
func genMyPublicFile() error {
|
||||
dirPath := path.Join(TeamsDir, "public")
|
||||
|
||||
if s, err := os.Stat(dirPath); os.IsNotExist(err) {
|
||||
os.MkdirAll(dirPath, 0777)
|
||||
} else if !s.IsDir() {
|
||||
return fmt.Errorf("%s is not a directory", dirPath)
|
||||
}
|
||||
|
||||
if my, err := fic.MyJSONTeam(nil, true); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "my.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
os.Symlink("my.json", path.Join(dirPath, "wait.json"))
|
||||
|
||||
if teams, err := fic.ExportTeams(true); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(teams); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(dirPath, "teams.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate general evemts.json file
|
||||
func genEventsFile() error {
|
||||
if evts, err := fic.GetLastEvents(); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(evts); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(TeamsDir, "events.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate general teams.json file
|
||||
func genTeamsFile() error {
|
||||
if teams, err := fic.ExportTeams(false); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(teams); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(TeamsDir, "teams.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if teams, err := fic.ExportTeams(true); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(teams); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(TeamsDir, "public", "teams.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate general themes.json file
|
||||
func genThemesFile() error {
|
||||
if themes, err := fic.ExportThemes(); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(themes); err != nil {
|
||||
return err
|
||||
} else if err = ioutil.WriteFile(path.Join(TeamsDir, "themes.json"), j, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func genAll() {
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenThemes})
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenTeams})
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenEvents})
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenPublic})
|
||||
|
||||
if teams, err := fic.GetActiveTeams(); err != nil {
|
||||
log.Println("Team retrieval error: ", err)
|
||||
} else {
|
||||
for _, team := range teams {
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenTeam, TeamId: team.Id})
|
||||
appendGenQueue(fic.GenStruct{Type: fic.GenTeamIssues, TeamId: team.Id})
|
||||
}
|
||||
}
|
||||
}
|
||||
151
generator/main.go
Normal file
151
generator/main.go
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
"srs.epita.fr/fic-server/settings"
|
||||
)
|
||||
|
||||
var TeamsDir string
|
||||
|
||||
var ChStarted = false
|
||||
var lastRegeneration time.Time
|
||||
var skipInitialGeneration = false
|
||||
|
||||
func reloadSettings(config *settings.Settings) {
|
||||
fic.HintCoefficient = config.HintCurCoefficient
|
||||
fic.WChoiceCoefficient = config.WChoiceCurCoefficient
|
||||
fic.ExerciceCurrentCoefficient = config.ExerciceCurCoefficient
|
||||
ChStarted = config.Start.Unix() > 0 && time.Since(config.Start) >= 0
|
||||
if lastRegeneration != config.Generation || fic.PartialValidation != config.PartialValidation || fic.UnlockedChallengeDepth != config.UnlockedChallengeDepth || fic.UnlockedChallengeUpTo != config.UnlockedChallengeUpTo || fic.DisplayAllFlags != config.DisplayAllFlags || fic.FirstBlood != config.FirstBlood || fic.SubmissionCostBase != config.SubmissionCostBase || fic.SubmissionUniqueness != config.SubmissionUniqueness || fic.DiscountedFactor != config.DiscountedFactor {
|
||||
fic.PartialValidation = config.PartialValidation
|
||||
fic.UnlockedChallengeDepth = config.UnlockedChallengeDepth
|
||||
fic.UnlockedChallengeUpTo = config.UnlockedChallengeUpTo
|
||||
fic.DisplayAllFlags = config.DisplayAllFlags
|
||||
|
||||
fic.FirstBlood = config.FirstBlood
|
||||
fic.SubmissionCostBase = config.SubmissionCostBase
|
||||
fic.SubmissionUniqueness = config.SubmissionUniqueness
|
||||
fic.GlobalScoreCoefficient = config.GlobalScoreCoefficient
|
||||
fic.CountOnlyNotGoodTries = config.CountOnlyNotGoodTries
|
||||
fic.DiscountedFactor = config.DiscountedFactor
|
||||
|
||||
if !skipInitialGeneration {
|
||||
log.Println("Generating files...")
|
||||
go func() {
|
||||
genAll()
|
||||
log.Println("Full generation done")
|
||||
}()
|
||||
} else {
|
||||
skipInitialGeneration = false
|
||||
log.Println("Regeneration skipped by option.")
|
||||
}
|
||||
lastRegeneration = config.Generation
|
||||
} else {
|
||||
log.Println("No change found. Skipping regeneration.")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if v, exists := os.LookupEnv("FIC_BASEURL"); exists {
|
||||
fic.FilesDir = v + "files"
|
||||
} else {
|
||||
fic.FilesDir = "/files"
|
||||
}
|
||||
|
||||
var bind = flag.String("bind", "./GENERATOR/generator.socket", "Bind path or port/socket")
|
||||
var dsn = flag.String("dsn", fic.DSNGenerator(), "DSN to connect to the MySQL server")
|
||||
flag.StringVar(&settings.SettingsDir, "settings", "./SETTINGSDIST", "Base directory where load and save settings")
|
||||
flag.StringVar(&TeamsDir, "teams", "./TEAMS", "Base directory where save teams JSON files")
|
||||
flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Request path prefix to reach files")
|
||||
flag.BoolVar(&skipInitialGeneration, "skipfullgeneration", skipInitialGeneration, "Skip the initial regeneration")
|
||||
flag.IntVar(¶llelJobs, "jobs", parallelJobs, "Number of generation workers")
|
||||
flag.Parse()
|
||||
|
||||
log.SetPrefix("[generator] ")
|
||||
|
||||
settings.SettingsDir = path.Clean(settings.SettingsDir)
|
||||
TeamsDir = path.Clean(TeamsDir)
|
||||
|
||||
launchWorkers()
|
||||
|
||||
log.Println("Opening DB...")
|
||||
if err := fic.DBInit(*dsn); err != nil {
|
||||
log.Fatal("Cannot open the database: ", err)
|
||||
}
|
||||
defer fic.DBClose()
|
||||
|
||||
// Load configuration
|
||||
settings.LoadAndWatchSettings(path.Join(settings.SettingsDir, settings.SettingsFile), reloadSettings)
|
||||
|
||||
// Register SIGUSR1, SIGUSR2
|
||||
interrupt1 := make(chan os.Signal, 1)
|
||||
signal.Notify(interrupt1, syscall.SIGUSR1)
|
||||
interrupt2 := make(chan os.Signal, 1)
|
||||
signal.Notify(interrupt2, syscall.SIGUSR2)
|
||||
|
||||
// Prepare graceful shutdown
|
||||
interrupt := make(chan os.Signal, 1)
|
||||
signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: *bind,
|
||||
}
|
||||
|
||||
http.HandleFunc("/enqueue", enqueueHandler)
|
||||
|
||||
// Serve pages
|
||||
go func() {
|
||||
if !strings.Contains(*bind, ":") {
|
||||
if _, err := os.Stat(*bind); !os.IsNotExist(err) {
|
||||
if err := os.Remove(*bind); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
os.MkdirAll(path.Dir(*bind), 0777)
|
||||
|
||||
unixListener, err := net.Listen("unix", *bind)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Fatal(srv.Serve(unixListener))
|
||||
} else if err := srv.ListenAndServe(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}()
|
||||
log.Println(fmt.Sprintf("Ready, listening on %s", *bind))
|
||||
|
||||
// Wait shutdown signal
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case <-interrupt:
|
||||
break loop
|
||||
case <-interrupt1:
|
||||
log.Println("SIGUSR1 received, regenerating all files...")
|
||||
genAll()
|
||||
log.Println("SIGUSR1 treated.")
|
||||
case <-interrupt2:
|
||||
inQueueMutex.Lock()
|
||||
log.Printf("SIGUSR2 received, dumping statistics:\n parallelJobs: %d\n genQueue: %d\n Teams in queue: %v\n Challenge started: %v\n Last regeneration: %v\n", parallelJobs, len(genQueue), inGenQueue, ChStarted, lastRegeneration)
|
||||
inQueueMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
log.Print("The service is shutting down...")
|
||||
srv.Shutdown(context.Background())
|
||||
log.Println("done")
|
||||
}
|
||||
Reference in a new issue