From 2b75287d16dc71e59e601acfe4bedd136254c8df Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 11 Jul 2019 19:52:13 +0200 Subject: [PATCH] backend: multithread generation --- backend/choices.go | 8 ++- backend/generation.go | 117 +++++++++++++++++++++++++++++++--------- backend/hint.go | 13 ++--- backend/main.go | 8 +-- backend/registration.go | 18 ++----- backend/rename.go | 10 ++-- backend/submission.go | 12 ++--- 7 files changed, 115 insertions(+), 71 deletions(-) diff --git a/backend/choices.go b/backend/choices.go index 72599433..eec60ac6 100644 --- a/backend/choices.go +++ b/backend/choices.go @@ -2,15 +2,15 @@ package main import ( "encoding/json" - "log" "io/ioutil" + "log" "os" "srs.epita.fr/fic-server/libfic" ) type wantChoices struct { - FlagId int64 `json:"id"` + FlagId int64 `json:"id"` } func treatWantChoices(pathname string, team fic.Team) { @@ -34,9 +34,7 @@ func treatWantChoices(pathname string, team fic.Team) { } else if err = team.DisplayChoices(flag); err != nil { log.Println("[ERR]", err) } else { - if err = genTeamMyFile(team); err != nil { - log.Println("my-", team.Id, ".json generation error: ", err) - } + genTeamQueue <- &team if err = os.Remove(pathname); err != nil { log.Println("[ERR]", err) } diff --git a/backend/generation.go b/backend/generation.go index 07c750a8..0c98edda 100644 --- a/backend/generation.go +++ b/backend/generation.go @@ -8,12 +8,92 @@ import ( "log" "os" "path" + "runtime" + "sync" "srs.epita.fr/fic-server/libfic" ) +type GenerateType int + +const ( + GenPublic GenerateType = iota + GenEvents + GenTeams + GenThemes +) + +type genStruct struct { + Id string + Type GenerateType +} + +var genTeamQueue chan *fic.Team +var genQueue chan genStruct +var inQueueMutex sync.RWMutex +var inGenQueue map[GenerateType]bool + +func init() { + genTeamQueue = make(chan *fic.Team) + genQueue = make(chan genStruct) + inGenQueue = map[GenerateType]bool{} + + for i := runtime.NumCPU(); i > 0; i-- { + go consumer() + } +} + +func appendGenQueue(gs genStruct) { + inQueueMutex.RLock() + if v, ok := inGenQueue[gs.Type]; !ok || !v { + inQueueMutex.RUnlock() + + inQueueMutex.Lock() + inGenQueue[gs.Type] = true + inQueueMutex.Unlock() + + genQueue <- gs + } else { + inQueueMutex.RUnlock() + } +} + +func consumer() { + for { + var id string + var err error + + select { + case gt := <-genTeamQueue: + err = genTeamMyFile(gt) + + case gs := <-genQueue: + id = gs.Id + + inQueueMutex.Lock() + inGenQueue[gs.Type] = false + inQueueMutex.Unlock() + + switch gs.Type { + case GenPublic: + err = genMyPublicFile() + case GenEvents: + err = genEventsFile() + case GenTeams: + err = genTeamsFile() + case GenThemes: + err = genThemesFile() + } + } + + if err != nil { + log.Println(id, "[ERR] Unable to generate:", err) + } + } +} + // Generate my.json and wait.json for a given team -func genTeamMyFile(team fic.Team) error { +func genTeamMyFile(team *fic.Team) error { dirPath := path.Join(TeamsDir, fmt.Sprintf("%d", team.Id)) if s, err := os.Stat(dirPath); os.IsNotExist(err) { @@ -22,7 +102,7 @@ func genTeamMyFile(team fic.Team) error { return errors.New(fmt.Sprintf("%s is not a directory", dirPath)) } - if my, err := fic.MyJSONTeam(&team, true); err != nil { + if my, err := fic.MyJSONTeam(team, true); err != nil { return err } else if j, err := json.Marshal(my); err != nil { return err @@ -32,7 +112,7 @@ func genTeamMyFile(team fic.Team) error { // Speed up generation when challenge is started if !ChStarted { - if my, err := fic.MyJSONTeam(&team, false); err != nil { + if my, err := fic.MyJSONTeam(team, false); err != nil { return err } else if j, err := json.Marshal(my); err != nil { return err @@ -106,32 +186,17 @@ func genThemesFile() error { return nil } -func genTeamAll(team fic.Team) { - if err := genThemesFile(); err != nil { - log.Println("themes.json generation error: ", err) - } else if err = genTeamsFile(); err != nil { - log.Println("teams.json generation error: ", err) - } else if err = genTeamMyFile(team); err != nil { - log.Println("my.json(", team.Id, ") generation error: ", err) - } -} - func genAll() { - if err := genThemesFile(); err != nil { - log.Println("themes.json generation error: ", err) - } else if err = genTeamsFile(); err != nil { - log.Println("teams.json generation error: ", err) - } else if err = genEventsFile(); err != nil { - log.Println("events.json generation error: ", err) - } else if err = genMyPublicFile(); err != nil { - log.Println("MyPublic generation error: ", err) - } else if teams, err := fic.GetActiveTeams(); err != nil { + appendGenQueue(genStruct{Type: GenThemes}) + appendGenQueue(genStruct{Type: GenTeams}) + appendGenQueue(genStruct{Type: GenEvents}) + appendGenQueue(genStruct{Type: GenPublic}) + + if teams, err := fic.GetActiveTeams(); err != nil { log.Println("Team retrieval error: ", err) } else { - for _, team := range(teams) { - if err = genTeamMyFile(team); err != nil { - log.Println("Team generation error: ", err) - } + for _, team := range teams { + genTeamQueue <- &team } } } diff --git a/backend/hint.go b/backend/hint.go index cf33356f..f991d4be 100644 --- a/backend/hint.go +++ b/backend/hint.go @@ -3,15 +3,15 @@ package main import ( "encoding/json" "fmt" - "log" "io/ioutil" + "log" "os" "srs.epita.fr/fic-server/libfic" ) type askOpenHint struct { - HintId int64 `json:"id"` + HintId int64 `json:"id"` } func treatOpeningHint(pathname string, team fic.Team) { @@ -42,13 +42,8 @@ func treatOpeningHint(pathname string, team fic.Team) { log.Println("[WRN] Unable to create event:", err) } - - if err = genTeamMyFile(team); err != nil { - log.Println("my-", team.Id, ".json generation error: ", err) - } - if err = genEventsFile(); err != nil { - log.Println("events.json generation error: ", err) - } + genTeamQueue <- &team + appendGenQueue(genStruct{Type: GenEvents}) if err = os.Remove(pathname); err != nil { log.Println("[ERR]", err) } diff --git a/backend/main.go b/backend/main.go index 72d401fd..b078ce54 100644 --- a/backend/main.go +++ b/backend/main.go @@ -31,7 +31,7 @@ func watchsubdir(watcher *fsnotify.Watcher, pathname string) error { } else { for _, d := range ds { p := path.Join(pathname, d.Name()) - if d.IsDir() && d.Name() != ".tmp" && d.Mode() & os.ModeSymlink == 0 { + if d.IsDir() && d.Name() != ".tmp" && d.Mode()&os.ModeSymlink == 0 { if err := watchsubdir(watcher, p); err != nil { return err } @@ -129,17 +129,17 @@ func main() { for { select { case ev := <-watcher.Events: - if d, err := os.Lstat(ev.Name); err == nil && ev.Op & fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode() & os.ModeSymlink == 0 && d.Name() != ".tmp" { + if d, err := os.Lstat(ev.Name); err == nil && ev.Op&fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode()&os.ModeSymlink == 0 && d.Name() != ".tmp" { // Register new subdirectory if err := watchsubdir(watcher, ev.Name); err != nil { log.Println(err) } - } else if ev.Op & watchedNotify == watchedNotify && d.Mode().IsRegular() { + } else if ev.Op&watchedNotify == watchedNotify && d.Mode().IsRegular() { if *debugINotify { log.Println("Treating event:", ev, "for", ev.Name) } go treat(ev.Name) - } else if ev.Op & fsnotify.Write == fsnotify.Write { + } else if ev.Op&fsnotify.Write == fsnotify.Write { log.Println("FSNOTIFY WRITE SEEN. Prefer looking at them, as it appears files are not atomically moved.") watchedNotify = fsnotify.Write } else if *debugINotify { diff --git a/backend/registration.go b/backend/registration.go index 3ed626ce..3e6f27d4 100644 --- a/backend/registration.go +++ b/backend/registration.go @@ -14,7 +14,7 @@ import ( var ( allowRegistration = false - canJoinTeam = false + canJoinTeam = false ) type uTeamRegistration struct { @@ -42,14 +42,8 @@ func registrationProcess(team fic.Team, members []fic.Member, team_id string) { log.Println("[ERR]", err) } - go func() { - if err := genTeamMyFile(team); err != nil { - log.Println("Team generation error: ", err) - } - if err := genTeamsFile(); err != nil { - log.Println("teams.json generation error: ", err) - } - }() + genTeamQueue <- &team + appendGenQueue(genStruct{Type: GenTeams}) } func treatRegistration(pathname string, team_id string) { @@ -88,11 +82,7 @@ func treatRegistration(pathname string, team_id string) { log.Println("[WRN] Unable to create event:", err) } - go func() { - if err := genEventsFile(); err != nil { - log.Println("events.json generation error: ", err) - } - }() + appendGenQueue(genStruct{Type: GenEvents}) } } } diff --git a/backend/rename.go b/backend/rename.go index 6c605c49..ed294dad 100644 --- a/backend/rename.go +++ b/backend/rename.go @@ -3,8 +3,8 @@ package main import ( "encoding/json" "fmt" - "log" "io/ioutil" + "log" "os" "regexp" @@ -28,15 +28,11 @@ func treatRename(pathname string, team fic.Team) { if _, err := team.Update(); err != nil { log.Println("[WRN] Unable to change team name:", err) } - if err := genTeamMyFile(team); err != nil { - log.Println("my-", team.Id, ".json generation error: ", err) - } + genTeamQueue <- &team if _, err := fic.NewEvent(fmt.Sprintf("Souhaitons bonne chance à l'équipe %s qui vient de nous rejoindre !", team.Name), "info"); err != nil { log.Println("[WRN] Unable to create event:", err) } - if err := genEventsFile(); err != nil { - log.Println("events.json generation error: ", err) - } + appendGenQueue(genStruct{Type: GenEvents}) if err := os.Remove(pathname); err != nil { log.Println("[ERR]", err) } diff --git a/backend/submission.go b/backend/submission.go index 6486f8a3..4b317409 100644 --- a/backend/submission.go +++ b/backend/submission.go @@ -109,7 +109,7 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) { solved, err := exercice.CheckResponse(cksum[:], responses.Keys, responses.MCQs, team) if err != nil { log.Println(id, "[ERR] Unable to CheckResponse:", err) - genTeamMyFile(team) + genTeamQueue <- &team return } @@ -130,7 +130,9 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) { } else if _, err := fic.NewEvent(fmt.Sprintf("L'équipe %s a résolu le %de défi %s !", team.Name, lvl, theme.Name), "success"); err != nil { log.Println(id, "[WRN] Unable to create event:", err) } - genTeamAll(team) + genTeamQueue <- &team + appendGenQueue(genStruct{id, GenThemes}) + appendGenQueue(genStruct{id, GenTeams}) } else { log.Printf("%s Team %d submit an invalid solution for exercice %d (%s : %s)\n", id, team.Id, exercice.Id, theme.Name, exercice.Title) @@ -142,10 +144,8 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) { log.Println(id, "[WRN] Unable to create event:", err) } } - genTeamMyFile(team) + genTeamQueue <- &team } - if err := genEventsFile(); err != nil { - log.Println("events.json generation error: ", err) - } + appendGenQueue(genStruct{id, GenEvents}) }