backend: multithread generation
This commit is contained in:
parent
3bcac39f5f
commit
2b75287d16
@ -2,15 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
)
|
||||
|
||||
type wantChoices struct {
|
||||
FlagId int64 `json:"id"`
|
||||
FlagId int64 `json:"id"`
|
||||
}
|
||||
|
||||
func treatWantChoices(pathname string, team fic.Team) {
|
||||
@ -34,9 +34,7 @@ func treatWantChoices(pathname string, team fic.Team) {
|
||||
} else if err = team.DisplayChoices(flag); err != nil {
|
||||
log.Println("[ERR]", err)
|
||||
} else {
|
||||
if err = genTeamMyFile(team); err != nil {
|
||||
log.Println("my-", team.Id, ".json generation error: ", err)
|
||||
}
|
||||
genTeamQueue <- &team
|
||||
if err = os.Remove(pathname); err != nil {
|
||||
log.Println("[ERR]", err)
|
||||
}
|
||||
|
@ -8,12 +8,92 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
)
|
||||
|
||||
type GenerateType int
|
||||
|
||||
const (
|
||||
GenPublic GenerateType = iota
|
||||
GenEvents
|
||||
GenTeams
|
||||
GenThemes
|
||||
)
|
||||
|
||||
type genStruct struct {
|
||||
Id string
|
||||
Type GenerateType
|
||||
}
|
||||
|
||||
var genTeamQueue chan *fic.Team
|
||||
var genQueue chan genStruct
|
||||
var inQueueMutex sync.RWMutex
|
||||
var inGenQueue map[GenerateType]bool
|
||||
|
||||
func init() {
|
||||
genTeamQueue = make(chan *fic.Team)
|
||||
genQueue = make(chan genStruct)
|
||||
inGenQueue = map[GenerateType]bool{}
|
||||
|
||||
for i := runtime.NumCPU(); i > 0; i-- {
|
||||
go consumer()
|
||||
}
|
||||
}
|
||||
|
||||
func appendGenQueue(gs genStruct) {
|
||||
inQueueMutex.RLock()
|
||||
if v, ok := inGenQueue[gs.Type]; !ok || !v {
|
||||
inQueueMutex.RUnlock()
|
||||
|
||||
inQueueMutex.Lock()
|
||||
inGenQueue[gs.Type] = true
|
||||
inQueueMutex.Unlock()
|
||||
|
||||
genQueue <- gs
|
||||
} else {
|
||||
inQueueMutex.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
func consumer() {
|
||||
for {
|
||||
var id string
|
||||
var err error
|
||||
|
||||
select {
|
||||
case gt := <-genTeamQueue:
|
||||
err = genTeamMyFile(gt)
|
||||
|
||||
case gs := <-genQueue:
|
||||
id = gs.Id
|
||||
|
||||
inQueueMutex.Lock()
|
||||
inGenQueue[gs.Type] = false
|
||||
inQueueMutex.Unlock()
|
||||
|
||||
switch gs.Type {
|
||||
case GenPublic:
|
||||
err = genMyPublicFile()
|
||||
case GenEvents:
|
||||
err = genEventsFile()
|
||||
case GenTeams:
|
||||
err = genTeamsFile()
|
||||
case GenThemes:
|
||||
err = genThemesFile()
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Println(id, "[ERR] Unable to generate:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate my.json and wait.json for a given team
|
||||
func genTeamMyFile(team fic.Team) error {
|
||||
func genTeamMyFile(team *fic.Team) error {
|
||||
dirPath := path.Join(TeamsDir, fmt.Sprintf("%d", team.Id))
|
||||
|
||||
if s, err := os.Stat(dirPath); os.IsNotExist(err) {
|
||||
@ -22,7 +102,7 @@ func genTeamMyFile(team fic.Team) error {
|
||||
return errors.New(fmt.Sprintf("%s is not a directory", dirPath))
|
||||
}
|
||||
|
||||
if my, err := fic.MyJSONTeam(&team, true); err != nil {
|
||||
if my, err := fic.MyJSONTeam(team, true); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
@ -32,7 +112,7 @@ func genTeamMyFile(team fic.Team) error {
|
||||
|
||||
// Speed up generation when challenge is started
|
||||
if !ChStarted {
|
||||
if my, err := fic.MyJSONTeam(&team, false); err != nil {
|
||||
if my, err := fic.MyJSONTeam(team, false); err != nil {
|
||||
return err
|
||||
} else if j, err := json.Marshal(my); err != nil {
|
||||
return err
|
||||
@ -106,32 +186,17 @@ func genThemesFile() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func genTeamAll(team fic.Team) {
|
||||
if err := genThemesFile(); err != nil {
|
||||
log.Println("themes.json generation error: ", err)
|
||||
} else if err = genTeamsFile(); err != nil {
|
||||
log.Println("teams.json generation error: ", err)
|
||||
} else if err = genTeamMyFile(team); err != nil {
|
||||
log.Println("my.json(", team.Id, ") generation error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func genAll() {
|
||||
if err := genThemesFile(); err != nil {
|
||||
log.Println("themes.json generation error: ", err)
|
||||
} else if err = genTeamsFile(); err != nil {
|
||||
log.Println("teams.json generation error: ", err)
|
||||
} else if err = genEventsFile(); err != nil {
|
||||
log.Println("events.json generation error: ", err)
|
||||
} else if err = genMyPublicFile(); err != nil {
|
||||
log.Println("MyPublic generation error: ", err)
|
||||
} else if teams, err := fic.GetActiveTeams(); err != nil {
|
||||
appendGenQueue(genStruct{Type: GenThemes})
|
||||
appendGenQueue(genStruct{Type: GenTeams})
|
||||
appendGenQueue(genStruct{Type: GenEvents})
|
||||
appendGenQueue(genStruct{Type: GenPublic})
|
||||
|
||||
if teams, err := fic.GetActiveTeams(); err != nil {
|
||||
log.Println("Team retrieval error: ", err)
|
||||
} else {
|
||||
for _, team := range(teams) {
|
||||
if err = genTeamMyFile(team); err != nil {
|
||||
log.Println("Team generation error: ", err)
|
||||
}
|
||||
for _, team := range teams {
|
||||
genTeamQueue <- &team
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -3,15 +3,15 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"srs.epita.fr/fic-server/libfic"
|
||||
)
|
||||
|
||||
type askOpenHint struct {
|
||||
HintId int64 `json:"id"`
|
||||
HintId int64 `json:"id"`
|
||||
}
|
||||
|
||||
func treatOpeningHint(pathname string, team fic.Team) {
|
||||
@ -42,13 +42,8 @@ func treatOpeningHint(pathname string, team fic.Team) {
|
||||
log.Println("[WRN] Unable to create event:", err)
|
||||
}
|
||||
|
||||
|
||||
if err = genTeamMyFile(team); err != nil {
|
||||
log.Println("my-", team.Id, ".json generation error: ", err)
|
||||
}
|
||||
if err = genEventsFile(); err != nil {
|
||||
log.Println("events.json generation error: ", err)
|
||||
}
|
||||
genTeamQueue <- &team
|
||||
appendGenQueue(genStruct{Type: GenEvents})
|
||||
if err = os.Remove(pathname); err != nil {
|
||||
log.Println("[ERR]", err)
|
||||
}
|
||||
|
@ -31,7 +31,7 @@ func watchsubdir(watcher *fsnotify.Watcher, pathname string) error {
|
||||
} else {
|
||||
for _, d := range ds {
|
||||
p := path.Join(pathname, d.Name())
|
||||
if d.IsDir() && d.Name() != ".tmp" && d.Mode() & os.ModeSymlink == 0 {
|
||||
if d.IsDir() && d.Name() != ".tmp" && d.Mode()&os.ModeSymlink == 0 {
|
||||
if err := watchsubdir(watcher, p); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -129,17 +129,17 @@ func main() {
|
||||
for {
|
||||
select {
|
||||
case ev := <-watcher.Events:
|
||||
if d, err := os.Lstat(ev.Name); err == nil && ev.Op & fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode() & os.ModeSymlink == 0 && d.Name() != ".tmp" {
|
||||
if d, err := os.Lstat(ev.Name); err == nil && ev.Op&fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode()&os.ModeSymlink == 0 && d.Name() != ".tmp" {
|
||||
// Register new subdirectory
|
||||
if err := watchsubdir(watcher, ev.Name); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
} else if ev.Op & watchedNotify == watchedNotify && d.Mode().IsRegular() {
|
||||
} else if ev.Op&watchedNotify == watchedNotify && d.Mode().IsRegular() {
|
||||
if *debugINotify {
|
||||
log.Println("Treating event:", ev, "for", ev.Name)
|
||||
}
|
||||
go treat(ev.Name)
|
||||
} else if ev.Op & fsnotify.Write == fsnotify.Write {
|
||||
} else if ev.Op&fsnotify.Write == fsnotify.Write {
|
||||
log.Println("FSNOTIFY WRITE SEEN. Prefer looking at them, as it appears files are not atomically moved.")
|
||||
watchedNotify = fsnotify.Write
|
||||
} else if *debugINotify {
|
||||
|
@ -14,7 +14,7 @@ import (
|
||||
|
||||
var (
|
||||
allowRegistration = false
|
||||
canJoinTeam = false
|
||||
canJoinTeam = false
|
||||
)
|
||||
|
||||
type uTeamRegistration struct {
|
||||
@ -42,14 +42,8 @@ func registrationProcess(team fic.Team, members []fic.Member, team_id string) {
|
||||
log.Println("[ERR]", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := genTeamMyFile(team); err != nil {
|
||||
log.Println("Team generation error: ", err)
|
||||
}
|
||||
if err := genTeamsFile(); err != nil {
|
||||
log.Println("teams.json generation error: ", err)
|
||||
}
|
||||
}()
|
||||
genTeamQueue <- &team
|
||||
appendGenQueue(genStruct{Type: GenTeams})
|
||||
}
|
||||
|
||||
func treatRegistration(pathname string, team_id string) {
|
||||
@ -88,11 +82,7 @@ func treatRegistration(pathname string, team_id string) {
|
||||
log.Println("[WRN] Unable to create event:", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := genEventsFile(); err != nil {
|
||||
log.Println("events.json generation error: ", err)
|
||||
}
|
||||
}()
|
||||
appendGenQueue(genStruct{Type: GenEvents})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -3,8 +3,8 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
@ -28,15 +28,11 @@ func treatRename(pathname string, team fic.Team) {
|
||||
if _, err := team.Update(); err != nil {
|
||||
log.Println("[WRN] Unable to change team name:", err)
|
||||
}
|
||||
if err := genTeamMyFile(team); err != nil {
|
||||
log.Println("my-", team.Id, ".json generation error: ", err)
|
||||
}
|
||||
genTeamQueue <- &team
|
||||
if _, err := fic.NewEvent(fmt.Sprintf("Souhaitons bonne chance à l'équipe <strong>%s</strong> qui vient de nous rejoindre !", team.Name), "info"); err != nil {
|
||||
log.Println("[WRN] Unable to create event:", err)
|
||||
}
|
||||
if err := genEventsFile(); err != nil {
|
||||
log.Println("events.json generation error: ", err)
|
||||
}
|
||||
appendGenQueue(genStruct{Type: GenEvents})
|
||||
if err := os.Remove(pathname); err != nil {
|
||||
log.Println("[ERR]", err)
|
||||
}
|
||||
|
@ -109,7 +109,7 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
|
||||
solved, err := exercice.CheckResponse(cksum[:], responses.Keys, responses.MCQs, team)
|
||||
if err != nil {
|
||||
log.Println(id, "[ERR] Unable to CheckResponse:", err)
|
||||
genTeamMyFile(team)
|
||||
genTeamQueue <- &team
|
||||
return
|
||||
}
|
||||
|
||||
@ -130,7 +130,9 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
|
||||
} else if _, err := fic.NewEvent(fmt.Sprintf("L'équipe %s a résolu le <strong>%d<sup>e</sup></strong> défi %s !", team.Name, lvl, theme.Name), "success"); err != nil {
|
||||
log.Println(id, "[WRN] Unable to create event:", err)
|
||||
}
|
||||
genTeamAll(team)
|
||||
genTeamQueue <- &team
|
||||
appendGenQueue(genStruct{id, GenThemes})
|
||||
appendGenQueue(genStruct{id, GenTeams})
|
||||
} else {
|
||||
log.Printf("%s Team %d submit an invalid solution for exercice %d (%s : %s)\n", id, team.Id, exercice.Id, theme.Name, exercice.Title)
|
||||
|
||||
@ -142,10 +144,8 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
|
||||
log.Println(id, "[WRN] Unable to create event:", err)
|
||||
}
|
||||
}
|
||||
genTeamMyFile(team)
|
||||
genTeamQueue <- &team
|
||||
}
|
||||
|
||||
if err := genEventsFile(); err != nil {
|
||||
log.Println("events.json generation error: ", err)
|
||||
}
|
||||
appendGenQueue(genStruct{id, GenEvents})
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user