backend: multithread generation

backend/choices.go

@@ -2,15 +2,15 @@ package main
 
 import (
 	"encoding/json"
-	"log"
 	"io/ioutil"
+	"log"
 	"os"
 
 	"srs.epita.fr/fic-server/libfic"
 )
 
 type wantChoices struct {
-	FlagId       int64 `json:"id"`
+	FlagId int64 `json:"id"`
 }
 
 func treatWantChoices(pathname string, team fic.Team) {
@@ -34,9 +34,7 @@ func treatWantChoices(pathname string, team fic.Team) {
 	} else if err = team.DisplayChoices(flag); err != nil {
 		log.Println("[ERR]", err)
 	} else {
-		if err = genTeamMyFile(team); err != nil {
+		genTeamQueue <- &team
-			log.Println("my-", team.Id, ".json generation error: ", err)
-		}
 		if err = os.Remove(pathname); err != nil {
 			log.Println("[ERR]", err)
 		}

backend/generation.go

@@ -8,12 +8,92 @@ import (
 	"log"
 	"os"
 	"path"
+	"runtime"
+	"sync"
 
 	"srs.epita.fr/fic-server/libfic"
 )
 
+type GenerateType int
+
+const (
+	GenPublic GenerateType = iota
+	GenEvents
+	GenTeams
+	GenThemes
+)
+
+type genStruct struct {
+	Id   string
+	Type GenerateType
+}
+
+var genTeamQueue chan *fic.Team
+var genQueue chan genStruct
+var inQueueMutex sync.RWMutex
+var inGenQueue map[GenerateType]bool
+
+func init() {
+	genTeamQueue = make(chan *fic.Team)
+	genQueue = make(chan genStruct)
+	inGenQueue = map[GenerateType]bool{}
+
+	for i := runtime.NumCPU(); i > 0; i-- {
+		go consumer()
+	}
+}
+
+func appendGenQueue(gs genStruct) {
+	inQueueMutex.RLock()
+	if v, ok := inGenQueue[gs.Type]; !ok || !v {
+		inQueueMutex.RUnlock()
+
+		inQueueMutex.Lock()
+		inGenQueue[gs.Type] = true
+		inQueueMutex.Unlock()
+
+		genQueue <- gs
+	} else {
+		inQueueMutex.RUnlock()
+	}
+}
+
+func consumer() {
+	for {
+		var id string
+		var err error
+
+		select {
+		case gt := <-genTeamQueue:
+			err = genTeamMyFile(gt)
+
+		case gs := <-genQueue:
+			id = gs.Id
+
+			inQueueMutex.Lock()
+			inGenQueue[gs.Type] = false
+			inQueueMutex.Unlock()
+
+			switch gs.Type {
+			case GenPublic:
+				err = genMyPublicFile()
+			case GenEvents:
+				err = genEventsFile()
+			case GenTeams:
+				err = genTeamsFile()
+			case GenThemes:
+				err = genThemesFile()
+			}
+		}
+
+		if err != nil {
+			log.Println(id, "[ERR] Unable to generate:", err)
+		}
+	}
+}
+
 // Generate my.json and wait.json for a given team
-func genTeamMyFile(team fic.Team) error {
+func genTeamMyFile(team *fic.Team) error {
 	dirPath := path.Join(TeamsDir, fmt.Sprintf("%d", team.Id))
 
 	if s, err := os.Stat(dirPath); os.IsNotExist(err) {
@@ -22,7 +102,7 @@ func genTeamMyFile(team fic.Team) error {
 		return errors.New(fmt.Sprintf("%s is not a directory", dirPath))
 	}
 
-	if my, err := fic.MyJSONTeam(&team, true); err != nil {
+	if my, err := fic.MyJSONTeam(team, true); err != nil {
 		return err
 	} else if j, err := json.Marshal(my); err != nil {
 		return err
@@ -32,7 +112,7 @@ func genTeamMyFile(team fic.Team) error {
 
 	// Speed up generation when challenge is started
 	if !ChStarted {
-		if my, err := fic.MyJSONTeam(&team, false); err != nil {
+		if my, err := fic.MyJSONTeam(team, false); err != nil {
 			return err
 		} else if j, err := json.Marshal(my); err != nil {
 			return err
@@ -106,32 +186,17 @@ func genThemesFile() error {
 	return nil
 }
 
-func genTeamAll(team fic.Team) {
-	if err := genThemesFile(); err != nil {
-		log.Println("themes.json generation error: ", err)
-	} else if err = genTeamsFile(); err != nil {
-		log.Println("teams.json generation error: ", err)
-	} else if err = genTeamMyFile(team); err != nil {
-		log.Println("my.json(", team.Id, ") generation error: ", err)
-	}
-}
-
 func genAll() {
-	if err := genThemesFile(); err != nil {
+	appendGenQueue(genStruct{Type: GenThemes})
-		log.Println("themes.json generation error: ", err)
+	appendGenQueue(genStruct{Type: GenTeams})
-	} else if err = genTeamsFile(); err != nil {
+	appendGenQueue(genStruct{Type: GenEvents})
-		log.Println("teams.json generation error: ", err)
+	appendGenQueue(genStruct{Type: GenPublic})
-	} else if err = genEventsFile(); err != nil {
+
-		log.Println("events.json generation error: ", err)
+	if teams, err := fic.GetActiveTeams(); err != nil {
-	} else if err = genMyPublicFile(); err != nil {
-		log.Println("MyPublic generation error: ", err)
-	} else if teams, err := fic.GetActiveTeams(); err != nil {
 		log.Println("Team retrieval error: ", err)
 	} else {
-		for _, team := range(teams) {
+		for _, team := range teams {
-			if err = genTeamMyFile(team); err != nil {
+			genTeamQueue <- &team
-				log.Println("Team generation error: ", err)
-			}
 		}
 	}
 }

backend/hint.go

@@ -3,15 +3,15 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"io/ioutil"
+	"log"
 	"os"
 
 	"srs.epita.fr/fic-server/libfic"
 )
 
 type askOpenHint struct {
-	HintId       int64 `json:"id"`
+	HintId int64 `json:"id"`
 }
 
 func treatOpeningHint(pathname string, team fic.Team) {
@@ -42,13 +42,8 @@ func treatOpeningHint(pathname string, team fic.Team) {
 			log.Println("[WRN] Unable to create event:", err)
 		}
 
-
+		genTeamQueue <- &team
-		if err = genTeamMyFile(team); err != nil {
+		appendGenQueue(genStruct{Type: GenEvents})
-			log.Println("my-", team.Id, ".json generation error: ", err)
-		}
-		if err = genEventsFile(); err != nil {
-			log.Println("events.json generation error: ", err)
-		}
 		if err = os.Remove(pathname); err != nil {
 			log.Println("[ERR]", err)
 		}

backend/main.go

@@ -31,7 +31,7 @@ func watchsubdir(watcher *fsnotify.Watcher, pathname string) error {
 	} else {
 		for _, d := range ds {
 			p := path.Join(pathname, d.Name())
-			if d.IsDir() && d.Name() != ".tmp" && d.Mode() & os.ModeSymlink == 0 {
+			if d.IsDir() && d.Name() != ".tmp" && d.Mode()&os.ModeSymlink == 0 {
 				if err := watchsubdir(watcher, p); err != nil {
 					return err
 				}
@@ -129,17 +129,17 @@ func main() {
 	for {
 		select {
 		case ev := <-watcher.Events:
-			if d, err := os.Lstat(ev.Name); err == nil && ev.Op & fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode() & os.ModeSymlink == 0 && d.Name() != ".tmp" {
+			if d, err := os.Lstat(ev.Name); err == nil && ev.Op&fsnotify.Create == fsnotify.Create && d.Mode().IsDir() && d.Mode()&os.ModeSymlink == 0 && d.Name() != ".tmp" {
 				// Register new subdirectory
 				if err := watchsubdir(watcher, ev.Name); err != nil {
 					log.Println(err)
 				}
-			} else if ev.Op & watchedNotify == watchedNotify && d.Mode().IsRegular() {
+			} else if ev.Op&watchedNotify == watchedNotify && d.Mode().IsRegular() {
 				if *debugINotify {
 					log.Println("Treating event:", ev, "for", ev.Name)
 				}
 				go treat(ev.Name)
-			} else if ev.Op & fsnotify.Write == fsnotify.Write {
+			} else if ev.Op&fsnotify.Write == fsnotify.Write {
 				log.Println("FSNOTIFY WRITE SEEN. Prefer looking at them, as it appears files are not atomically moved.")
 				watchedNotify = fsnotify.Write
 			} else if *debugINotify {

backend/registration.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	allowRegistration = false
-	canJoinTeam = false
+	canJoinTeam       = false
 )
 
 type uTeamRegistration struct {
@@ -42,14 +42,8 @@ func registrationProcess(team fic.Team, members []fic.Member, team_id string) {
 		log.Println("[ERR]", err)
 	}
 
-	go func() {
+	genTeamQueue <- &team
-		if err := genTeamMyFile(team); err != nil {
+	appendGenQueue(genStruct{Type: GenTeams})
-			log.Println("Team generation error: ", err)
-		}
-		if err := genTeamsFile(); err != nil {
-			log.Println("teams.json generation error: ", err)
-		}
-	}()
 }
 
 func treatRegistration(pathname string, team_id string) {
@@ -88,11 +82,7 @@ func treatRegistration(pathname string, team_id string) {
 				log.Println("[WRN] Unable to create event:", err)
 			}
 
-			go func() {
+			appendGenQueue(genStruct{Type: GenEvents})
-				if err := genEventsFile(); err != nil {
-					log.Println("events.json generation error: ", err)
-				}
-			}()
 		}
 	}
 }

backend/rename.go

@@ -3,8 +3,8 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"io/ioutil"
+	"log"
 	"os"
 	"regexp"
 
@@ -28,15 +28,11 @@ func treatRename(pathname string, team fic.Team) {
 		if _, err := team.Update(); err != nil {
 			log.Println("[WRN] Unable to change team name:", err)
 		}
-		if err := genTeamMyFile(team); err != nil {
+		genTeamQueue <- &team
-			log.Println("my-", team.Id, ".json generation error: ", err)
-		}
 		if _, err := fic.NewEvent(fmt.Sprintf("Souhaitons bonne chance à l'équipe <strong>%s</strong> qui vient de nous rejoindre&#160;!", team.Name), "info"); err != nil {
 			log.Println("[WRN] Unable to create event:", err)
 		}
-		if err := genEventsFile(); err != nil {
+		appendGenQueue(genStruct{Type: GenEvents})
-			log.Println("events.json generation error: ", err)
-		}
 		if err := os.Remove(pathname); err != nil {
 			log.Println("[ERR]", err)
 		}

backend/submission.go

@@ -109,7 +109,7 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
 	solved, err := exercice.CheckResponse(cksum[:], responses.Keys, responses.MCQs, team)
 	if err != nil {
 		log.Println(id, "[ERR] Unable to CheckResponse:", err)
-		genTeamMyFile(team)
+		genTeamQueue <- &team
 		return
 	}
 
@@ -130,7 +130,9 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
 		} else if _, err := fic.NewEvent(fmt.Sprintf("L'équipe %s a résolu le <strong>%d<sup>e</sup></strong> défi %s&#160;!", team.Name, lvl, theme.Name), "success"); err != nil {
 			log.Println(id, "[WRN] Unable to create event:", err)
 		}
-		genTeamAll(team)
+		genTeamQueue <- &team
+		appendGenQueue(genStruct{id, GenThemes})
+		appendGenQueue(genStruct{id, GenTeams})
 	} else {
 		log.Printf("%s Team %d submit an invalid solution for exercice %d (%s : %s)\n", id, team.Id, exercice.Id, theme.Name, exercice.Title)
 
@@ -142,10 +144,8 @@ func treatSubmission(pathname string, team fic.Team, exercice_id string) {
 				log.Println(id, "[WRN] Unable to create event:", err)
 			}
 		}
-		genTeamMyFile(team)
+		genTeamQueue <- &team
 	}
 
-	if err := genEventsFile(); err != nil {
+	appendGenQueue(genStruct{id, GenEvents})
-		log.Println("events.json generation error: ", err)
-	}
 }