fic
/
server
2
0
Fork 0
Code Issues Pull Requests 10 Releases Wiki Activity

generator: Can perform synchronous generation

This commit is contained in:
nemunaire 2023-07-10 12:10:03 +02:00
parent ec98e521dc
commit 1769938205
13 changed files with 214 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
admin/api/claim.go
View File

@ -1,25 +1,21 @@
package api
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"path"
"strconv"
"strings"
"time"
"srs.epita.fr/fic-server/admin/generation"
"srs.epita.fr/fic-server/libfic"
"github.com/gin-gonic/gin"
)
var GeneratorSocket string
func declareClaimsRoutes(router *gin.RouterGroup) {
// Tasks
router.GET("/claims", getClaims)
@ -292,7 +288,7 @@ func clearClaims(c *gin.Context) {
}
func generateTeamIssuesFile(team fic.Team) error {
if GeneratorSocket == "" {
if generation.GeneratorSocket == "" {
if my, err := team.MyIssueFile(); err != nil {
return fmt.Errorf("Unable to generate issue FILE (tid=%d): %w", team.Id, err)
} else if j, err := json.Marshal(my); err != nil {
@ -301,35 +297,16 @@ func generateTeamIssuesFile(team fic.Team) error {
return fmt.Errorf("Unable to write issues' file: %w", err)
}
} else {
buf, err := json.Marshal(fic.GenStruct{Type: fic.GenTeamIssues, TeamId: team.Id})
if err != nil {
return fmt.Errorf("Something is wrong with JSON encoder: %w", err)
}
sockType := "unix"
if strings.Contains(GeneratorSocket, ":") {
sockType = "tcp"
}
socket, err := net.Dial(sockType, GeneratorSocket)
resp, err := generation.PerformGeneration(fic.GenStruct{Type: fic.GenTeamIssues, TeamId: team.Id})
if err != nil {
return err
}
defer socket.Close()
defer resp.Body.Close()
httpClient := &http.Client{
Transport: &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
return socket, nil
},
},
if resp.StatusCode != http.StatusOK {
v, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("%s", string(v))
}
resp, err := httpClient.Post("http://localhost/enqueue", "application/json", bytes.NewReader(buf))
if err != nil {
return fmt.Errorf("Unable to enqueue new generation event: %w", err)
}
resp.Body.Close()
}
return nil
}

17
admin/api/settings.go
View File

@ -12,6 +12,7 @@ import (
"strconv"
"time"
"srs.epita.fr/fic-server/admin/generation"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
"srs.epita.fr/fic-server/settings"
@ -47,6 +48,7 @@ func declareSettingsRoutes(router *gin.RouterGroup) {
apiNextSettingsRoutes.DELETE("", deleteNextSettings)
router.POST("/reset", reset)
router.POST("/full-generation", fullGeneration)
router.GET("/prod", func(c *gin.Context) {
c.JSON(http.StatusOK, IsProductionEnv)
@ -80,6 +82,21 @@ func NextSettingsHandler(c *gin.Context) {
c.Next()
}
func fullGeneration(c *gin.Context) {
resp, err := generation.FullGeneration()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"errmsg": err.Error(),
})
}
defer resp.Body.Close()
v, _ := io.ReadAll(resp.Body)
c.JSON(resp.StatusCode, gin.H{
"errmsg": v,
})
}
func getROSettings(c *gin.Context) {
syncMtd := "Disabled"
if sync.GlobalImporter != nil {

7
admin/api/sync.go
View File

@ -8,9 +8,9 @@ import (
"path"
"strings"
"srs.epita.fr/fic-server/admin/generation"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
"srs.epita.fr/fic-server/settings"
"github.com/gin-gonic/gin"
)
@ -292,7 +292,10 @@ func autoSync(c *gin.Context) {
sync.EditDeepReport(&sync.SyncReport{Themes: map[string][]string{theTheme.Name: st}}, false)
sync.DeepSyncProgress = 255
settings.ForceRegeneration()
resp, err := generation.FullGeneration()
if err == nil {
defer resp.Body.Close()
}
c.JSON(http.StatusOK, st)
}

60
admin/generation/generation.go Normal file
View File

@ -0,0 +1,60 @@
package generation
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"strings"
"srs.epita.fr/fic-server/libfic"
)
var GeneratorSocket string
func doGeneration(uri string, contenttype string, buf io.Reader) (*http.Response, error) {
sockType := "unix"
if strings.Contains(GeneratorSocket, ":") {
sockType = "tcp"
}
socket, err := net.Dial(sockType, GeneratorSocket)
if err != nil {
return nil, err
}
defer socket.Close()
httpClient := &http.Client{
Transport: &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
return socket, nil
},
},
}
return httpClient.Post("http://localhost"+uri, contenttype, buf)
}
func EnqueueGeneration(gs fic.GenStruct) (*http.Response, error) {
buf, err := json.Marshal(gs)
if err != nil {
return nil, fmt.Errorf("Something is wrong with JSON encoder: %w", err)
}
return doGeneration("/enqueue", "application/json", bytes.NewReader(buf))
}
func PerformGeneration(gs fic.GenStruct) (*http.Response, error) {
buf, err := json.Marshal(gs)
if err != nil {
return nil, fmt.Errorf("Something is wrong with JSON encoder: %w", err)
}
return doGeneration("/perform", "application/json", bytes.NewReader(buf))
}
func FullGeneration() (*http.Response, error) {
return doGeneration("/full", "application/json", nil)
}

8
admin/index.go
View File

@ -128,7 +128,13 @@ const indextpl = `<!DOCTYPE html>
<div style="position: absolute;" id="circle1" class="circle-anim border-danger"></div>
<div style="position: absolute;" id="circle2" class="circle-anim border-info"></div>
</div>
<button type="button" class="mr-2 btn btn-sm" ng-class="{'btn-info':staticFilesNeedUpdate,'btn-secondary':!staticFilesNeedUpdate}" ng-click="regenerateStaticFiles()"><span class="glyphicon glyphicon-refresh" aria-hidden="true" title="Regénérer les fichiers statiques"></span><span ng-if="staticFilesNeedUpdate"> {{ "{{ staticFilesNeedUpdate }}" }}</span></button>
<button type="button" class="mr-2 btn btn-sm" ng-class="{'btn-info':staticFilesNeedUpdate,'btn-secondary':!staticFilesNeedUpdate}" ng-click="regenerateStaticFiles()" ng-disabled="staticRegenerationInProgress">
<span class="glyphicon glyphicon-refresh" aria-hidden="true" title="Regénérer les fichiers statiques" ng-show="!staticRegenerationInProgress"></span>
<div class="spinner-border spinner-border-sm" role="status" ng-show="staticRegenerationInProgress">
<span class="sr-only">Loading...</span>
</div>
<span ng-if="staticFilesNeedUpdate"> {{ "{{ staticFilesNeedUpdate }}" }}</span>
</button>
<span ng-show="startIn > 0">
Démarrage dans :
<span>{{"{{ startIn }}"}}</span>"

3
admin/main.go
View File

@ -14,6 +14,7 @@ import (
"syscall"
"srs.epita.fr/fic-server/admin/api"
"srs.epita.fr/fic-server/admin/generation"
"srs.epita.fr/fic-server/admin/pki"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
@ -108,7 +109,7 @@ func main() {
flag.StringVar(&api.DashboardDir, "dashbord", "./DASHBOARD", "Base directory where save public JSON files")
flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where load and save settings")
flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Base directory where found challenges files, local part")
flag.StringVar(&api.GeneratorSocket, "generator", "./GENERATOR/generator.socket", "Path to the generator socket (used to trigger issues.json generations, use an empty string to generate locally)")
flag.StringVar(&generation.GeneratorSocket, "generator", "./GENERATOR/generator.socket", "Path to the generator socket (used to trigger issues.json generations, use an empty string to generate locally)")
flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory,
"Base directory where found challenges files to import, local part")
flag.BoolVar(&localImporterSymlink, "localimportsymlink", localImporterSymlink,

22
admin/static/js/app.js
View File

@ -480,15 +480,16 @@ angular.module("FICApp")
}
$rootScope.staticFilesNeedUpdate = 0;
$rootScope.staticRegenerationInProgress = false;
$rootScope.regenerateStaticFiles = function() {
Settings.get().$promise.then(function(config) {
config.generation = (new Date()).toISOString();
config.$update(function() {
$rootScope.staticFilesNeedUpdate = 0;
$rootScope.addToast('success', "Regeneration in progress...");
}, function (response) {
$rootScope.addToast('success', 'An error occurs when saving settings:', response.data.errmsg);
})
$rootScope.staticRegenerationInProgress = true;
$http.post("api/full-generation").then(function(response) {
$rootScope.staticFilesNeedUpdate = 0;
$rootScope.staticRegenerationInProgress = false;
$rootScope.addToast('success', 'Regeneration ended');
}, function (response) {
$rootScope.staticRegenerationInProgress = false;
$rootScope.addToast('error', 'An error occurs when saving settings:', response.data.errmsg);
})
}
@ -633,11 +634,6 @@ angular.module("FICApp")
$scope.addToast('danger', 'An error occurs when saving settings:', response.data.errmsg);
});
}
$scope.regenerate = function() {
this.config.generation = (new Date()).toISOString();
$rootScope.settings.generation = new Date(this.config.generation);
$scope.saveSettings("Regeneration in progress...");
}
$scope.launchChallenge = function() {
var ts = Date.now() - Date.now() % 60000;
var d = new Date(ts + 120000);

10
admin/static/views/settings.html
View File

@ -31,8 +31,6 @@
</h2>
</div>
<div class="card-body pb-1">
<input type="hidden" class="form-control form-control-sm" id="lastRegeneration" ng-model="config.generation">
<div class="form-group row">
<label for="startTime" class="col-sm-3 col-form-label col-form-label-sm" ng-class="{'text-primary font-weight-bold': config.start != dist_config.start}">Début du challenge</label>
<div class="col-sm-9">
@ -382,7 +380,13 @@
<div class="col-3 pt-2 d-flex flex-column justify-content-between">
<div>
<div class="d-flex flex-column">
<button ng-click="regenerate()" class="btn btn-info my-1" type="button"><span class="glyphicon glyphicon-share" aria-hidden="true"></span> Regénérer les fichiers statiques</button>
<button ng-click="regenerateStaticFiles()" class="btn btn-info my-1" type="button" ng-disabled="staticRegenerationInProgress">
<span class="glyphicon glyphicon-share" aria-hidden="true" ng-show="!staticRegenerationInProgress"></span>
<div class="spinner-border spinner-border-sm" role="status" ng-show="staticRegenerationInProgress">
<span class="sr-only">Loading...</span>
</div>
Regénérer les fichiers statiques
</button>
<button ng-if="settings.wip" ng-click="switchToProd()" class="btn btn-danger my-1" type="button"><span class="glyphicon glyphicon-lock" aria-hidden="true"></span> Activer le mode challenge</button>
</div>
<hr>

11
admin/sync/full.go
View File

@ -2,13 +2,14 @@ package sync
import (
"encoding/json"
"io"
"log"
"os"
"sync"
"time"
"srs.epita.fr/fic-server/admin/generation"
"srs.epita.fr/fic-server/libfic"
"srs.epita.fr/fic-server/settings"
)
// DeepReportPath stores the path to the report generated during full recursive import.
@ -132,8 +133,14 @@ func SyncDeep(i Importer) (errs SyncReport) {
EditDeepReport(&errs, true)
if err := settings.ForceRegeneration(); err != nil {
resp, err := generation.FullGeneration()
if err != nil {
errs.Regeneration = append(errs.Regeneration, err.Error())
} else {
defer resp.Body.Close()
v, _ := io.ReadAll(resp.Body)
errs.Regeneration = append(errs.Regeneration, string(v))
}
DeepSyncProgress = 255

80
generator/generation.go
View File

@ -10,17 +10,18 @@ import (
"path"
"runtime"
"sync"
"time"
"srs.epita.fr/fic-server/libfic"
)
var parallelJobs = runtime.NumCPU()
var genQueue chan fic.GenStruct
var genQueue chan *fic.GenStruct
var inQueueMutex sync.RWMutex
var inGenQueue map[fic.GenerateType]bool
func init() {
genQueue = make(chan fic.GenStruct)
genQueue = make(chan *fic.GenStruct)
inGenQueue = map[fic.GenerateType]bool{}
}
@ -47,12 +48,54 @@ func enqueueHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, "OK", http.StatusOK)
}
func appendGenQueue(gs fic.GenStruct) {
if gs.Type == fic.GenTeam || gs.Type == fic.GenTeamIssues {
genQueue <- gs
func performHandler(w http.ResponseWriter, r *http.Request) {
var gs fic.GenStruct
dec := json.NewDecoder(r.Body)
err := dec.Decode(&gs)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Printf("POST /perform | %v", gs)
err = <-appendGenQueue(gs).GenEnded()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
} else {
http.Error(w, "OK", http.StatusOK)
}
}
func performFullResyncHandler(w http.ResponseWriter, r *http.Request) {
log.Printf("POST /full-resync started")
waitList := genAll()
var errs string
for _, e := range waitList {
err := <-e
if err != nil {
errs += err.Error() + "\n"
}
}
if errs != "" {
lastRegeneration = time.Now()
http.Error(w, errs, http.StatusInternalServerError)
} else {
http.Error(w, "done", http.StatusOK)
}
log.Printf("POST /full-resync done")
}
func appendGenQueue(gs fic.GenStruct) *fic.GenStruct {
if gs.Type == fic.GenTeam || gs.Type == fic.GenTeamIssues {
genQueue <- &gs
return &gs
}
// Append only if not already in queue
inQueueMutex.RLock()
if v, ok := inGenQueue[gs.Type]; !ok || !v {
@ -62,10 +105,12 @@ func appendGenQueue(gs fic.GenStruct) {
inGenQueue[gs.Type] = true
inQueueMutex.Unlock()
genQueue <- gs
genQueue <- &gs
} else {
inQueueMutex.RUnlock()
}
return &gs
}
func consumer() {
@ -98,6 +143,7 @@ func consumer() {
if err != nil {
log.Println(id, "[ERR] Unable to generate:", err)
}
gs.End(err)
}
}
@ -263,18 +309,26 @@ func genThemesFile() error {
return nil
}
func genAll() {
appendGenQueue(fic.GenStruct{Type: fic.GenThemes})
appendGenQueue(fic.GenStruct{Type: fic.GenTeams})
appendGenQueue(fic.GenStruct{Type: fic.GenEvents})
appendGenQueue(fic.GenStruct{Type: fic.GenPublic})
func genAll() (waitList []chan error) {
waitList = append(
waitList,
appendGenQueue(fic.GenStruct{Type: fic.GenThemes}).GenEnded(),
appendGenQueue(fic.GenStruct{Type: fic.GenTeams}).GenEnded(),
appendGenQueue(fic.GenStruct{Type: fic.GenEvents}).GenEnded(),
appendGenQueue(fic.GenStruct{Type: fic.GenPublic}).GenEnded(),
)
if teams, err := fic.GetActiveTeams(); err != nil {
log.Println("Team retrieval error: ", err)
} else {
for _, team := range teams {
appendGenQueue(fic.GenStruct{Type: fic.GenTeam, TeamId: team.Id})
appendGenQueue(fic.GenStruct{Type: fic.GenTeamIssues, TeamId: team.Id})
waitList = append(
waitList,
appendGenQueue(fic.GenStruct{Type: fic.GenTeam, TeamId: team.Id}).GenEnded(),
appendGenQueue(fic.GenStruct{Type: fic.GenTeamIssues, TeamId: team.Id}).GenEnded(),
)
}
}
return
}

11
generator/main.go
View File

@ -29,7 +29,7 @@ func reloadSettings(config *settings.Settings) {
fic.WChoiceCoefficient = config.WChoiceCurCoefficient
fic.ExerciceCurrentCoefficient = config.ExerciceCurCoefficient
ChStarted = config.Start.Unix() > 0 && time.Since(config.Start) >= 0
if lastRegeneration != config.Generation || fic.PartialValidation != config.PartialValidation || fic.UnlockedChallengeDepth != config.UnlockedChallengeDepth || fic.UnlockedChallengeUpTo != config.UnlockedChallengeUpTo || fic.DisplayAllFlags != config.DisplayAllFlags || fic.FirstBlood != config.FirstBlood || fic.SubmissionCostBase != config.SubmissionCostBase || fic.SubmissionUniqueness != config.SubmissionUniqueness || fic.DiscountedFactor != config.DiscountedFactor {
if fic.PartialValidation != config.PartialValidation || fic.UnlockedChallengeDepth != config.UnlockedChallengeDepth || fic.UnlockedChallengeUpTo != config.UnlockedChallengeUpTo || fic.DisplayAllFlags != config.DisplayAllFlags || fic.FirstBlood != config.FirstBlood || fic.SubmissionCostBase != config.SubmissionCostBase || fic.SubmissionUniqueness != config.SubmissionUniqueness || fic.DiscountedFactor != config.DiscountedFactor {
fic.PartialValidation = config.PartialValidation
fic.UnlockedChallengeDepth = config.UnlockedChallengeDepth
fic.UnlockedChallengeUpTo = config.UnlockedChallengeUpTo
@ -45,14 +45,17 @@ func reloadSettings(config *settings.Settings) {
if !skipInitialGeneration {
log.Println("Generating files...")
go func() {
genAll()
waitList := genAll()
for _, e := range waitList {
<-e
}
log.Println("Full generation done")
}()
lastRegeneration = time.Now()
} else {
skipInitialGeneration = false
log.Println("Regeneration skipped by option.")
}
lastRegeneration = config.Generation
} else {
log.Println("No change found. Skipping regeneration.")
}
@ -105,6 +108,8 @@ func main() {
}
http.HandleFunc("/enqueue", enqueueHandler)
http.HandleFunc("/perform", performHandler)
http.HandleFunc("/full", performFullResyncHandler)
// Serve pages
go func() {

16
libfic/generation.go
View File

@ -15,4 +15,20 @@ type GenStruct struct {
Id string `json:"id"`
Type GenerateType `json:"type"`
TeamId int64 `json:"team_id,omitempty"`
ended chan error
}
func (gs *GenStruct) GenEnded() chan error {
gs.ended = make(chan error, 1)
return gs.ended
}
func (gs *GenStruct) GetEnded() chan error {
return gs.ended
}
func (gs *GenStruct) End(err error) {
if gs.ended != nil {
gs.ended <- err
}
}

13
settings/settings.go
View File

@ -29,8 +29,6 @@ type Settings struct {
Start time.Time `json:"start"`
// End is the expected end time (if empty their is no end-date).
End *time.Time `json:"end,omitempty"`
// Generation is a value used to regenerate static files.
Generation time.Time `json:"generation"`
// ActivateTime is the time when the current file should be proceed.
ActivateTime time.Time `json:"activateTime"`
@ -130,17 +128,6 @@ func SaveSettings(path string, s interface{}) error {
}
}
// ForceRegeneration makes a small change to the settings structure in order to force the regeneration of all static files.
func ForceRegeneration() error {
location := path.Join(SettingsDir, SettingsFile)
if settings, err := ReadSettings(location); err != nil {
return err
} else {
settings.Generation = time.Now()
return SaveSettings(location, settings)
}
}
// LoadAndWatchSettings is the function you are looking for!
// Giving the location and a callback, this function will first call your reload function
// before returning (if the file can be parsed); then it starts watching modifications made to