diff --git a/.drone-manifest-fic-evdist.yml b/.drone-manifest-fic-evdist.yml new file mode 100644 index 00000000..70b1e5c5 --- /dev/null +++ b/.drone-manifest-fic-evdist.yml @@ -0,0 +1,22 @@ +image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/.drone.yml b/.drone.yml index 83007dc6..a7fc62af 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,6 +18,7 @@ steps: - apk --no-cache add git - go get -v -d srs.epita.fr/fic-server/admin - go get -v -d srs.epita.fr/fic-server/backend + - go get -v -d srs.epita.fr/fic-server/evdist - go get -v -d srs.epita.fr/fic-server/frontend - go get -v -d srs.epita.fr/fic-server/dashboard - go get -v -d srs.epita.fr/fic-server/repochecker @@ -31,6 +32,7 @@ steps: - go vet -v -buildvcs=false -tags gitgo srs.epita.fr/fic-server/admin - go vet -v -buildvcs=false srs.epita.fr/fic-server/admin - go vet -v -buildvcs=false srs.epita.fr/fic-server/backend + - go vet -v -buildvcs=false srs.epita.fr/fic-server/evdist - go vet -v -buildvcs=false srs.epita.fr/fic-server/frontend - go vet -v -buildvcs=false srs.epita.fr/fic-server/dashboard - go vet -v -buildvcs=false srs.epita.fr/fic-server/repochecker @@ -52,6 +54,13 @@ steps: environment: CGO_ENABLED: 0 + - name: build evdist + image: golang:alpine + commands: + - go build -v -buildvcs=false -o deploy/evdist-${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} srs.epita.fr/fic-server/evdist + environment: + CGO_ENABLED: 0 + - name: build frontend image: golang:alpine commands: @@ -151,6 +160,21 @@ steps: branch: - master + - name: docker evdist + image: plugins/docker + settings: + username: + from_secret: docker_username + password: + from_secret: docker_password + repo: nemunaire/fic-evdist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile-evdist + when: + branch: + - master + - name: docker frontend image: plugins/docker settings: @@ -297,6 +321,7 @@ steps: - apk --no-cache add git - go get -v -d srs.epita.fr/fic-server/admin - go get -v -d srs.epita.fr/fic-server/backend + - go get -v -d srs.epita.fr/fic-server/evdist - go get -v -d srs.epita.fr/fic-server/frontend - go get -v -d srs.epita.fr/fic-server/dashboard @@ -314,6 +339,13 @@ steps: environment: CGO_ENABLED: 0 + - name: build evdist + image: golang:alpine + commands: + - go build -v -buildvcs=false -o deploy/evdist-${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} srs.epita.fr/fic-server/evdist + environment: + CGO_ENABLED: 0 + - name: build frontend image: golang:alpine commands: @@ -410,6 +442,21 @@ steps: branch: - master + - name: docker evdist + image: plugins/docker + settings: + username: + from_secret: docker_username + password: + from_secret: docker_password + repo: nemunaire/fic-evdist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile-evdist + when: + branch: + - master + - name: docker frontend image: plugins/docker settings: @@ -517,6 +564,17 @@ steps: password: from_secret: docker_password + - name: publish evdist + image: plugins/manifest + settings: + auto_tag: true + ignore_missing: true + spec: .drone-manifest-fic-evdist.yml + username: + from_secret: docker_username + password: + from_secret: docker_password + - name: publish frontend image: plugins/manifest settings: diff --git a/Dockerfile-evdist b/Dockerfile-evdist new file mode 100644 index 00000000..d70f2c78 --- /dev/null +++ b/Dockerfile-evdist @@ -0,0 +1,21 @@ +FROM golang:1-alpine as gobuild + +RUN apk add --no-cache git + +WORKDIR /go/src/srs.epita.fr/fic-server/ + +COPY go.mod go.sum ./ +COPY settings settings/ +COPY evdist ./evdist/ + +RUN go get -d -v ./evdist && \ + go build -v -buildvcs=false -o evdist/evdist ./evdist + + +FROM alpine:3.16 + +WORKDIR /srv + +ENTRYPOINT ["/srv/evdist"] + +COPY --from=gobuild /go/src/srs.epita.fr/fic-server/evdist/evdist /srv/evdist diff --git a/evdist/.gitignore b/evdist/.gitignore new file mode 100644 index 00000000..f2b553d5 --- /dev/null +++ b/evdist/.gitignore @@ -0,0 +1 @@ +evdist \ No newline at end of file diff --git a/evdist/main.go b/evdist/main.go new file mode 100644 index 00000000..2aec1960 --- /dev/null +++ b/evdist/main.go @@ -0,0 +1,169 @@ +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "strconv" + "strings" + "time" + + "srs.epita.fr/fic-server/settings" + + "gopkg.in/fsnotify.v1" +) + +var SettingsDistDir = "./SETTINGSDIST/" +var TmpSettingsDirectory string +var TmpSettingsDistDirectory string + +func watchsubdir(l *distList, watcher *fsnotify.Watcher, pathname string) error { + log.Println("Watch new directory:", pathname) + if err := watcher.Add(pathname); err != nil { + return err + } + + if ds, err := ioutil.ReadDir(pathname); err != nil { + return err + } else { + for _, d := range ds { + p := path.Join(pathname, d.Name()) + if d.Mode().IsRegular() && d.Name() != ".tmp" { + l.treat(p) + } + } + return nil + } +} + +func main() { + flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where read settings") + flag.StringVar(&SettingsDistDir, "settingsDist", SettingsDistDir, "Directory where place settings to distribute") + var debugINotify = flag.Bool("debuginotify", false, "Show skipped inotofy events") + flag.Parse() + + log.SetPrefix("[evdist] ") + + settings.SettingsDir = path.Clean(settings.SettingsDir) + + log.Println("Creating settingsDist directory...") + TmpSettingsDistDirectory = path.Join(SettingsDistDir, ".tmp") + if _, err := os.Stat(TmpSettingsDistDirectory); os.IsNotExist(err) { + if err = os.MkdirAll(TmpSettingsDistDirectory, 0755); err != nil { + log.Fatal("Unable to create settingsdist directory:", err) + } + } + + TmpSettingsDirectory = path.Join(settings.SettingsDir, ".tmp") + if _, err := os.Stat(TmpSettingsDirectory); os.IsNotExist(err) { + if err = os.MkdirAll(TmpSettingsDirectory, 0755); err != nil { + log.Fatal("Unable to create settings directory:", err) + } + } + + log.Println("Registering directory events...") + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + l := &distList{} + l.Timer = time.NewTimer(time.Minute) + + if err := watchsubdir(l, watcher, settings.SettingsDir); err != nil { + log.Fatal(err) + } + + watchedNotify := fsnotify.Create + + for { + select { + case <-l.Timer.C: + if v := l.Pop(); v != nil { + log.Printf("TREATING DIFF: %v", v) + + v, err = settings.ReadNextSettingsFile(path.Join(settings.SettingsDir, fmt.Sprintf("%d.json", v.Id)), v.Id) + if err != nil { + log.Printf("Unable to read %d.json: %s", v.Id, err.Error()) + } else if cur_settings, err := settings.ReadSettings(path.Join(settings.SettingsDir, settings.SettingsFile)); err != nil { + log.Printf("Unable to read settings.json: %s", err.Error()) + } else { + cur_settings = settings.MergeSettings(*cur_settings, v.Values) + + if err = settings.SaveSettings(path.Join(TmpSettingsDirectory, "settings.json"), cur_settings); err != nil { + log.Printf("Unable to save settings.json to tmp dir: %s", err.Error()) + } else if err = os.Rename(path.Join(TmpSettingsDirectory, "settings.json"), path.Join(settings.SettingsDir, "settings.json")); err != nil { + log.Printf("Unable to move settings.json to dest dir: %s", err.Error()) + } else if err = os.Remove(path.Join(settings.SettingsDir, fmt.Sprintf("%d.json", v.Id))); err != nil { + log.Printf("Unable to remove initial diff file (%d.json): %s", v.Id, err.Error()) + } + } + } + l.ResetTimer() + case ev := <-watcher.Events: + if d, err := os.Lstat(ev.Name); err == nil && ev.Op&watchedNotify == watchedNotify && d.Name() != ".tmp" && d.Mode().IsRegular() { + if *debugINotify { + log.Println("Treating event:", ev, "for", ev.Name) + } + go l.treat(ev.Name) + } else if ev.Op&fsnotify.Write == fsnotify.Write { + log.Println("FSNOTIFY WRITE SEEN. Prefer looking at them, as it appears files are not atomically moved.") + watchedNotify = fsnotify.Write + } else if *debugINotify { + log.Println("Skipped event:", ev, "for", ev.Name) + } + case err := <-watcher.Errors: + log.Println("error:", err) + } + } +} + +func (l *distList) treat(raw_path string) { + bpath := path.Base(raw_path) + + if bpath == "challenge.json" || bpath == "settings.json" { + log.Printf("Copying %s to SETTINGDIST...", bpath) + // Copy content through tmp file + fd, err := os.Open(raw_path) + if err != nil { + log.Printf("ERROR: Unable to open %s: %s", raw_path, err.Error()) + return + } + defer fd.Close() + + tmpfile, err := ioutil.TempFile(TmpSettingsDistDirectory, "") + if err != nil { + log.Printf("ERROR: Unable to create temporary file for %s: %s", bpath, err.Error()) + return + } + + _, err = io.Copy(tmpfile, fd) + tmpfile.Close() + + if err != nil { + log.Printf("ERROR: Unable to copy content to temporary file (%s): %s", bpath, err.Error()) + return + } + + if err = os.Rename(tmpfile.Name(), path.Join(SettingsDistDir, bpath)); err != nil { + log.Println("ERROR: Unable to move file:", err) + return + } + } else if ts, err := strconv.ParseInt(strings.TrimSuffix(bpath, ".json"), 10, 64); err == nil { + activateTime := time.Unix(ts, 0) + + log.Printf("Preparing %s: activation time at %s", bpath, activateTime) + + l.AddEvent(&settings.NextSettingsFile{ + Id: ts, + Date: activateTime, + }) + } else { + log.Println("WARNING: Unknown file to treat: not a valid timestamp:", err.Error()) + } +} diff --git a/evdist/settings.go b/evdist/settings.go new file mode 100644 index 00000000..caba3e5d --- /dev/null +++ b/evdist/settings.go @@ -0,0 +1,110 @@ +package main + +import ( + "log" + "sync" + "time" + + "srs.epita.fr/fic-server/settings" +) + +// distList maintain a nextSettingsFile list up-to-date. +type distList struct { + List []*settings.NextSettingsFile + Lock sync.RWMutex + Timer *time.Timer +} + +// NewDistList creates a distList from the given src directory +func NewDistList(src string) (*distList, error) { + list, err := settings.ListNextSettingsFiles() + if err != nil { + return nil, err + } + return &distList{List: list, Timer: time.NewTimer(time.Minute)}, nil +} + +func (l *distList) TimerNextEvent() *time.Timer { + l.Lock.RLock() + defer l.Lock.RUnlock() + + var min *time.Time + + for _, f := range l.List { + if min == nil || f.Date.Before(*min) { + min = &f.Date + } + } + + if min == nil { + return nil + } + + if min == nil { + return nil + } + + return time.NewTimer(time.Until(*min)) +} + +func (l *distList) AddEvent(nsf *settings.NextSettingsFile) { + l.Lock.Lock() + + istop := len(l.List) + for i, n := range l.List { + if n.Id == nsf.Id { + return + } else if n.Date.After(nsf.Date) { + istop = i + break + } + } + + l.List = append(l.List, nsf) + copy(l.List[istop+1:], l.List[istop:]) + l.List[istop] = nsf + + l.Lock.Unlock() + + if istop == 0 { + l.ResetTimer() + } +} + +func (l *distList) ResetTimer() { + l.Lock.RLock() + defer l.Lock.RUnlock() + + if len(l.List) == 0 { + l.Timer.Reset(time.Minute) + } else { + l.Timer.Reset(time.Until(l.List[0].Date)) + } +} + +func (l *distList) Pop() *settings.NextSettingsFile { + l.Lock.Lock() + defer l.Lock.Unlock() + + if len(l.List) == 0 { + return nil + } + + if time.Now().Before(l.List[0].Date) { + return nil + } + + ret := l.List[0] + l.List = l.List[1:] + return ret +} + +func (l *distList) Print() { + l.Lock.RLock() + defer l.Lock.RUnlock() + + log.Println("Seeing distlist") + for n, i := range l.List { + log.Printf("#%d: %v", n, *i) + } +} diff --git a/frontend/main.go b/frontend/main.go index 8b307de9..75827103 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -41,13 +41,6 @@ func main() { } } - log.Println("Creating settingsDist directory...") - if _, err := os.Stat(SettingsDistDir); os.IsNotExist(err) { - if err = os.MkdirAll(SettingsDistDir, 0755); err != nil { - log.Fatal("Unable to create settingsdist directory:", err) - } - } - *prefix = strings.TrimRight(*prefix, "/") // Load configuration diff --git a/frontend/settings.go b/frontend/settings.go index 3455c81f..311b3edb 100644 --- a/frontend/settings.go +++ b/frontend/settings.go @@ -12,8 +12,6 @@ import ( var startedFile = "started" -var SettingsDistDir = "./SETTINGSDIST/" - var touchTimer *time.Timer = nil var challengeStart time.Time var challengeEnd time.Time diff --git a/settings/diff.go b/settings/diff.go index 34a2f855..f5535b58 100644 --- a/settings/diff.go +++ b/settings/diff.go @@ -3,7 +3,6 @@ package settings import ( "encoding/json" "fmt" - "log" "os" "path" "reflect" @@ -67,6 +66,9 @@ func ListNextSettingsFiles() ([]*NextSettingsFile, error) { var ret []*NextSettingsFile for _, file := range files { + if len(file.Name()) < 10 { + continue + } ts, err := strconv.ParseInt(file.Name()[:10], 10, 64) if err == nil { nsf, err := ReadNextSettingsFile(path.Join(SettingsDir, file.Name()), ts) @@ -108,7 +110,6 @@ func MergeSettings(current Settings, new map[string]interface{}) *Settings { } if v, ok := new[name]; ok { - log.Println(name, field.Name, v) reflect.ValueOf(¤t).Elem().FieldByName(field.Name).Set(reflect.ValueOf(v)) } }