fic/server
1
0
Fork 0
Code Issues Pull Requests 10 Releases Wiki Activity

evdist: New project to handle settings programming

Browse Source
This commit is contained in:
nemunaire 2022-05-30 17:25:07 +02:00
parent 0831ea6088
commit af6e86d4ef
9 changed files with 384 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.drone-manifest-fic-evdist.yml Normal file
View File

@ -0,0 +1,22 @@
image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
- image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: nemunaire/fic-evdist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

58
.drone.yml
View File

@ -18,6 +18,7 @@ steps:
- apk --no-cache add git
- go get -v -d srs.epita.fr/fic-server/admin
- go get -v -d srs.epita.fr/fic-server/backend
- go get -v -d srs.epita.fr/fic-server/evdist
- go get -v -d srs.epita.fr/fic-server/frontend
- go get -v -d srs.epita.fr/fic-server/dashboard
- go get -v -d srs.epita.fr/fic-server/repochecker
@ -31,6 +32,7 @@ steps:
- go vet -v -buildvcs=false -tags gitgo srs.epita.fr/fic-server/admin
- go vet -v -buildvcs=false srs.epita.fr/fic-server/admin
- go vet -v -buildvcs=false srs.epita.fr/fic-server/backend
- go vet -v -buildvcs=false srs.epita.fr/fic-server/evdist
- go vet -v -buildvcs=false srs.epita.fr/fic-server/frontend
- go vet -v -buildvcs=false srs.epita.fr/fic-server/dashboard
- go vet -v -buildvcs=false srs.epita.fr/fic-server/repochecker
@ -52,6 +54,13 @@ steps:
environment:
CGO_ENABLED: 0
- name: build evdist
image: golang:alpine
commands:
- go build -v -buildvcs=false -o deploy/evdist-${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} srs.epita.fr/fic-server/evdist
environment:
CGO_ENABLED: 0
- name: build frontend
image: golang:alpine
commands:
@ -151,6 +160,21 @@ steps:
branch:
- master
- name: docker evdist
image: plugins/docker
settings:
username:
from_secret: docker_username
password:
from_secret: docker_password
repo: nemunaire/fic-evdist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile-evdist
when:
branch:
- master
- name: docker frontend
image: plugins/docker
settings:
@ -297,6 +321,7 @@ steps:
- apk --no-cache add git
- go get -v -d srs.epita.fr/fic-server/admin
- go get -v -d srs.epita.fr/fic-server/backend
- go get -v -d srs.epita.fr/fic-server/evdist
- go get -v -d srs.epita.fr/fic-server/frontend
- go get -v -d srs.epita.fr/fic-server/dashboard
@ -314,6 +339,13 @@ steps:
environment:
CGO_ENABLED: 0
- name: build evdist
image: golang:alpine
commands:
- go build -v -buildvcs=false -o deploy/evdist-${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} srs.epita.fr/fic-server/evdist
environment:
CGO_ENABLED: 0
- name: build frontend
image: golang:alpine
commands:
@ -410,6 +442,21 @@ steps:
branch:
- master
- name: docker evdist
image: plugins/docker
settings:
username:
from_secret: docker_username
password:
from_secret: docker_password
repo: nemunaire/fic-evdist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile-evdist
when:
branch:
- master
- name: docker frontend
image: plugins/docker
settings:
@ -517,6 +564,17 @@ steps:
password:
from_secret: docker_password
- name: publish evdist
image: plugins/manifest
settings:
auto_tag: true
ignore_missing: true
spec: .drone-manifest-fic-evdist.yml
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish frontend
image: plugins/manifest
settings:

21
Dockerfile-evdist Normal file
View File

@ -0,0 +1,21 @@
FROM golang:1-alpine as gobuild
RUN apk add --no-cache git
WORKDIR /go/src/srs.epita.fr/fic-server/
COPY go.mod go.sum ./
COPY settings settings/
COPY evdist ./evdist/
RUN go get -d -v ./evdist && \
go build -v -buildvcs=false -o evdist/evdist ./evdist
FROM alpine:3.16
WORKDIR /srv
ENTRYPOINT ["/srv/evdist"]
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/evdist/evdist /srv/evdist

1
evdist/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
evdist

169
evdist/main.go Normal file
View File

@ -0,0 +1,169 @@
package main
import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path"
"strconv"
"strings"
"time"
"srs.epita.fr/fic-server/settings"
"gopkg.in/fsnotify.v1"
)
var SettingsDistDir = "./SETTINGSDIST/"
var TmpSettingsDirectory string
var TmpSettingsDistDirectory string
func watchsubdir(l *distList, watcher *fsnotify.Watcher, pathname string) error {
log.Println("Watch new directory:", pathname)
if err := watcher.Add(pathname); err != nil {
return err
}
if ds, err := ioutil.ReadDir(pathname); err != nil {
return err
} else {
for _, d := range ds {
p := path.Join(pathname, d.Name())
if d.Mode().IsRegular() && d.Name() != ".tmp" {
l.treat(p)
}
}
return nil
}
}
func main() {
flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where read settings")
flag.StringVar(&SettingsDistDir, "settingsDist", SettingsDistDir, "Directory where place settings to distribute")
var debugINotify = flag.Bool("debuginotify", false, "Show skipped inotofy events")
flag.Parse()
log.SetPrefix("[evdist] ")
settings.SettingsDir = path.Clean(settings.SettingsDir)
log.Println("Creating settingsDist directory...")
TmpSettingsDistDirectory = path.Join(SettingsDistDir, ".tmp")
if _, err := os.Stat(TmpSettingsDistDirectory); os.IsNotExist(err) {
if err = os.MkdirAll(TmpSettingsDistDirectory, 0755); err != nil {
log.Fatal("Unable to create settingsdist directory:", err)
}
}
TmpSettingsDirectory = path.Join(settings.SettingsDir, ".tmp")
if _, err := os.Stat(TmpSettingsDirectory); os.IsNotExist(err) {
if err = os.MkdirAll(TmpSettingsDirectory, 0755); err != nil {
log.Fatal("Unable to create settings directory:", err)
}
}
log.Println("Registering directory events...")
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
l := &distList{}
l.Timer = time.NewTimer(time.Minute)
if err := watchsubdir(l, watcher, settings.SettingsDir); err != nil {
log.Fatal(err)
}
watchedNotify := fsnotify.Create
for {
select {
case <-l.Timer.C:
if v := l.Pop(); v != nil {
log.Printf("TREATING DIFF: %v", v)
v, err = settings.ReadNextSettingsFile(path.Join(settings.SettingsDir, fmt.Sprintf("%d.json", v.Id)), v.Id)
if err != nil {
log.Printf("Unable to read %d.json: %s", v.Id, err.Error())
} else if cur_settings, err := settings.ReadSettings(path.Join(settings.SettingsDir, settings.SettingsFile)); err != nil {
log.Printf("Unable to read settings.json: %s", err.Error())
} else {
cur_settings = settings.MergeSettings(*cur_settings, v.Values)
if err = settings.SaveSettings(path.Join(TmpSettingsDirectory, "settings.json"), cur_settings); err != nil {
log.Printf("Unable to save settings.json to tmp dir: %s", err.Error())
} else if err = os.Rename(path.Join(TmpSettingsDirectory, "settings.json"), path.Join(settings.SettingsDir, "settings.json")); err != nil {
log.Printf("Unable to move settings.json to dest dir: %s", err.Error())
} else if err = os.Remove(path.Join(settings.SettingsDir, fmt.Sprintf("%d.json", v.Id))); err != nil {
log.Printf("Unable to remove initial diff file (%d.json): %s", v.Id, err.Error())
}
}
}
l.ResetTimer()
case ev := <-watcher.Events:
if d, err := os.Lstat(ev.Name); err == nil && ev.Op&watchedNotify == watchedNotify && d.Name() != ".tmp" && d.Mode().IsRegular() {
if *debugINotify {
log.Println("Treating event:", ev, "for", ev.Name)
}
go l.treat(ev.Name)
} else if ev.Op&fsnotify.Write == fsnotify.Write {
log.Println("FSNOTIFY WRITE SEEN. Prefer looking at them, as it appears files are not atomically moved.")
watchedNotify = fsnotify.Write
} else if *debugINotify {
log.Println("Skipped event:", ev, "for", ev.Name)
}
case err := <-watcher.Errors:
log.Println("error:", err)
}
}
}
func (l *distList) treat(raw_path string) {
bpath := path.Base(raw_path)
if bpath == "challenge.json" || bpath == "settings.json" {
log.Printf("Copying %s to SETTINGDIST...", bpath)
// Copy content through tmp file
fd, err := os.Open(raw_path)
if err != nil {
log.Printf("ERROR: Unable to open %s: %s", raw_path, err.Error())
return
}
defer fd.Close()
tmpfile, err := ioutil.TempFile(TmpSettingsDistDirectory, "")
if err != nil {
log.Printf("ERROR: Unable to create temporary file for %s: %s", bpath, err.Error())
return
}
_, err = io.Copy(tmpfile, fd)
tmpfile.Close()
if err != nil {
log.Printf("ERROR: Unable to copy content to temporary file (%s): %s", bpath, err.Error())
return
}
if err = os.Rename(tmpfile.Name(), path.Join(SettingsDistDir, bpath)); err != nil {
log.Println("ERROR: Unable to move file:", err)
return
}
} else if ts, err := strconv.ParseInt(strings.TrimSuffix(bpath, ".json"), 10, 64); err == nil {
activateTime := time.Unix(ts, 0)
log.Printf("Preparing %s: activation time at %s", bpath, activateTime)
l.AddEvent(&settings.NextSettingsFile{
Id: ts,
Date: activateTime,
})
} else {
log.Println("WARNING: Unknown file to treat: not a valid timestamp:", err.Error())
}
}

110
evdist/settings.go Normal file
View File

@ -0,0 +1,110 @@
package main
import (
"log"
"sync"
"time"
"srs.epita.fr/fic-server/settings"
)
// distList maintain a nextSettingsFile list up-to-date.
type distList struct {
List []*settings.NextSettingsFile
Lock sync.RWMutex
Timer *time.Timer
}
// NewDistList creates a distList from the given src directory
func NewDistList(src string) (*distList, error) {
list, err := settings.ListNextSettingsFiles()
if err != nil {
return nil, err
}
return &distList{List: list, Timer: time.NewTimer(time.Minute)}, nil
}
func (l *distList) TimerNextEvent() *time.Timer {
l.Lock.RLock()
defer l.Lock.RUnlock()
var min *time.Time
for _, f := range l.List {
if min == nil || f.Date.Before(*min) {
min = &f.Date
}
}
if min == nil {
return nil
}
if min == nil {
return nil
}
return time.NewTimer(time.Until(*min))
}
func (l *distList) AddEvent(nsf *settings.NextSettingsFile) {
l.Lock.Lock()
istop := len(l.List)
for i, n := range l.List {
if n.Id == nsf.Id {
return
} else if n.Date.After(nsf.Date) {
istop = i
break
}
}
l.List = append(l.List, nsf)
copy(l.List[istop+1:], l.List[istop:])
l.List[istop] = nsf
l.Lock.Unlock()
if istop == 0 {
l.ResetTimer()
}
}
func (l *distList) ResetTimer() {
l.Lock.RLock()
defer l.Lock.RUnlock()
if len(l.List) == 0 {
l.Timer.Reset(time.Minute)
} else {
l.Timer.Reset(time.Until(l.List[0].Date))
}
}
func (l *distList) Pop() *settings.NextSettingsFile {
l.Lock.Lock()
defer l.Lock.Unlock()
if len(l.List) == 0 {
return nil
}
if time.Now().Before(l.List[0].Date) {
return nil
}
ret := l.List[0]
l.List = l.List[1:]
return ret
}
func (l *distList) Print() {
l.Lock.RLock()
defer l.Lock.RUnlock()
log.Println("Seeing distlist")
for n, i := range l.List {
log.Printf("#%d: %v", n, *i)
}
}

7
frontend/main.go
View File

@ -41,13 +41,6 @@ func main() {
}
}
log.Println("Creating settingsDist directory...")
if _, err := os.Stat(SettingsDistDir); os.IsNotExist(err) {
if err = os.MkdirAll(SettingsDistDir, 0755); err != nil {
log.Fatal("Unable to create settingsdist directory:", err)
}
}
*prefix = strings.TrimRight(*prefix, "/")
// Load configuration

2
frontend/settings.go
View File

@ -12,8 +12,6 @@ import (
var startedFile = "started"
var SettingsDistDir = "./SETTINGSDIST/"
var touchTimer *time.Timer = nil
var challengeStart time.Time
var challengeEnd time.Time

5
settings/diff.go
View File

@ -3,7 +3,6 @@ package settings
import (
"encoding/json"
"fmt"
"log"
"os"
"path"
"reflect"
@ -67,6 +66,9 @@ func ListNextSettingsFiles() ([]*NextSettingsFile, error) {
var ret []*NextSettingsFile
for _, file := range files {
if len(file.Name()) < 10 {
continue
}
ts, err := strconv.ParseInt(file.Name()[:10], 10, 64)
if err == nil {
nsf, err := ReadNextSettingsFile(path.Join(SettingsDir, file.Name()), ts)
@ -108,7 +110,6 @@ func MergeSettings(current Settings, new map[string]interface{}) *Settings {
}
if v, ok := new[name]; ok {
log.Println(name, field.Name, v)
reflect.ValueOf(&current).Elem().FieldByName(field.Name).Set(reflect.ValueOf(v))
}
}