server/admin/sync/exercices.go
Pierre-Olivier Mercier 651d428223
All checks were successful
continuous-integration/drone/push Build is passing
sync: Prefer challenge.toml over challenge.txt
2024-05-16 13:09:13 +02:00

527 lines
17 KiB
Go
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package sync
import (
"bytes"
"fmt"
"hash/adler32"
"image"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"github.com/BurntSushi/toml"
"github.com/gin-gonic/gin"
"github.com/yuin/goldmark"
"go.uber.org/multierr"
"srs.epita.fr/fic-server/libfic"
)
// Set AllowWIPExercice if WIP exercices are accepted.
var AllowWIPExercice bool = false
func fixnbsp(s string) string {
return strings.Replace(strings.Replace(strings.Replace(s, " ?", " ?", -1), " !", " !", -1), " :", " :", -1)
}
// GetExercices returns all exercice directories existing in a given theme, considering the given Importer.
func GetExercices(i Importer, theme *fic.Theme) ([]string, error) {
var exercices []string
if len(theme.Path) == 0 {
return []string{}, nil
} else if dirs, err := i.ListDir(theme.Path); err != nil {
return []string{}, err
} else {
for _, dir := range dirs {
if _, err := i.ListDir(path.Join(theme.Path, dir)); err == nil {
if dir[0] != '.' && strings.Contains(dir, "-") {
exercices = append(exercices, dir)
}
}
}
}
return exercices, nil
}
func buildDependancyMap(i Importer, theme *fic.Theme) (dmap map[int64]*fic.Exercice, err error) {
var exercices []string
if exercices, err = GetExercices(i, theme); err != nil {
return
} else {
dmap = map[int64]*fic.Exercice{}
for _, edir := range exercices {
var eid int
var ename string
eid, ename, err = parseExerciceDirname(edir)
if err != nil {
err = nil
continue
}
var e *fic.Exercice
e, err = theme.GetExerciceByTitle(ename)
if err != nil {
return
}
dmap[int64(eid)] = e
}
return
}
}
func parseExerciceDirname(edir string) (eid int, ename string, err error) {
edir_splt := strings.SplitN(edir, "-", 2)
if len(edir_splt) != 2 {
err = fmt.Errorf("%q is not a valid exercice directory: missing id prefix", edir)
return
}
eid, err = strconv.Atoi(edir_splt[0])
if err != nil {
err = fmt.Errorf("%q: invalid exercice identifier: %s", edir, err)
return
}
// ID 0: peak a deterministic-random-ordered ID instead
if eid == 0 {
eid = int(adler32.Checksum([]byte(edir_splt[1])))
}
ename = edir_splt[1]
return
}
// BuildExercice creates an Exercice from a given importer.
func BuildExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, p ExerciceParams, eid int, exceptions *CheckExceptions, edir string, errs error) {
e = &fic.Exercice{}
e.Path = epath
edir = path.Base(epath)
var err error
eid, e.Title, err = parseExerciceDirname(edir)
if err != nil {
// Ignore eid if we are certain this is an exercice directory, eid will be 0
if !i.Exists(path.Join(epath, "title.txt")) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to parse exercice directory: %w", err), theme))
return nil, p, eid, exceptions_in, edir, errs
}
}
// Get exceptions
exceptions = LoadExerciceException(i, theme, e, exceptions_in)
//log.Printf("Kept repochecker exceptions for this exercice: %v", exceptions)
if theme != nil {
e.Language = theme.Language
}
// Overwrite language if language.txt exists
if language, err := GetFileContent(i, path.Join(epath, "language.txt")); err == nil {
language = strings.TrimSpace(language)
if strings.Contains(language, "\n") {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("language.txt: Language can't contain new lines"), theme))
} else {
e.Language = language
}
}
// Overwrite title if title.txt exists
if myTitle, err := GetFileContent(i, path.Join(epath, "title.txt")); err == nil {
myTitle = strings.TrimSpace(myTitle)
if strings.Contains(myTitle, "\n") {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title.txt: Title can't contain new lines"), theme))
} else {
e.Title = myTitle
}
}
// Character reserved for WIP exercices
if len(e.Title) > 0 && e.Title[0] == '%' {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title can't contain start by '%%'"), theme))
}
e.URLId = fic.ToURLid(e.Title)
e.Title = fixnbsp(e.Title)
if i.Exists(path.Join(epath, "AUTHORS.txt")) {
if authors, err := getAuthors(i, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get AUTHORS.txt: %w", err)))
} else {
// Format authors
e.Authors = strings.Join(authors, ", ")
}
}
// Process headline
if i.Exists(path.Join(epath, "headline.txt")) {
e.Headline, err = GetFileContent(i, path.Join(epath, "headline.txt"))
} else if i.Exists(path.Join(epath, "headline.md")) {
e.Headline, err = GetFileContent(i, path.Join(epath, "headline.md"))
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get exercice's headline: %w", err)))
}
if e.Headline != "" {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Headline, e.Language, exceptions.GetFileExceptions("headline.md", "headline.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("headline.md: %w", err)))
}
}
}
// Texts to format using Markdown
if i.Exists(path.Join(epath, "overview.txt")) {
e.Overview, err = GetFileContent(i, path.Join(epath, "overview.txt"))
} else if i.Exists(path.Join(epath, "overview.md")) {
e.Overview, err = GetFileContent(i, path.Join(epath, "overview.md"))
} else {
err = fmt.Errorf("Unable to find overview.txt nor overview.md")
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.txt: %s", err), theme))
} else {
e.Overview = fixnbsp(e.Overview)
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Overview, e.Language, exceptions.GetFileExceptions("overview.md", "overview.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: %w", err)))
}
}
var buf bytes.Buffer
if e.Headline == "" {
err := goldmark.Convert([]byte(strings.Split(e.Overview, "\n")[0]), &buf)
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating of the headline: %w", err), theme))
} else {
e.Headline = string(buf.Bytes())
}
}
if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating: %w", err), theme))
}
}
if i.Exists(path.Join(epath, "statement.txt")) {
e.Statement, err = GetFileContent(i, path.Join(epath, "statement.txt"))
} else if i.Exists(path.Join(epath, "statement.md")) {
e.Statement, err = GetFileContent(i, path.Join(epath, "statement.md"))
} else {
err = fmt.Errorf("Unable to find statement.txt nor statement.md")
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err), theme))
} else {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Statement, e.Language, exceptions.GetFileExceptions("statement.md", "statement.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err)))
}
}
if e.Statement, err = ProcessMarkdown(i, fixnbsp(e.Statement), epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: an error occurs during markdown formating: %w", err), theme))
}
}
if i.Exists(path.Join(epath, "finished.txt")) {
e.Finished, err = GetFileContent(i, path.Join(epath, "finished.txt"))
} else if i.Exists(path.Join(epath, "finished.md")) {
e.Finished, err = GetFileContent(i, path.Join(epath, "finished.md"))
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err), theme))
} else if len(e.Finished) > 0 {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Finished, e.Language, exceptions.GetFileExceptions("finished.md", "finished.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err)))
}
}
if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: an error occurs during markdown formating: %w", err), theme))
}
}
if i.Exists(path.Join(epath, "heading.jpg")) {
e.Image = path.Join(epath, "heading.jpg")
} else if i.Exists(path.Join(epath, "heading.png")) {
e.Image = path.Join(epath, "heading.png")
} else if theme == nil || theme.Image == "" {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("heading.jpg: No such file")))
}
// Parse challenge.txt
var md toml.MetaData
p, md, err = parseExerciceParams(i, epath)
if err != nil {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, err, theme))
return
}
// Alert about unknown keys in challenge.txt
if len(md.Undecoded()) > 0 {
for _, k := range md.Undecoded() {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("unknown key %q found, check https://fic.srs.epita.fr/doc/files/challenge/", k), theme))
}
}
e.WIP = p.WIP
if p.WIP && !AllowWIPExercice {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("exercice declared Work In Progress in challenge.toml"), theme))
}
if p.Gain == 0 {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("Undefined gain for challenge"), theme))
} else {
e.Gain = p.Gain
}
// Handle dependency
if len(p.Dependencies) > 0 {
if len(p.Dependencies[0].Theme) > 0 && (theme == nil || p.Dependencies[0].Theme != theme.Name) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to treat dependency to another theme (%q): not implemented.", p.Dependencies[0].Theme), theme))
} else {
if dmap == nil {
if dmap2, err := buildDependancyMap(i, theme); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to build dependency map: %w", err), theme))
} else {
dmap = &dmap2
}
}
if dmap != nil {
for edk, ed := range *dmap {
if edk == p.Dependencies[0].Id {
e.Depend = &ed.Id
break
}
}
if e.Depend == nil {
dmap_keys := []string{}
for k, _ := range *dmap {
dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k))
}
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("Unable to find required exercice dependancy %d (available at time of processing: %s)", p.Dependencies[0].Id, strings.Join(dmap_keys, ",")), theme))
}
}
}
}
// Handle resolutions
resolutionFound := false
e.VideoURI = path.Join(epath, "resolution.mp4")
if !i.Exists(e.VideoURI) {
e.VideoURI = ""
} else if size, err := GetFileSize(i, e.VideoURI); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: %w", err), theme))
e.VideoURI = ""
} else if size == 0 {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: The file is empty!"), theme))
e.VideoURI = ""
} else {
e.VideoURI = strings.Replace(url.PathEscape(path.Join("$RFILES$", e.VideoURI)), "%2F", "/", -1)
resolutionFound = true
}
writeup := path.Join(epath, "resolution.md")
if !i.Exists(writeup) {
writeup = path.Join(epath, "resolution.txt")
}
if i.Exists(writeup) {
if size, err := GetFileSize(i, writeup); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
} else if size == 0 {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: The file is empty!"), theme))
} else if e.Resolution, err = GetFileContent(i, writeup); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
} else {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Resolution, e.Language, exceptions.GetFileExceptions("resolution.md"), p.GetRawFlags()...)) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err)))
}
}
if e.Resolution, err = ProcessMarkdown(i, e.Resolution, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: error during markdown processing: %w", err), theme))
} else {
resolutionFound = true
}
}
}
if !resolutionFound {
errs = multierr.Append(errs, NewExerciceError(e, ErrResolutionNotFound, theme))
}
// Call checks hooks
for _, h := range hooks.exerciceHooks {
for _, err := range multierr.Errors(h(e, exceptions)) {
errs = multierr.Append(errs, NewExerciceError(e, err))
}
}
return
}
// SyncExercice imports new or updates existing given exercice.
func SyncExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, eid int, exceptions *CheckExceptions, errs error) {
var err error
var p ExerciceParams
var berrors error
e, p, eid, exceptions, _, berrors = BuildExercice(i, theme, epath, dmap, exceptions_in)
errs = multierr.Append(errs, berrors)
if e != nil {
if len(e.Image) > 0 {
if _, err := i.importFile(e.Image,
func(filePath string, origin string) (interface{}, error) {
if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil {
return nil, err
}
e.Image = strings.TrimPrefix(filePath, fic.FilesDir)
e.BackgroundColor, _ = getBackgroundColor(filePath)
// If the theme has no image yet, use the first exercice's image found
if theme != nil && theme.Image == "" {
theme.Image = e.Image
_, err := theme.Update()
if err != nil {
return nil, err
}
}
return nil, nil
}); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to import heading image: %w", err)))
}
}
// Create or update the exercice
err = theme.SaveNamedExercice(e)
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("error on exercice save: %w", err), theme))
return
}
// Import eercice tags
if _, err := e.WipeTags(); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to wipe tags: %w", err), theme))
}
for _, tag := range p.Tags {
if _, err := e.AddTag(tag); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to add tag: %w", err), theme))
return
}
}
}
return
}
// SyncExercices imports new or updates existing exercices, in a given theme.
func SyncExercices(i Importer, theme *fic.Theme, exceptions *CheckExceptions) (exceptions_out map[int]*CheckExceptions, errs error) {
if exercices, err := GetExercices(i, theme); err != nil {
errs = multierr.Append(errs, err)
} else {
exceptions_out = make(map[int]*CheckExceptions)
emap := map[string]int{}
dmap, _ := buildDependancyMap(i, theme)
for _, edir := range exercices {
e, eid, ex_exceptions, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap, exceptions)
if e != nil {
emap[e.Title] = eid
dmap[int64(eid)] = e
exceptions_out[eid] = ex_exceptions
errs = multierr.Append(errs, cur_errs)
}
}
// Remove old exercices
if exercices, err := theme.GetExercices(); err == nil {
for _, ex := range exercices {
if _, ok := emap[ex.Title]; !ok {
ex.DeleteCascade()
}
}
}
}
return
}
// ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list.
func ApiListRemoteExercices(c *gin.Context) {
if c.Params.ByName("thid") == "_" {
exercices, err := GetExercices(GlobalImporter, &fic.Theme{Path: StandaloneExercicesDirectory})
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
return
}
c.JSON(http.StatusOK, exercices)
return
}
theme, _, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
if theme != nil {
exercices, err := GetExercices(GlobalImporter, theme)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
return
}
c.JSON(http.StatusOK, exercices)
} else {
c.AbortWithStatusJSON(http.StatusInternalServerError, fmt.Errorf("%q", errs))
return
}
}
// ApiListRemoteExercice is an accessor letting foreign packages to access remote exercice attributes.
func ApiGetRemoteExercice(c *gin.Context) {
if c.Params.ByName("thid") == "_" {
exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, nil, path.Join(StandaloneExercicesDirectory, c.Params.ByName("exid")), nil, nil)
if exercice != nil {
c.JSON(http.StatusOK, exercice)
return
} else {
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
}
theme, exceptions, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
if theme != nil {
exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, theme, path.Join(theme.Path, c.Params.ByName("exid")), nil, exceptions)
if exercice != nil {
c.JSON(http.StatusOK, exercice)
return
} else {
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
} else {
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
}