server/admin/sync/exercices.go

527 lines
17 KiB
Go
Raw Normal View History

package sync
import (
"bytes"
"fmt"
2024-03-18 09:34:42 +00:00
"hash/adler32"
2023-06-14 15:34:18 +00:00
"image"
2022-05-16 09:38:46 +00:00
"net/http"
2022-05-24 15:36:33 +00:00
"net/url"
"path"
"strconv"
"strings"
"github.com/BurntSushi/toml"
2022-05-16 09:38:46 +00:00
"github.com/gin-gonic/gin"
"github.com/yuin/goldmark"
"go.uber.org/multierr"
"srs.epita.fr/fic-server/libfic"
)
// Set AllowWIPExercice if WIP exercices are accepted.
var AllowWIPExercice bool = false
func fixnbsp(s string) string {
return strings.Replace(strings.Replace(strings.Replace(s, " ?", " ?", -1), " !", " !", -1), " :", " :", -1)
}
// GetExercices returns all exercice directories existing in a given theme, considering the given Importer.
2021-11-22 14:35:07 +00:00
func GetExercices(i Importer, theme *fic.Theme) ([]string, error) {
var exercices []string
if len(theme.Path) == 0 {
return []string{}, nil
2023-11-25 16:13:31 +00:00
} else if dirs, err := i.ListDir(theme.Path); err != nil {
return []string{}, err
} else {
for _, dir := range dirs {
2023-11-25 16:13:31 +00:00
if _, err := i.ListDir(path.Join(theme.Path, dir)); err == nil {
if dir[0] != '.' && strings.Contains(dir, "-") {
exercices = append(exercices, dir)
}
}
}
}
return exercices, nil
}
2021-11-22 14:35:07 +00:00
func buildDependancyMap(i Importer, theme *fic.Theme) (dmap map[int64]*fic.Exercice, err error) {
var exercices []string
if exercices, err = GetExercices(i, theme); err != nil {
return
} else {
2021-11-22 14:35:07 +00:00
dmap = map[int64]*fic.Exercice{}
for _, edir := range exercices {
var eid int
var ename string
eid, ename, err = parseExerciceDirname(edir)
if err != nil {
err = nil
continue
}
2021-11-22 14:35:07 +00:00
var e *fic.Exercice
e, err = theme.GetExerciceByTitle(ename)
if err != nil {
return
}
dmap[int64(eid)] = e
}
return
}
}
func parseExerciceDirname(edir string) (eid int, ename string, err error) {
edir_splt := strings.SplitN(edir, "-", 2)
if len(edir_splt) != 2 {
2020-04-15 05:39:38 +00:00
err = fmt.Errorf("%q is not a valid exercice directory: missing id prefix", edir)
return
}
eid, err = strconv.Atoi(edir_splt[0])
if err != nil {
2020-04-15 05:39:38 +00:00
err = fmt.Errorf("%q: invalid exercice identifier: %s", edir, err)
return
}
2024-03-18 09:34:42 +00:00
// ID 0: peak a deterministic-random-ordered ID instead
if eid == 0 {
eid = int(adler32.Checksum([]byte(edir_splt[1])))
}
ename = edir_splt[1]
return
}
// BuildExercice creates an Exercice from a given importer.
func BuildExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, p ExerciceParams, eid int, exceptions *CheckExceptions, edir string, errs error) {
e = &fic.Exercice{}
e.Path = epath
edir = path.Base(epath)
var err error
eid, e.Title, err = parseExerciceDirname(edir)
if err != nil {
// Ignore eid if we are certain this is an exercice directory, eid will be 0
if !i.Exists(path.Join(epath, "title.txt")) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to parse exercice directory: %w", err), theme))
return nil, p, eid, exceptions_in, edir, errs
}
}
// Get exceptions
exceptions = LoadExerciceException(i, theme, e, exceptions_in)
//log.Printf("Kept repochecker exceptions for this exercice: %v", exceptions)
if theme != nil {
e.Language = theme.Language
}
// Overwrite language if language.txt exists
if language, err := GetFileContent(i, path.Join(epath, "language.txt")); err == nil {
language = strings.TrimSpace(language)
if strings.Contains(language, "\n") {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("language.txt: Language can't contain new lines"), theme))
} else {
e.Language = language
}
}
2022-01-20 15:16:52 +00:00
// Overwrite title if title.txt exists
if myTitle, err := GetFileContent(i, path.Join(epath, "title.txt")); err == nil {
2022-01-20 15:16:52 +00:00
myTitle = strings.TrimSpace(myTitle)
if strings.Contains(myTitle, "\n") {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title.txt: Title can't contain new lines"), theme))
2022-01-20 15:16:52 +00:00
} else {
e.Title = myTitle
}
}
// Character reserved for WIP exercices
if len(e.Title) > 0 && e.Title[0] == '%' {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title can't contain start by '%%'"), theme))
}
e.URLId = fic.ToURLid(e.Title)
e.Title = fixnbsp(e.Title)
if i.Exists(path.Join(epath, "AUTHORS.txt")) {
if authors, err := getAuthors(i, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get AUTHORS.txt: %w", err)))
} else {
// Format authors
e.Authors = strings.Join(authors, ", ")
}
}
// Process headline
if i.Exists(path.Join(epath, "headline.txt")) {
e.Headline, err = GetFileContent(i, path.Join(epath, "headline.txt"))
} else if i.Exists(path.Join(epath, "headline.md")) {
e.Headline, err = GetFileContent(i, path.Join(epath, "headline.md"))
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get exercice's headline: %w", err)))
}
if e.Headline != "" {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Headline, e.Language, exceptions.GetFileExceptions("headline.md", "headline.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("headline.md: %w", err)))
}
}
}
// Texts to format using Markdown
2023-05-03 08:54:02 +00:00
if i.Exists(path.Join(epath, "overview.txt")) {
e.Overview, err = GetFileContent(i, path.Join(epath, "overview.txt"))
2023-05-03 08:54:02 +00:00
} else if i.Exists(path.Join(epath, "overview.md")) {
e.Overview, err = GetFileContent(i, path.Join(epath, "overview.md"))
} else {
err = fmt.Errorf("Unable to find overview.txt nor overview.md")
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.txt: %s", err), theme))
} else {
e.Overview = fixnbsp(e.Overview)
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Overview, e.Language, exceptions.GetFileExceptions("overview.md", "overview.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: %w", err)))
}
}
var buf bytes.Buffer
if e.Headline == "" {
err := goldmark.Convert([]byte(strings.Split(e.Overview, "\n")[0]), &buf)
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating of the headline: %w", err), theme))
} else {
e.Headline = string(buf.Bytes())
}
}
if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating: %w", err), theme))
}
}
2023-05-03 08:54:02 +00:00
if i.Exists(path.Join(epath, "statement.txt")) {
e.Statement, err = GetFileContent(i, path.Join(epath, "statement.txt"))
2023-05-03 08:54:02 +00:00
} else if i.Exists(path.Join(epath, "statement.md")) {
e.Statement, err = GetFileContent(i, path.Join(epath, "statement.md"))
} else {
err = fmt.Errorf("Unable to find statement.txt nor statement.md")
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err), theme))
} else {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Statement, e.Language, exceptions.GetFileExceptions("statement.md", "statement.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err)))
}
}
if e.Statement, err = ProcessMarkdown(i, fixnbsp(e.Statement), epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: an error occurs during markdown formating: %w", err), theme))
}
}
2023-05-03 08:54:02 +00:00
if i.Exists(path.Join(epath, "finished.txt")) {
e.Finished, err = GetFileContent(i, path.Join(epath, "finished.txt"))
2023-05-03 08:54:02 +00:00
} else if i.Exists(path.Join(epath, "finished.md")) {
e.Finished, err = GetFileContent(i, path.Join(epath, "finished.md"))
}
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err), theme))
} else if len(e.Finished) > 0 {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Finished, e.Language, exceptions.GetFileExceptions("finished.md", "finished.txt"))) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err)))
}
}
if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: an error occurs during markdown formating: %w", err), theme))
}
}
2023-06-14 15:34:18 +00:00
if i.Exists(path.Join(epath, "heading.jpg")) {
e.Image = path.Join(epath, "heading.jpg")
} else if i.Exists(path.Join(epath, "heading.png")) {
e.Image = path.Join(epath, "heading.png")
} else if theme == nil || theme.Image == "" {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("heading.jpg: No such file")))
2023-06-14 15:34:18 +00:00
}
// Parse challenge.txt
var md toml.MetaData
p, md, err = parseExerciceParams(i, epath)
if err != nil {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, err, theme))
return
}
// Alert about unknown keys in challenge.txt
if len(md.Undecoded()) > 0 {
for _, k := range md.Undecoded() {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("unknown key %q found, check https://fic.srs.epita.fr/doc/files/challenge/", k), theme))
}
}
e.WIP = p.WIP
if p.WIP && !AllowWIPExercice {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("exercice declared Work In Progress in challenge.toml"), theme))
}
if p.Gain == 0 {
errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("Undefined gain for challenge"), theme))
} else {
e.Gain = p.Gain
}
// Handle dependency
if len(p.Dependencies) > 0 {
if len(p.Dependencies[0].Theme) > 0 && (theme == nil || p.Dependencies[0].Theme != theme.Name) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to treat dependency to another theme (%q): not implemented.", p.Dependencies[0].Theme), theme))
} else {
if dmap == nil {
if dmap2, err := buildDependancyMap(i, theme); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to build dependency map: %w", err), theme))
} else {
dmap = &dmap2
}
}
if dmap != nil {
for edk, ed := range *dmap {
if edk == p.Dependencies[0].Id {
e.Depend = &ed.Id
break
}
}
if e.Depend == nil {
dmap_keys := []string{}
for k, _ := range *dmap {
dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k))
}
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("Unable to find required exercice dependancy %d (available at time of processing: %s)", p.Dependencies[0].Id, strings.Join(dmap_keys, ",")), theme))
}
}
}
}
2021-12-10 17:55:47 +00:00
// Handle resolutions
resolutionFound := false
e.VideoURI = path.Join(epath, "resolution.mp4")
2023-05-03 08:54:02 +00:00
if !i.Exists(e.VideoURI) {
e.VideoURI = ""
2022-11-21 17:55:38 +00:00
} else if size, err := GetFileSize(i, e.VideoURI); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: %w", err), theme))
e.VideoURI = ""
} else if size == 0 {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: The file is empty!"), theme))
e.VideoURI = ""
2021-12-10 17:55:47 +00:00
} else {
2022-06-12 10:15:39 +00:00
e.VideoURI = strings.Replace(url.PathEscape(path.Join("$RFILES$", e.VideoURI)), "%2F", "/", -1)
2021-12-10 17:55:47 +00:00
resolutionFound = true
}
writeup := path.Join(epath, "resolution.md")
2023-05-03 08:54:02 +00:00
if !i.Exists(writeup) {
2021-12-10 17:55:47 +00:00
writeup = path.Join(epath, "resolution.txt")
}
2023-05-03 08:54:02 +00:00
if i.Exists(writeup) {
2022-11-21 17:55:38 +00:00
if size, err := GetFileSize(i, writeup); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
2021-12-10 17:55:47 +00:00
} else if size == 0 {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: The file is empty!"), theme))
} else if e.Resolution, err = GetFileContent(i, writeup); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
2021-12-10 17:55:47 +00:00
} else {
// Call checks hooks
for _, h := range hooks.mdTextHooks {
for _, err := range multierr.Errors(h(e.Resolution, e.Language, exceptions.GetFileExceptions("resolution.md"), p.GetRawFlags()...)) {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err)))
}
}
if e.Resolution, err = ProcessMarkdown(i, e.Resolution, epath); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: error during markdown processing: %w", err), theme))
} else {
resolutionFound = true
}
2021-12-10 17:55:47 +00:00
}
}
if !resolutionFound {
errs = multierr.Append(errs, NewExerciceError(e, ErrResolutionNotFound, theme))
}
// Call checks hooks
for _, h := range hooks.exerciceHooks {
for _, err := range multierr.Errors(h(e, exceptions)) {
errs = multierr.Append(errs, NewExerciceError(e, err))
}
}
return
}
2018-08-17 19:18:10 +00:00
// SyncExercice imports new or updates existing given exercice.
func SyncExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, eid int, exceptions *CheckExceptions, errs error) {
var err error
var p ExerciceParams
var berrors error
e, p, eid, exceptions, _, berrors = BuildExercice(i, theme, epath, dmap, exceptions_in)
errs = multierr.Append(errs, berrors)
if e != nil {
2023-06-14 15:34:18 +00:00
if len(e.Image) > 0 {
if _, err := i.importFile(e.Image,
func(filePath string, origin string) (interface{}, error) {
if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil {
return nil, err
}
e.Image = strings.TrimPrefix(filePath, fic.FilesDir)
e.BackgroundColor, _ = getBackgroundColor(filePath)
2023-06-14 15:34:18 +00:00
// If the theme has no image yet, use the first exercice's image found
if theme != nil && theme.Image == "" {
theme.Image = e.Image
_, err := theme.Update()
if err != nil {
return nil, err
}
}
2023-06-14 15:34:18 +00:00
return nil, nil
2023-06-14 15:34:18 +00:00
}); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to import heading image: %w", err)))
2023-06-14 15:34:18 +00:00
}
}
// Create or update the exercice
err = theme.SaveNamedExercice(e)
if err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("error on exercice save: %w", err), theme))
return
}
// Import eercice tags
if _, err := e.WipeTags(); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to wipe tags: %w", err), theme))
}
for _, tag := range p.Tags {
if _, err := e.AddTag(tag); err != nil {
errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to add tag: %w", err), theme))
return
}
}
}
return
}
// SyncExercices imports new or updates existing exercices, in a given theme.
func SyncExercices(i Importer, theme *fic.Theme, exceptions *CheckExceptions) (exceptions_out map[int]*CheckExceptions, errs error) {
if exercices, err := GetExercices(i, theme); err != nil {
errs = multierr.Append(errs, err)
} else {
2023-07-24 21:37:44 +00:00
exceptions_out = make(map[int]*CheckExceptions)
emap := map[string]int{}
dmap, _ := buildDependancyMap(i, theme)
for _, edir := range exercices {
e, eid, ex_exceptions, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap, exceptions)
if e != nil {
emap[e.Title] = eid
2021-11-22 14:35:07 +00:00
dmap[int64(eid)] = e
exceptions_out[eid] = ex_exceptions
errs = multierr.Append(errs, cur_errs)
}
}
// Remove old exercices
if exercices, err := theme.GetExercices(); err == nil {
for _, ex := range exercices {
if _, ok := emap[ex.Title]; !ok {
ex.DeleteCascade()
}
}
}
}
return
}
2018-05-11 23:08:37 +00:00
// ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list.
2022-05-16 09:38:46 +00:00
func ApiListRemoteExercices(c *gin.Context) {
if c.Params.ByName("thid") == "_" {
exercices, err := GetExercices(GlobalImporter, &fic.Theme{Path: StandaloneExercicesDirectory})
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
return
}
c.JSON(http.StatusOK, exercices)
return
}
theme, _, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
if theme != nil {
2022-05-16 09:38:46 +00:00
exercices, err := GetExercices(GlobalImporter, theme)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
return
}
c.JSON(http.StatusOK, exercices)
} else {
2022-05-16 09:38:46 +00:00
c.AbortWithStatusJSON(http.StatusInternalServerError, fmt.Errorf("%q", errs))
return
}
}
// ApiListRemoteExercice is an accessor letting foreign packages to access remote exercice attributes.
2022-05-16 09:38:46 +00:00
func ApiGetRemoteExercice(c *gin.Context) {
if c.Params.ByName("thid") == "_" {
exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, nil, path.Join(StandaloneExercicesDirectory, c.Params.ByName("exid")), nil, nil)
if exercice != nil {
c.JSON(http.StatusOK, exercice)
return
} else {
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
}
theme, exceptions, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
if theme != nil {
exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, theme, path.Join(theme.Path, c.Params.ByName("exid")), nil, exceptions)
if exercice != nil {
2022-05-16 09:38:46 +00:00
c.JSON(http.StatusOK, exercice)
return
} else {
2022-05-16 09:38:46 +00:00
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
} else {
2022-05-16 09:38:46 +00:00
c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
return
}
}