package sync import ( "bytes" "fmt" "hash/adler32" "image" "net/http" "net/url" "path" "strconv" "strings" "github.com/BurntSushi/toml" "github.com/gin-gonic/gin" "github.com/yuin/goldmark" "go.uber.org/multierr" "srs.epita.fr/fic-server/libfic" ) // Set AllowWIPExercice if WIP exercices are accepted. var AllowWIPExercice bool = false func fixnbsp(s string) string { return strings.Replace(strings.Replace(strings.Replace(s, " ?", " ?", -1), " !", " !", -1), " :", " :", -1) } // GetExercices returns all exercice directories existing in a given theme, considering the given Importer. func GetExercices(i Importer, theme *fic.Theme) ([]string, error) { var exercices []string if len(theme.Path) == 0 { return []string{}, nil } else if dirs, err := i.ListDir(theme.Path); err != nil { return []string{}, err } else { for _, dir := range dirs { if _, err := i.ListDir(path.Join(theme.Path, dir)); err == nil { if dir[0] != '.' && strings.Contains(dir, "-") { exercices = append(exercices, dir) } } } } return exercices, nil } func buildDependancyMap(i Importer, theme *fic.Theme) (dmap map[int64]*fic.Exercice, err error) { var exercices []string if exercices, err = GetExercices(i, theme); err != nil { return } else { dmap = map[int64]*fic.Exercice{} for _, edir := range exercices { var eid int var ename string eid, ename, err = parseExerciceDirname(edir) if err != nil { err = nil continue } var e *fic.Exercice e, err = theme.GetExerciceByTitle(ename) if err != nil { return } dmap[int64(eid)] = e } return } } func parseExerciceDirname(edir string) (eid int, ename string, err error) { edir_splt := strings.SplitN(edir, "-", 2) if len(edir_splt) != 2 { err = fmt.Errorf("%q is not a valid exercice directory: missing id prefix", edir) return } eid, err = strconv.Atoi(edir_splt[0]) if err != nil { err = fmt.Errorf("%q: invalid exercice identifier: %s", edir, err) return } // ID 0: peak a deterministic-random-ordered ID instead if eid == 0 { eid = int(adler32.Checksum([]byte(edir_splt[1]))) } ename = edir_splt[1] return } // BuildExercice creates an Exercice from a given importer. func BuildExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, p ExerciceParams, eid int, exceptions *CheckExceptions, edir string, errs error) { e = &fic.Exercice{} e.Path = epath edir = path.Base(epath) var err error eid, e.Title, err = parseExerciceDirname(edir) if err != nil { // Ignore eid if we are certain this is an exercice directory, eid will be 0 if !i.Exists(path.Join(epath, "title.txt")) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to parse exercice directory: %w", err), theme)) return nil, p, eid, exceptions_in, edir, errs } } // Get exceptions exceptions = LoadExerciceException(i, theme, e, exceptions_in) //log.Printf("Kept repochecker exceptions for this exercice: %v", exceptions) if theme != nil { e.Language = theme.Language } // Overwrite language if language.txt exists if language, err := GetFileContent(i, path.Join(epath, "language.txt")); err == nil { language = strings.TrimSpace(language) if strings.Contains(language, "\n") { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("language.txt: Language can't contain new lines"), theme)) } else { e.Language = language } } // Overwrite title if title.txt exists if myTitle, err := GetFileContent(i, path.Join(epath, "title.txt")); err == nil { myTitle = strings.TrimSpace(myTitle) if strings.Contains(myTitle, "\n") { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title.txt: Title can't contain new lines"), theme)) } else { e.Title = myTitle } } // Character reserved for WIP exercices if len(e.Title) > 0 && e.Title[0] == '%' { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title can't contain start by '%%'"), theme)) } e.URLId = fic.ToURLid(e.Title) e.Title = fixnbsp(e.Title) if i.Exists(path.Join(epath, "AUTHORS.txt")) { if authors, err := getAuthors(i, epath); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get AUTHORS.txt: %w", err))) } else { // Format authors e.Authors = strings.Join(authors, ", ") } } // Process headline if i.Exists(path.Join(epath, "headline.txt")) { e.Headline, err = GetFileContent(i, path.Join(epath, "headline.txt")) } else if i.Exists(path.Join(epath, "headline.md")) { e.Headline, err = GetFileContent(i, path.Join(epath, "headline.md")) } if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get exercice's headline: %w", err))) } if e.Headline != "" { // Call checks hooks for _, h := range hooks.mdTextHooks { for _, err := range multierr.Errors(h(e.Headline, e.Language, exceptions.GetFileExceptions("headline.md", "headline.txt"))) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("headline.md: %w", err))) } } } // Texts to format using Markdown if i.Exists(path.Join(epath, "overview.txt")) { e.Overview, err = GetFileContent(i, path.Join(epath, "overview.txt")) } else if i.Exists(path.Join(epath, "overview.md")) { e.Overview, err = GetFileContent(i, path.Join(epath, "overview.md")) } else { err = fmt.Errorf("Unable to find overview.txt nor overview.md") } if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.txt: %s", err), theme)) } else { e.Overview = fixnbsp(e.Overview) // Call checks hooks for _, h := range hooks.mdTextHooks { for _, err := range multierr.Errors(h(e.Overview, e.Language, exceptions.GetFileExceptions("overview.md", "overview.txt"))) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: %w", err))) } } var buf bytes.Buffer if e.Headline == "" { err := goldmark.Convert([]byte(strings.Split(e.Overview, "\n")[0]), &buf) if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating of the headline: %w", err), theme)) } else { e.Headline = string(buf.Bytes()) } } if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating: %w", err), theme)) } } if i.Exists(path.Join(epath, "statement.txt")) { e.Statement, err = GetFileContent(i, path.Join(epath, "statement.txt")) } else if i.Exists(path.Join(epath, "statement.md")) { e.Statement, err = GetFileContent(i, path.Join(epath, "statement.md")) } else { err = fmt.Errorf("Unable to find statement.txt nor statement.md") } if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err), theme)) } else { // Call checks hooks for _, h := range hooks.mdTextHooks { for _, err := range multierr.Errors(h(e.Statement, e.Language, exceptions.GetFileExceptions("statement.md", "statement.txt"))) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err))) } } if e.Statement, err = ProcessMarkdown(i, fixnbsp(e.Statement), epath); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: an error occurs during markdown formating: %w", err), theme)) } } if i.Exists(path.Join(epath, "finished.txt")) { e.Finished, err = GetFileContent(i, path.Join(epath, "finished.txt")) } else if i.Exists(path.Join(epath, "finished.md")) { e.Finished, err = GetFileContent(i, path.Join(epath, "finished.md")) } if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err), theme)) } else if len(e.Finished) > 0 { // Call checks hooks for _, h := range hooks.mdTextHooks { for _, err := range multierr.Errors(h(e.Finished, e.Language, exceptions.GetFileExceptions("finished.md", "finished.txt"))) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err))) } } if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: an error occurs during markdown formating: %w", err), theme)) } } if i.Exists(path.Join(epath, "heading.jpg")) { e.Image = path.Join(epath, "heading.jpg") } else if i.Exists(path.Join(epath, "heading.png")) { e.Image = path.Join(epath, "heading.png") } else if theme == nil || theme.Image == "" { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("heading.jpg: No such file"))) } // Parse challenge.txt var md toml.MetaData p, md, err = parseExerciceParams(i, epath) if err != nil { errs = multierr.Append(errs, NewChallengeTxtError(e, 0, err, theme)) return } // Alert about unknown keys in challenge.txt if len(md.Undecoded()) > 0 { for _, k := range md.Undecoded() { errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("unknown key %q found, check https://fic.srs.epita.fr/doc/files/challenge/", k), theme)) } } e.WIP = p.WIP if p.WIP && !AllowWIPExercice { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("exercice declared Work In Progress in challenge.toml"), theme)) } if p.Gain == 0 { errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("Undefined gain for challenge"), theme)) } else { e.Gain = p.Gain } // Handle dependency if len(p.Dependencies) > 0 { if len(p.Dependencies[0].Theme) > 0 && (theme == nil || p.Dependencies[0].Theme != theme.Name) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to treat dependency to another theme (%q): not implemented.", p.Dependencies[0].Theme), theme)) } else { if dmap == nil { if dmap2, err := buildDependancyMap(i, theme); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to build dependency map: %w", err), theme)) } else { dmap = &dmap2 } } if dmap != nil { for edk, ed := range *dmap { if edk == p.Dependencies[0].Id { e.Depend = &ed.Id break } } if e.Depend == nil { dmap_keys := []string{} for k, _ := range *dmap { dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k)) } errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("Unable to find required exercice dependancy %d (available at time of processing: %s)", p.Dependencies[0].Id, strings.Join(dmap_keys, ",")), theme)) } } } } // Handle resolutions resolutionFound := false e.VideoURI = path.Join(epath, "resolution.mp4") if !i.Exists(e.VideoURI) { e.VideoURI = "" } else if size, err := GetFileSize(i, e.VideoURI); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: %w", err), theme)) e.VideoURI = "" } else if size == 0 { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: The file is empty!"), theme)) e.VideoURI = "" } else { e.VideoURI = strings.Replace(url.PathEscape(path.Join("$RFILES$", e.VideoURI)), "%2F", "/", -1) resolutionFound = true } writeup := path.Join(epath, "resolution.md") if !i.Exists(writeup) { writeup = path.Join(epath, "resolution.txt") } if i.Exists(writeup) { if size, err := GetFileSize(i, writeup); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme)) } else if size == 0 { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: The file is empty!"), theme)) } else if e.Resolution, err = GetFileContent(i, writeup); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme)) } else { // Call checks hooks for _, h := range hooks.mdTextHooks { for _, err := range multierr.Errors(h(e.Resolution, e.Language, exceptions.GetFileExceptions("resolution.md"), p.GetRawFlags()...)) { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err))) } } if e.Resolution, err = ProcessMarkdown(i, e.Resolution, epath); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: error during markdown processing: %w", err), theme)) } else { resolutionFound = true } } } if !resolutionFound { errs = multierr.Append(errs, NewExerciceError(e, ErrResolutionNotFound, theme)) } // Call checks hooks for _, h := range hooks.exerciceHooks { for _, err := range multierr.Errors(h(e, exceptions)) { errs = multierr.Append(errs, NewExerciceError(e, err)) } } return } // SyncExercice imports new or updates existing given exercice. func SyncExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, eid int, exceptions *CheckExceptions, errs error) { var err error var p ExerciceParams var berrors error e, p, eid, exceptions, _, berrors = BuildExercice(i, theme, epath, dmap, exceptions_in) errs = multierr.Append(errs, berrors) if e != nil { if len(e.Image) > 0 { if _, err := i.importFile(e.Image, func(filePath string, origin string) (interface{}, error) { if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil { return nil, err } e.Image = strings.TrimPrefix(filePath, fic.FilesDir) e.BackgroundColor, _ = getBackgroundColor(filePath) // If the theme has no image yet, use the first exercice's image found if theme != nil && theme.Image == "" { theme.Image = e.Image _, err := theme.Update() if err != nil { return nil, err } } return nil, nil }); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to import heading image: %w", err))) } } // Create or update the exercice err = theme.SaveNamedExercice(e) if err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("error on exercice save: %w", err), theme)) return } // Import eercice tags if _, err := e.WipeTags(); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to wipe tags: %w", err), theme)) } for _, tag := range p.Tags { if _, err := e.AddTag(tag); err != nil { errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to add tag: %w", err), theme)) return } } } return } // SyncExercices imports new or updates existing exercices, in a given theme. func SyncExercices(i Importer, theme *fic.Theme, exceptions *CheckExceptions) (exceptions_out map[int]*CheckExceptions, errs error) { if exercices, err := GetExercices(i, theme); err != nil { errs = multierr.Append(errs, err) } else { exceptions_out = make(map[int]*CheckExceptions) emap := map[string]int{} dmap, _ := buildDependancyMap(i, theme) for _, edir := range exercices { e, eid, ex_exceptions, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap, exceptions) if e != nil { emap[e.Title] = eid dmap[int64(eid)] = e exceptions_out[eid] = ex_exceptions errs = multierr.Append(errs, cur_errs) } } // Remove old exercices if exercices, err := theme.GetExercices(); err == nil { for _, ex := range exercices { if _, ok := emap[ex.Title]; !ok { ex.DeleteCascade() } } } } return } // ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list. func ApiListRemoteExercices(c *gin.Context) { if c.Params.ByName("thid") == "_" { exercices, err := GetExercices(GlobalImporter, &fic.Theme{Path: StandaloneExercicesDirectory}) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()}) return } c.JSON(http.StatusOK, exercices) return } theme, _, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid")) if theme != nil { exercices, err := GetExercices(GlobalImporter, theme) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()}) return } c.JSON(http.StatusOK, exercices) } else { c.AbortWithStatusJSON(http.StatusInternalServerError, fmt.Errorf("%q", errs)) return } } // ApiListRemoteExercice is an accessor letting foreign packages to access remote exercice attributes. func ApiGetRemoteExercice(c *gin.Context) { if c.Params.ByName("thid") == "_" { exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, nil, path.Join(StandaloneExercicesDirectory, c.Params.ByName("exid")), nil, nil) if exercice != nil { c.JSON(http.StatusOK, exercice) return } else { c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)}) return } } theme, exceptions, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid")) if theme != nil { exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, theme, path.Join(theme.Path, c.Params.ByName("exid")), nil, exceptions) if exercice != nil { c.JSON(http.StatusOK, exercice) return } else { c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)}) return } } else { c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)}) return } }