server/admin/sync/exercices.go

package sync

import (
	"bytes"
	"fmt"
	"hash/adler32"
	"image"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/gin-gonic/gin"
	"github.com/yuin/goldmark"
	"go.uber.org/multierr"

	"srs.epita.fr/fic-server/libfic"
)

// Set AllowWIPExercice if WIP exercices are accepted.
var AllowWIPExercice bool = false

func fixnbsp(s string) string {
	return strings.Replace(strings.Replace(strings.Replace(s, " ?", " ?", -1), " !", " !", -1), " :", " :", -1)
}

// GetExercices returns all exercice directories existing in a given theme, considering the given Importer.
func GetExercices(i Importer, theme *fic.Theme) ([]string, error) {
	var exercices []string

	if len(theme.Path) == 0 {
		return []string{}, nil
	} else if dirs, err := i.ListDir(theme.Path); err != nil {
		return []string{}, err
	} else {
		for _, dir := range dirs {
			if _, err := i.ListDir(path.Join(theme.Path, dir)); err == nil {
				if dir[0] != '.' && strings.Contains(dir, "-") {
					exercices = append(exercices, dir)
				}
			}
		}
	}

	return exercices, nil
}

func buildDependancyMap(i Importer, theme *fic.Theme) (dmap map[int64]*fic.Exercice, err error) {
	var exercices []string
	if exercices, err = GetExercices(i, theme); err != nil {
		return
	} else {
		dmap = map[int64]*fic.Exercice{}

		for _, edir := range exercices {
			var eid int
			var ename string
			eid, ename, err = parseExerciceDirname(edir)
			if err != nil {
				err = nil
				continue
			}

			var e *fic.Exercice
			e, err = theme.GetExerciceByTitle(ename)
			if err != nil {
				return
			}

			dmap[int64(eid)] = e
		}

		return
	}
}

func parseExerciceDirname(edir string) (eid int, ename string, err error) {
	edir_splt := strings.SplitN(edir, "-", 2)
	if len(edir_splt) != 2 {
		err = fmt.Errorf("%q is not a valid exercice directory: missing id prefix", edir)
		return
	}

	eid, err = strconv.Atoi(edir_splt[0])
	if err != nil {
		err = fmt.Errorf("%q: invalid exercice identifier: %s", edir, err)
		return
	}

	// ID 0: peak a deterministic-random-ordered ID instead
	if eid == 0 {
		eid = int(adler32.Checksum([]byte(edir_splt[1])))
	}

	ename = edir_splt[1]

	return
}

// BuildExercice creates an Exercice from a given importer.
func BuildExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, p ExerciceParams, eid int, exceptions *CheckExceptions, edir string, errs error) {
	e = &fic.Exercice{}

	e.Path = epath
	edir = path.Base(epath)

	var err error
	eid, e.Title, err = parseExerciceDirname(edir)
	if err != nil {
		// Ignore eid if we are certain this is an exercice directory, eid will be 0
		if !i.Exists(path.Join(epath, "title.txt")) {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to parse exercice directory: %w", err), theme))
			return nil, p, eid, exceptions_in, edir, errs
		}
	}

	// Get exceptions
	exceptions = LoadExerciceException(i, theme, e, exceptions_in)
	//log.Printf("Kept repochecker exceptions for this exercice: %v", exceptions)

	if theme != nil {
		e.Language = theme.Language
	}
	// Overwrite language if language.txt exists
	if language, err := GetFileContent(i, path.Join(epath, "language.txt")); err == nil {
		language = strings.TrimSpace(language)
		if strings.Contains(language, "\n") {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("language.txt: Language can't contain new lines"), theme))
		} else {
			e.Language = language
		}
	}

	// Overwrite title if title.txt exists
	if myTitle, err := GetFileContent(i, path.Join(epath, "title.txt")); err == nil {
		myTitle = strings.TrimSpace(myTitle)
		if strings.Contains(myTitle, "\n") {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title.txt: Title can't contain new lines"), theme))
		} else {
			e.Title = myTitle
		}
	}

	// Character reserved for WIP exercices
	if len(e.Title) > 0 && e.Title[0] == '%' {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("title can't contain start by '%%'"), theme))
	}

	e.URLId = fic.ToURLid(e.Title)
	e.Title = fixnbsp(e.Title)

	if i.Exists(path.Join(epath, "AUTHORS.txt")) {
		if authors, err := getAuthors(i, epath); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get AUTHORS.txt: %w", err)))
		} else {
			// Format authors
			e.Authors = strings.Join(authors, ", ")
		}
	}

	// Process headline
	if i.Exists(path.Join(epath, "headline.txt")) {
		e.Headline, err = GetFileContent(i, path.Join(epath, "headline.txt"))
	} else if i.Exists(path.Join(epath, "headline.md")) {
		e.Headline, err = GetFileContent(i, path.Join(epath, "headline.md"))
	}
	if err != nil {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to get exercice's headline: %w", err)))
	}
	if e.Headline != "" {
		// Call checks hooks
		for _, h := range hooks.mdTextHooks {
			for _, err := range multierr.Errors(h(e.Headline, e.Language, exceptions.GetFileExceptions("headline.md", "headline.txt"))) {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("headline.md: %w", err)))
			}
		}
	}

	// Texts to format using Markdown
	if i.Exists(path.Join(epath, "overview.txt")) {
		e.Overview, err = GetFileContent(i, path.Join(epath, "overview.txt"))
	} else if i.Exists(path.Join(epath, "overview.md")) {
		e.Overview, err = GetFileContent(i, path.Join(epath, "overview.md"))
	} else {
		err = fmt.Errorf("Unable to find overview.txt nor overview.md")
	}
	if err != nil {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.txt: %s", err), theme))
	} else {
		e.Overview = fixnbsp(e.Overview)

		// Call checks hooks
		for _, h := range hooks.mdTextHooks {
			for _, err := range multierr.Errors(h(e.Overview, e.Language, exceptions.GetFileExceptions("overview.md", "overview.txt"))) {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: %w", err)))
			}
		}

		var buf bytes.Buffer
		if e.Headline == "" {
			err := goldmark.Convert([]byte(strings.Split(e.Overview, "\n")[0]), &buf)
			if err != nil {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating of the headline: %w", err), theme))
			} else {
				e.Headline = string(buf.Bytes())
			}
		}

		if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("overview.md: an error occurs during markdown formating: %w", err), theme))
		}
	}

	if i.Exists(path.Join(epath, "statement.txt")) {
		e.Statement, err = GetFileContent(i, path.Join(epath, "statement.txt"))
	} else if i.Exists(path.Join(epath, "statement.md")) {
		e.Statement, err = GetFileContent(i, path.Join(epath, "statement.md"))
	} else {
		err = fmt.Errorf("Unable to find statement.txt nor statement.md")
	}
	if err != nil {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err), theme))
	} else {
		// Call checks hooks
		for _, h := range hooks.mdTextHooks {
			for _, err := range multierr.Errors(h(e.Statement, e.Language, exceptions.GetFileExceptions("statement.md", "statement.txt"))) {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: %w", err)))
			}
		}

		if e.Statement, err = ProcessMarkdown(i, fixnbsp(e.Statement), epath); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("statement.md: an error occurs during markdown formating: %w", err), theme))
		}
	}

	if i.Exists(path.Join(epath, "finished.txt")) {
		e.Finished, err = GetFileContent(i, path.Join(epath, "finished.txt"))
	} else if i.Exists(path.Join(epath, "finished.md")) {
		e.Finished, err = GetFileContent(i, path.Join(epath, "finished.md"))
	}
	if err != nil {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err), theme))
	} else if len(e.Finished) > 0 {
		// Call checks hooks
		for _, h := range hooks.mdTextHooks {
			for _, err := range multierr.Errors(h(e.Finished, e.Language, exceptions.GetFileExceptions("finished.md", "finished.txt"))) {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: %w", err)))
			}
		}

		if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("finished.md: an error occurs during markdown formating: %w", err), theme))
		}
	}

	if i.Exists(path.Join(epath, "heading.jpg")) {
		e.Image = path.Join(epath, "heading.jpg")
	} else if i.Exists(path.Join(epath, "heading.png")) {
		e.Image = path.Join(epath, "heading.png")
	} else if theme == nil || theme.Image == "" {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("heading.jpg: No such file")))
	}

	// Parse challenge.txt
	var md toml.MetaData
	p, md, err = parseExerciceParams(i, epath)
	if err != nil {
		errs = multierr.Append(errs, NewChallengeTxtError(e, 0, err, theme))
		return
	}

	// Alert about unknown keys in challenge.txt
	if len(md.Undecoded()) > 0 {
		for _, k := range md.Undecoded() {
			errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("unknown key %q found, check https://fic.srs.epita.fr/doc/files/challenge/", k), theme))
		}
	}

	e.WIP = p.WIP
	if p.WIP && !AllowWIPExercice {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("exercice declared Work In Progress in challenge.txt"), theme))
	}

	if p.Gain == 0 {
		errs = multierr.Append(errs, NewChallengeTxtError(e, 0, fmt.Errorf("Undefined gain for challenge"), theme))
	} else {
		e.Gain = p.Gain
	}

	// Handle dependency
	if len(p.Dependencies) > 0 {
		if len(p.Dependencies[0].Theme) > 0 && (theme == nil || p.Dependencies[0].Theme != theme.Name) {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to treat dependency to another theme (%q): not implemented.", p.Dependencies[0].Theme), theme))
		} else {
			if dmap == nil {
				if dmap2, err := buildDependancyMap(i, theme); err != nil {
					errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to build dependency map: %w", err), theme))
				} else {
					dmap = &dmap2
				}
			}

			if dmap != nil {
				for edk, ed := range *dmap {
					if edk == p.Dependencies[0].Id {
						e.Depend = &ed.Id
						break
					}
				}
				if e.Depend == nil {
					dmap_keys := []string{}
					for k, _ := range *dmap {
						dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k))
					}
					errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("Unable to find required exercice dependancy %d (available at time of processing: %s)", p.Dependencies[0].Id, strings.Join(dmap_keys, ",")), theme))
				}
			}
		}
	}

	// Handle resolutions
	resolutionFound := false

	e.VideoURI = path.Join(epath, "resolution.mp4")
	if !i.Exists(e.VideoURI) {
		e.VideoURI = ""
	} else if size, err := GetFileSize(i, e.VideoURI); err != nil {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: %w", err), theme))
		e.VideoURI = ""
	} else if size == 0 {
		errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.mp4: The file is empty!"), theme))
		e.VideoURI = ""
	} else {
		e.VideoURI = strings.Replace(url.PathEscape(path.Join("$RFILES$", e.VideoURI)), "%2F", "/", -1)
		resolutionFound = true
	}

	writeup := path.Join(epath, "resolution.md")
	if !i.Exists(writeup) {
		writeup = path.Join(epath, "resolution.txt")
	}

	if i.Exists(writeup) {
		if size, err := GetFileSize(i, writeup); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
		} else if size == 0 {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: The file is empty!"), theme))
		} else if e.Resolution, err = GetFileContent(i, writeup); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err), theme))
		} else {
			// Call checks hooks
			for _, h := range hooks.mdTextHooks {
				for _, err := range multierr.Errors(h(e.Resolution, e.Language, exceptions.GetFileExceptions("resolution.md"), p.GetRawFlags()...)) {
					errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: %w", err)))
				}
			}

			if e.Resolution, err = ProcessMarkdown(i, e.Resolution, epath); err != nil {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("resolution.md: error during markdown processing: %w", err), theme))
			} else {
				resolutionFound = true
			}
		}
	}

	if !resolutionFound {
		errs = multierr.Append(errs, NewExerciceError(e, ErrResolutionNotFound, theme))
	}

	// Call checks hooks
	for _, h := range hooks.exerciceHooks {
		for _, err := range multierr.Errors(h(e, exceptions)) {
			errs = multierr.Append(errs, NewExerciceError(e, err))
		}
	}
	return
}

// SyncExercice imports new or updates existing given exercice.
func SyncExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*fic.Exercice, exceptions_in *CheckExceptions) (e *fic.Exercice, eid int, exceptions *CheckExceptions, errs error) {
	var err error
	var p ExerciceParams
	var berrors error

	e, p, eid, exceptions, _, berrors = BuildExercice(i, theme, epath, dmap, exceptions_in)
	errs = multierr.Append(errs, berrors)

	if e != nil {
		if len(e.Image) > 0 {
			if _, err := i.importFile(e.Image,
				func(filePath string, origin string) (interface{}, error) {
					if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil {
						return nil, err
					}

					e.Image = strings.TrimPrefix(filePath, fic.FilesDir)

					e.BackgroundColor, _ = getBackgroundColor(filePath)

					// If the theme has no image yet, use the first exercice's image found
					if theme != nil && theme.Image == "" {
						theme.Image = e.Image
						_, err := theme.Update()
						if err != nil {
							return nil, err
						}
					}

					return nil, nil
				}); err != nil {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to import heading image: %w", err)))
			}
		}

		// Create or update the exercice
		err = theme.SaveNamedExercice(e)
		if err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("error on exercice save: %w", err), theme))
			return
		}

		// Import eercice tags
		if _, err := e.WipeTags(); err != nil {
			errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to wipe tags: %w", err), theme))
		}
		for _, tag := range p.Tags {
			if _, err := e.AddTag(tag); err != nil {
				errs = multierr.Append(errs, NewExerciceError(e, fmt.Errorf("unable to add tag: %w", err), theme))
				return
			}
		}
	}

	return
}

// SyncExercices imports new or updates existing exercices, in a given theme.
func SyncExercices(i Importer, theme *fic.Theme, exceptions *CheckExceptions) (exceptions_out map[int]*CheckExceptions, errs error) {
	if exercices, err := GetExercices(i, theme); err != nil {
		errs = multierr.Append(errs, err)
	} else {
		exceptions_out = make(map[int]*CheckExceptions)
		emap := map[string]int{}

		dmap, _ := buildDependancyMap(i, theme)

		for _, edir := range exercices {
			e, eid, ex_exceptions, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap, exceptions)
			if e != nil {
				emap[e.Title] = eid
				dmap[int64(eid)] = e
				exceptions_out[eid] = ex_exceptions
				errs = multierr.Append(errs, cur_errs)
			}
		}

		// Remove old exercices
		if exercices, err := theme.GetExercices(); err == nil {
			for _, ex := range exercices {
				if _, ok := emap[ex.Title]; !ok {
					ex.DeleteCascade()
				}
			}
		}
	}
	return
}

// ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list.
func ApiListRemoteExercices(c *gin.Context) {
	if c.Params.ByName("thid") == "_" {
		exercices, err := GetExercices(GlobalImporter, &fic.Theme{Path: StandaloneExercicesDirectory})
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
			return
		}

		c.JSON(http.StatusOK, exercices)
		return
	}

	theme, _, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
	if theme != nil {
		exercices, err := GetExercices(GlobalImporter, theme)
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": err.Error()})
			return
		}

		c.JSON(http.StatusOK, exercices)
	} else {
		c.AbortWithStatusJSON(http.StatusInternalServerError, fmt.Errorf("%q", errs))
		return
	}
}

// ApiListRemoteExercice is an accessor letting foreign packages to access remote exercice attributes.
func ApiGetRemoteExercice(c *gin.Context) {
	if c.Params.ByName("thid") == "_" {
		exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, nil, path.Join(StandaloneExercicesDirectory, c.Params.ByName("exid")), nil, nil)
		if exercice != nil {
			c.JSON(http.StatusOK, exercice)
			return
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
			return
		}
	}

	theme, exceptions, errs := BuildTheme(GlobalImporter, c.Params.ByName("thid"))
	if theme != nil {
		exercice, _, _, _, _, errs := BuildExercice(GlobalImporter, theme, path.Join(theme.Path, c.Params.ByName("exid")), nil, exceptions)
		if exercice != nil {
			c.JSON(http.StatusOK, exercice)
			return
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
			return
		}
	} else {
		c.JSON(http.StatusInternalServerError, gin.H{"errmsg": fmt.Errorf("%q", errs)})
		return
	}
}