From dc4a4925e3fee461800db8c9bc071b6c2b62b285 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 5 Dec 2018 05:02:27 +0100 Subject: [PATCH] sync: refactor exercice synchronization --- admin/sync/exercice_hints.go | 4 +- admin/sync/exercices.go | 293 ++++++++++++++++++++--------------- admin/sync/markdown.go | 17 +- admin/sync/themes.go | 5 +- libfic/exercice.go | 86 ++++++++-- 5 files changed, 254 insertions(+), 151 deletions(-) diff --git a/admin/sync/exercice_hints.go b/admin/sync/exercice_hints.go index 143288c5..3afa1222 100644 --- a/admin/sync/exercice_hints.go +++ b/admin/sync/exercice_hints.go @@ -72,8 +72,8 @@ func SyncExerciceHints(i Importer, exercice fic.Exercice) (errs []string) { } else if hint.Content == "" { errs = append(errs, fmt.Sprintf("%q: challenge.txt: hint %s (%d): content and filename can't be empty at the same time", path.Base(exercice.Path), hint.Title, n+1)) continue - } else { - hint.Content = ProcessMarkdown(i, hint.Content, exercice.Path) + } else if hint.Content, err = ProcessMarkdown(i, hint.Content, exercice.Path); err != nil{ + errs = append(errs, fmt.Sprintf("%q: challenge.txt: hint %s (%d): error during markdown formating: %s", path.Base(exercice.Path), hint.Title, n+1, err)) } // Import hint diff --git a/admin/sync/exercices.go b/admin/sync/exercices.go index f06c8517..b8a484fd 100644 --- a/admin/sync/exercices.go +++ b/admin/sync/exercices.go @@ -1,6 +1,7 @@ package sync import ( + "errors" "fmt" "path" "strings" @@ -29,133 +30,183 @@ func getExercices(i Importer, theme fic.Theme) ([]string, error) { return exercices, nil } -// SyncExercices imports new or updates existing exercices, in a given theme. -func SyncExercices(i Importer, theme fic.Theme) []string { - var errs []string +func buildDependancyMap(i Importer, theme fic.Theme) (dmap map[int64]fic.Exercice, err error) { + var exercices []string + if exercices, err = getExercices(i, theme); err != nil { + return + } else { + dmap = map[int64]fic.Exercice{} + for _, edir := range exercices { + var eid int + var ename string + eid, ename, err = parseExerciceDirname(edir) + if err != nil { + return + } + + var e fic.Exercice + e, err = theme.GetExerciceByTitle(ename) + if err != nil { + return + } + + dmap[int64(eid)] = e + } + + return + } +} + +func parseExerciceDirname(edir string) (eid int, ename string, err error) { + edir_splt := strings.SplitN(edir, "-", 2) + if len(edir_splt) != 2 { + err = errors.New(fmt.Sprintf("%q is not a valid exercice directory: missing id prefix", edir)) + return + } + + eid, err = strconv.Atoi(edir_splt[0]) + if err != nil { + err = errors.New(fmt.Sprintf("%q: invalid exercice identifier: %s", edir, err)) + return + } + + ename = edir_splt[1] + + return +} + +// SyncExercice imports new or updates existing given exercice. +func SyncExercice(i Importer, theme fic.Theme, epath string, dmap *map[int64]fic.Exercice) (e fic.Exercice, eid int, errs []string) { + var err error + + e.Path = epath + edir := path.Base(epath) + + eid, e.Title, err = parseExerciceDirname(edir) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: unable to parse exercice directory: %s", edir, err)) + return + } + + e.URLId = fic.ToURLid(e.Title) + + // Texts to format using Markdown + e.Overview, err = getFileContent(i, path.Join(epath, "overview.txt")) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: overview.txt: %s", edir, err)) + } else { + e.Headline = string(blackfriday.Run([]byte(strings.Split(e.Overview, "\n")[0]))) + if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil { + errs = append(errs, fmt.Sprintf("%q: overview.txt: an error occurs during markdown formating: %s", edir, err)) + } + } + + e.Statement, err = getFileContent(i, path.Join(epath, "statement.txt")) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err)) + } else { + if e.Statement, err = ProcessMarkdown(i, e.Statement, epath); err != nil { + errs = append(errs, fmt.Sprintf("%q: statement.txt: an error occurs during markdown formating: %s", edir, err)) + } + } + + if i.exists(path.Join(epath, "finished.txt")) { + e.Finished, err = getFileContent(i, path.Join(epath, "finished.txt")) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: finished.txt: %s", edir, err)) + } else { + if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil { + errs = append(errs, fmt.Sprintf("%q: finished.txt: an error occurs during markdown formating: %s", edir, err)) + } + } + } + + // Parse challenge.txt + p, err := parseExerciceParams(i, epath) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: challenge.txt: %s", edir, err)) + return + } + + if p.Gain == 0 { + errs = append(errs, fmt.Sprintf("%q: challenge.txt: Undefined gain for challenge", edir)) + } else { + e.Gain = p.Gain + } + + // Handle dependency + if len(p.Dependencies) > 0 { + if len(p.Dependencies[0].Theme) > 0 && p.Dependencies[0].Theme != theme.Name { + errs = append(errs, fmt.Sprintf("%q: unable to treat dependency to another theme (%q): not implemented.", edir, p.Dependencies[0].Theme)) + } else { + if dmap == nil { + if dmap2, err := buildDependancyMap(i, theme); err != nil { + errs = append(errs, fmt.Sprintf("%q: unable to build dependency map: %s", edir, err)) + } else { + dmap = &dmap2 + } + } + + if dmap != nil { + for edk, ed := range *dmap { + if edk == p.Dependencies[0].Id { + e.Depend = &ed.Id + break + } + } + if e.Depend == nil { + dmap_keys := []string{} + for k, _ := range *dmap { + dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k)) + } + errs = append(errs, fmt.Sprintf("%q: Unable to find required dependancy %d (available at time of processing: %s)", edir, p.Dependencies[0].Id, strings.Join(dmap_keys, ","))) + } + } + } + } + + // Handle video + e.VideoURI = path.Join(epath, "resolution.mp4") + if !i.exists(e.VideoURI) { + errs = append(errs, fmt.Sprintf("%q: resolution.mp4: no video file found at %s", edir, e.VideoURI)) + e.VideoURI = "" + } + + // Create or update the exercice + err = theme.SaveNamedExercice(&e) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: error on exercice save: %s", edir, err)) + return + } + + // Import eercice tags + if _, err := e.WipeTags(); err != nil { + errs = append(errs, fmt.Sprintf("%q: Unable to wipe tags: %s", edir, err)) + } + for _, tag := range p.Tags { + if _, err := e.AddTag(tag); err != nil { + errs = append(errs, fmt.Sprintf("%q: Unable to add tag: %s", edir, err)) + return + } + } + return +} + +// SyncExercices imports new or updates existing exercices, in a given theme. +func SyncExercices(i Importer, theme fic.Theme) (errs []string) { if exercices, err := getExercices(i, theme); err != nil { errs = append(errs, err.Error()) } else { - dmap := map[int64]fic.Exercice{} emap := map[string]int{} + + dmap, _ := buildDependancyMap(i, theme) + for _, edir := range exercices { - edir_splt := strings.SplitN(edir, "-", 2) - if len(edir_splt) != 2 { - errs = append(errs, fmt.Sprintf("%q is not a valid exercice directory: missing id prefix", edir)) - continue - } - - eid, err := strconv.Atoi(edir_splt[0]) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: invalid exercice identifier: %s", edir, err)) - continue - } - ename := edir_splt[1] - - emap[ename] = eid - - // Overview and scenario - overview, err := getFileContent(i, path.Join(theme.Path, edir, "overview.txt")) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: overview.txt: %s", edir, err)) - } - ovrvw := strings.Split(overview, "\n") - headline := ovrvw[0] - - statement, err := getFileContent(i, path.Join(theme.Path, edir, "statement.txt")) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err)) - continue - } - - var finished string - if i.exists(path.Join(theme.Path, edir, "finished.txt")) { - finished, err = getFileContent(i, path.Join(theme.Path, edir, "finished.txt")) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err)) - continue - } - } - - // Handle score gain - var gain int64 - var depend *fic.Exercice - var tags []string - if p, err := parseExerciceParams(i, path.Join(theme.Path, edir)); err != nil { - errs = append(errs, fmt.Sprintf("%q: challenge.txt: %s", edir, err)) - continue - } else if p.Gain == 0 { - errs = append(errs, fmt.Sprintf("%q: challenge.txt: Undefined gain for challenge", edir)) - } else { - gain = p.Gain - tags = p.Tags - - // Handle dependency - if len(p.Dependencies) > 0 { - if len(p.Dependencies[0].Theme) > 0 && p.Dependencies[0].Theme != theme.Name { - errs = append(errs, fmt.Sprintf("%q: unable to treat dependency to another theme: not implemented.", edir)) - } else { - for ed, e := range dmap { - if ed == p.Dependencies[0].Id { - depend = &e - break - } - } - if depend == nil { - dmap_keys := []string{} - for k, _ := range dmap { - dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k)) - } - errs = append(errs, fmt.Sprintf("%q: Unable to find required dependancy %d (available at time of processing: %s)", edir, p.Dependencies[0].Id, strings.Join(dmap_keys, ","))) - } - } - } - } - - // Handle video - videoURI := path.Join(theme.Path, edir, "resolution.mp4") - if !i.exists(videoURI) { - errs = append(errs, fmt.Sprintf("%q: resolution.mp4: no video file found at %s", edir, videoURI)) - videoURI = "" - } - - // Markdown pre-formating - statement = ProcessMarkdown(i, statement, edir) - overview = ProcessMarkdown(i, overview, edir) - headline = string(blackfriday.Run([]byte(headline))) - finished = ProcessMarkdown(i, finished, edir) - - e, err := theme.GetExerciceByTitle(ename) - if err != nil { - if e, err = theme.AddExercice(ename, fic.ToURLid(ename), path.Join(theme.Path, edir), statement, overview, headline, depend, gain, videoURI, finished); err != nil { - errs = append(errs, fmt.Sprintf("%q: error on exercice add: %s", edir, err)) - continue - } - } else if e.Title != ename || e.URLId == "" || e.Statement != statement || e.Overview != overview || e.Headline != headline || e.Gain != gain || e.VideoURI != videoURI || e.Finished != finished { - e.Title = ename - e.URLId = fic.ToURLid(ename) - e.Statement = statement - e.Overview = overview - e.Headline = headline - e.Finished = finished - e.Gain = gain - e.VideoURI = videoURI - if _, err := e.Update(); err != nil { - errs = append(errs, fmt.Sprintf("%q: error on exercice update: %s", edir, err)) - continue - } - } + e, eid, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap) + emap[e.Title] = eid dmap[int64(eid)] = e - - if _, err := e.WipeTags(); err != nil { - errs = append(errs, fmt.Sprintf("%q: Unable to wipe tags: %s", edir, err)) - } - for _, tag := range tags { - if _, err := e.AddTag(tag); err != nil { - errs = append(errs, fmt.Sprintf("%q: Unable to add tag: %s", edir, err)) - continue - } - } + errs = append(errs, cur_errs...) } // Remove old exercices @@ -167,7 +218,7 @@ func SyncExercices(i Importer, theme fic.Theme) []string { } } } - return errs + return } // ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list. diff --git a/admin/sync/markdown.go b/admin/sync/markdown.go index a505408d..fda44bed 100644 --- a/admin/sync/markdown.go +++ b/admin/sync/markdown.go @@ -3,7 +3,6 @@ package sync import ( "bufio" "encoding/base32" - "log" "os" "path" "regexp" @@ -15,7 +14,7 @@ import ( "gopkg.in/russross/blackfriday.v2" ) -func ProcessMarkdown(i Importer, input string, rootDir string) (output string) { +func ProcessMarkdown(i Importer, input string, rootDir string) (output string, err error) { // Define the path where save linked files hash := blake2b.Sum512([]byte(rootDir)) absPath := "$FILES$/" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])) @@ -32,9 +31,9 @@ func ProcessMarkdown(i Importer, input string, rootDir string) (output string) { )) // Import files - re, err := regexp.Compile(strings.Replace(absPath, "$", "\\$", -1) + "/[^\"]+") + var re *regexp.Regexp + re, err = regexp.Compile(strings.Replace(absPath, "$", "\\$", -1) + "/[^\"]+") if err != nil { - log.Println("Unable to compile regexp:", err) return } files := re.FindAllString(output, -1) @@ -43,20 +42,18 @@ func ProcessMarkdown(i Importer, input string, rootDir string) (output string) { iPath := strings.TrimPrefix(filePath, absPath) dPath := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), iPath) - if err := os.MkdirAll(path.Dir(dPath), 0755); err != nil { - log.Println("Unable to make directories:", err) + if err = os.MkdirAll(path.Dir(dPath), 0755); err != nil { return } - if fdto, err := os.Create(dPath); err != nil { - log.Println("Unable to create destination file:", err) + var fdto *os.File + if fdto, err = os.Create(dPath); err != nil { return } else { defer fdto.Close() writer := bufio.NewWriter(fdto) - if err := getFile(i, rootDir + iPath, writer); err != nil { + if err = getFile(i, rootDir + iPath, writer); err != nil { os.Remove(dPath) - log.Println("Unable to create destination file:", err) return } } diff --git a/admin/sync/themes.go b/admin/sync/themes.go index daf85756..8fe2f162 100644 --- a/admin/sync/themes.go +++ b/admin/sync/themes.go @@ -99,7 +99,10 @@ func SyncThemes(i Importer) []string { authors_str := strings.Join(authors, ", ") // Format overview (markdown) - intro = ProcessMarkdown(i, intro, tdir) + intro, err = ProcessMarkdown(i, intro, tdir) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: overview.txt: an error occurs during markdown formating: %s", tdir, err)) + } headline = string(blackfriday.Run([]byte(headline))) if i.exists(path.Join(tdir, "heading.jpg")) { diff --git a/libfic/exercice.go b/libfic/exercice.go index cb71ffcd..e0877f40 100644 --- a/libfic/exercice.go +++ b/libfic/exercice.go @@ -2,6 +2,7 @@ package fic import ( "errors" + "fmt" "time" ) @@ -122,25 +123,76 @@ func (t Theme) GetExercices() ([]Exercice, error) { } } -// AddExercice creates and fills a new struct Exercice and registers it into the database. -func (t Theme) AddExercice(title string, urlId string, path string, statement string, overview string, headline string, depend *Exercice, gain int64, videoURI string, finished string) (Exercice, error) { - var dpd interface{} - if depend == nil { - dpd = nil - } else { - dpd = depend.Id - } - if res, err := DBExec("INSERT INTO exercices (id_theme, title, url_id, path, statement, overview, headline, issue, depend, gain, video_uri, finished) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", t.Id, title, urlId, path, statement, overview, headline, "", dpd, gain, videoURI, finished); err != nil { - return Exercice{}, err - } else if eid, err := res.LastInsertId(); err != nil { - return Exercice{}, err - } else { - if depend == nil { - return Exercice{eid, title, urlId, path, statement, overview, headline, finished, "", "info", nil, gain, 1.0, videoURI}, nil - } else { - return Exercice{eid, title, urlId, path, statement, overview, headline, finished, "", "info", &depend.Id, gain, 1.0, videoURI}, nil +// SaveNamedExercice looks for an exercice with the same title to update it, or create it if it doesn't exists yet. +func (t Theme) SaveNamedExercice(e *Exercice) (err error) { + var search Exercice + if search, err = t.GetExerciceByTitle(e.Title); err == nil { + // Force ID + e.Id = search.Id + + // Don't expect those values + if e.Coefficient == 0 { + e.Coefficient = search.Coefficient } + if len(e.Issue) == 0 { + e.Issue = search.Issue + } + if len(e.IssueKind) == 0 { + e.IssueKind = search.IssueKind + } + + _, err = e.Update() + } else { + err = t.addExercice(e) } + return +} + +func (t Theme) addExercice(e *Exercice) (err error) { + var ik = "DEFAULT" + if len(e.IssueKind) > 0 { + ik = fmt.Sprintf("%q", e.IssueKind) + } + + var cc = "DEFAULT" + if e.Coefficient != 0 { + cc = fmt.Sprintf("%f", e.Coefficient) + } + + if res, err := DBExec("INSERT INTO exercices (id_theme, title, url_id, path, statement, overview, finished, headline, issue, depend, gain, video_uri, issue_kind, coefficient_cur) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + ik + ", " + cc + ")", t.Id, e.Title, e.URLId, e.Path, e.Statement, e.Overview, e.Finished, e.Headline, e.Issue, e.Depend, e.Gain, e.VideoURI); err != nil { + return err + } else if eid, err := res.LastInsertId(); err != nil { + return err + } else { + e.Id = eid + + return nil + } +} + +// AddExercice creates and fills a new struct Exercice and registers it into the database. +func (t Theme) AddExercice(title string, urlId string, path string, statement string, overview string, headline string, depend *Exercice, gain int64, videoURI string, finished string) (e Exercice, err error) { + var dpd *int64 = nil + if depend != nil { + dpd = &depend.Id + } + + e = Exercice{ + Title: title, + URLId: urlId, + Path: path, + Statement: statement, + Overview: overview, + Headline: headline, + Depend: dpd, + Finished: finished, + Gain: gain, + VideoURI: videoURI, + } + + err = t.addExercice(&e) + + return } // Update applies modifications back to the database.