sync: refactor exercice synchronization

admin/sync/exercice_hints.go

@@ -72,8 +72,8 @@ func SyncExerciceHints(i Importer, exercice fic.Exercice) (errs []string) {
 			} else if hint.Content == "" {
 				errs = append(errs, fmt.Sprintf("%q: challenge.txt: hint %s (%d): content and filename can't be empty at the same time", path.Base(exercice.Path), hint.Title, n+1))
 				continue
-			} else {
+			} else if hint.Content, err = ProcessMarkdown(i, hint.Content, exercice.Path); err != nil{
-				hint.Content = ProcessMarkdown(i, hint.Content, exercice.Path)
+				errs = append(errs, fmt.Sprintf("%q: challenge.txt: hint %s (%d): error during markdown formating: %s", path.Base(exercice.Path), hint.Title, n+1, err))
 			}
 
 			// Import hint

admin/sync/exercices.go

@@ -1,6 +1,7 @@
 package sync
 
 import (
+	"errors"
 	"fmt"
 	"path"
 	"strings"
@@ -29,133 +30,183 @@ func getExercices(i Importer, theme fic.Theme) ([]string, error) {
 	return exercices, nil
 }
 
-// SyncExercices imports new or updates existing exercices, in a given theme.
+func buildDependancyMap(i Importer, theme fic.Theme) (dmap map[int64]fic.Exercice, err error) {
-func SyncExercices(i Importer, theme fic.Theme) []string {
+	var exercices []string
-	var errs []string
+	if exercices, err = getExercices(i, theme); err != nil {
+		return
+	} else {
+		dmap = map[int64]fic.Exercice{}
 
+		for _, edir := range exercices {
+			var eid int
+			var ename string
+			eid, ename, err = parseExerciceDirname(edir)
+			if err != nil {
+				return
+			}
+
+			var e fic.Exercice
+			e, err = theme.GetExerciceByTitle(ename)
+			if err != nil {
+				return
+			}
+
+			dmap[int64(eid)] = e
+		}
+
+		return
+	}
+}
+
+func parseExerciceDirname(edir string) (eid int, ename string, err error) {
+	edir_splt := strings.SplitN(edir, "-", 2)
+	if len(edir_splt) != 2 {
+		err = errors.New(fmt.Sprintf("%q is not a valid exercice directory: missing id prefix", edir))
+		return
+	}
+
+	eid, err = strconv.Atoi(edir_splt[0])
+	if err != nil {
+		err = errors.New(fmt.Sprintf("%q: invalid exercice identifier: %s", edir, err))
+		return
+	}
+
+	ename = edir_splt[1]
+
+	return
+}
+
+// SyncExercice imports new or updates existing given exercice.
+func SyncExercice(i Importer, theme fic.Theme, epath string, dmap *map[int64]fic.Exercice) (e fic.Exercice, eid int, errs []string) {
+	var err error
+
+	e.Path = epath
+	edir := path.Base(epath)
+
+	eid, e.Title, err = parseExerciceDirname(edir)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("%q: unable to parse exercice directory: %s", edir, err))
+		return
+	}
+
+	e.URLId = fic.ToURLid(e.Title)
+
+	// Texts to format using Markdown
+	e.Overview, err = getFileContent(i, path.Join(epath, "overview.txt"))
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("%q: overview.txt: %s", edir, err))
+	} else {
+		e.Headline = string(blackfriday.Run([]byte(strings.Split(e.Overview, "\n")[0])))
+		if e.Overview, err = ProcessMarkdown(i, e.Overview, epath); err != nil {
+			errs = append(errs, fmt.Sprintf("%q: overview.txt: an error occurs during markdown formating: %s", edir, err))
+		}
+	}
+
+	e.Statement, err = getFileContent(i, path.Join(epath, "statement.txt"))
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err))
+	} else {
+		if e.Statement, err = ProcessMarkdown(i, e.Statement, epath); err != nil {
+			errs = append(errs, fmt.Sprintf("%q: statement.txt: an error occurs during markdown formating: %s", edir, err))
+		}
+	}
+
+	if i.exists(path.Join(epath, "finished.txt")) {
+		e.Finished, err = getFileContent(i, path.Join(epath, "finished.txt"))
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("%q: finished.txt: %s", edir, err))
+		} else {
+			if e.Finished, err = ProcessMarkdown(i, e.Finished, epath); err != nil {
+				errs = append(errs, fmt.Sprintf("%q: finished.txt: an error occurs during markdown formating: %s", edir, err))
+			}
+		}
+	}
+
+	// Parse challenge.txt
+	p, err := parseExerciceParams(i, epath)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("%q: challenge.txt: %s", edir, err))
+		return
+	}
+
+	if p.Gain == 0 {
+		errs = append(errs, fmt.Sprintf("%q: challenge.txt: Undefined gain for challenge", edir))
+	} else {
+		e.Gain = p.Gain
+	}
+
+	// Handle dependency
+	if len(p.Dependencies) > 0 {
+		if len(p.Dependencies[0].Theme) > 0 && p.Dependencies[0].Theme != theme.Name {
+			errs = append(errs, fmt.Sprintf("%q: unable to treat dependency to another theme (%q): not implemented.", edir, p.Dependencies[0].Theme))
+		} else {
+			if dmap == nil {
+				if dmap2, err := buildDependancyMap(i, theme); err != nil {
+					errs = append(errs, fmt.Sprintf("%q: unable to build dependency map: %s", edir, err))
+				} else {
+					dmap = &dmap2
+				}
+			}
+
+			if dmap != nil {
+				for edk, ed := range *dmap {
+					if edk == p.Dependencies[0].Id {
+						e.Depend = &ed.Id
+						break
+					}
+				}
+				if e.Depend == nil {
+					dmap_keys := []string{}
+					for k, _ := range *dmap {
+						dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k))
+					}
+					errs = append(errs, fmt.Sprintf("%q: Unable to find required dependancy %d (available at time of processing: %s)", edir, p.Dependencies[0].Id, strings.Join(dmap_keys, ",")))
+				}
+			}
+		}
+	}
+
+	// Handle video
+	e.VideoURI = path.Join(epath, "resolution.mp4")
+	if !i.exists(e.VideoURI) {
+		errs = append(errs, fmt.Sprintf("%q: resolution.mp4: no video file found at %s", edir, e.VideoURI))
+		e.VideoURI = ""
+	}
+
+	// Create or update the exercice
+	err = theme.SaveNamedExercice(&e)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("%q: error on exercice save: %s", edir, err))
+		return
+	}
+
+	// Import eercice tags
+	if _, err := e.WipeTags(); err != nil {
+		errs = append(errs, fmt.Sprintf("%q: Unable to wipe tags: %s", edir, err))
+	}
+	for _, tag := range p.Tags {
+		if _, err := e.AddTag(tag); err != nil {
+			errs = append(errs, fmt.Sprintf("%q: Unable to add tag: %s", edir, err))
+			return
+		}
+	}
+	return
+}
+
+// SyncExercices imports new or updates existing exercices, in a given theme.
+func SyncExercices(i Importer, theme fic.Theme) (errs []string) {
 	if exercices, err := getExercices(i, theme); err != nil {
 		errs = append(errs, err.Error())
 	} else {
-		dmap := map[int64]fic.Exercice{}
 		emap := map[string]int{}
+
+		dmap, _ := buildDependancyMap(i, theme)
+
 		for _, edir := range exercices {
-			edir_splt := strings.SplitN(edir, "-", 2)
+			e, eid, cur_errs := SyncExercice(i, theme, path.Join(theme.Path, edir), &dmap)
-			if len(edir_splt) != 2 {
+			emap[e.Title] = eid
-				errs = append(errs, fmt.Sprintf("%q is not a valid exercice directory: missing id prefix", edir))
-				continue
-			}
-
-			eid, err := strconv.Atoi(edir_splt[0])
-			if err != nil {
-				errs = append(errs, fmt.Sprintf("%q: invalid exercice identifier: %s", edir, err))
-				continue
-			}
-			ename := edir_splt[1]
-
-			emap[ename] = eid
-
-			// Overview and scenario
-			overview, err := getFileContent(i, path.Join(theme.Path, edir, "overview.txt"))
-			if err != nil {
-				errs = append(errs, fmt.Sprintf("%q: overview.txt: %s", edir, err))
-			}
-			ovrvw := strings.Split(overview, "\n")
-			headline := ovrvw[0]
-
-			statement, err := getFileContent(i, path.Join(theme.Path, edir, "statement.txt"))
-			if err != nil {
-				errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err))
-				continue
-			}
-
-			var finished string
-			if i.exists(path.Join(theme.Path, edir, "finished.txt")) {
-				finished, err = getFileContent(i, path.Join(theme.Path, edir, "finished.txt"))
-				if err != nil {
-					errs = append(errs, fmt.Sprintf("%q: statement.txt: %s", edir, err))
-					continue
-				}
-			}
-
-			// Handle score gain
-			var gain int64
-			var depend *fic.Exercice
-			var tags []string
-			if p, err := parseExerciceParams(i, path.Join(theme.Path, edir)); err != nil {
-				errs = append(errs, fmt.Sprintf("%q: challenge.txt: %s", edir, err))
-				continue
-			} else if p.Gain == 0 {
-				errs = append(errs, fmt.Sprintf("%q: challenge.txt: Undefined gain for challenge", edir))
-			} else {
-				gain = p.Gain
-				tags = p.Tags
-
-				// Handle dependency
-				if len(p.Dependencies) > 0 {
-					if len(p.Dependencies[0].Theme) > 0 && p.Dependencies[0].Theme != theme.Name {
-						errs = append(errs, fmt.Sprintf("%q: unable to treat dependency to another theme: not implemented.", edir))
-					} else {
-						for ed, e := range dmap {
-							if ed == p.Dependencies[0].Id {
-								depend = &e
-								break
-							}
-						}
-						if depend == nil {
-							dmap_keys := []string{}
-							for k, _ := range dmap {
-								dmap_keys = append(dmap_keys, fmt.Sprintf("%d", k))
-							}
-							errs = append(errs, fmt.Sprintf("%q: Unable to find required dependancy %d (available at time of processing: %s)", edir, p.Dependencies[0].Id, strings.Join(dmap_keys, ",")))
-						}
-					}
-				}
-			}
-
-			// Handle video
-			videoURI := path.Join(theme.Path, edir, "resolution.mp4")
-			if !i.exists(videoURI) {
-				errs = append(errs, fmt.Sprintf("%q: resolution.mp4: no video file found at %s", edir, videoURI))
-				videoURI = ""
-			}
-
-			// Markdown pre-formating
-			statement = ProcessMarkdown(i, statement, edir)
-			overview = ProcessMarkdown(i, overview, edir)
-			headline = string(blackfriday.Run([]byte(headline)))
-			finished = ProcessMarkdown(i, finished, edir)
-
-			e, err := theme.GetExerciceByTitle(ename)
-			if err != nil {
-				if e, err = theme.AddExercice(ename, fic.ToURLid(ename), path.Join(theme.Path, edir), statement, overview, headline, depend, gain, videoURI, finished); err != nil {
-					errs = append(errs, fmt.Sprintf("%q: error on exercice add: %s", edir, err))
-					continue
-				}
-			} else if e.Title != ename || e.URLId == "" || e.Statement != statement || e.Overview != overview || e.Headline != headline || e.Gain != gain || e.VideoURI != videoURI || e.Finished != finished {
-				e.Title = ename
-				e.URLId = fic.ToURLid(ename)
-				e.Statement = statement
-				e.Overview = overview
-				e.Headline = headline
-				e.Finished = finished
-				e.Gain = gain
-				e.VideoURI = videoURI
-				if _, err := e.Update(); err != nil {
-					errs = append(errs, fmt.Sprintf("%q: error on exercice update: %s", edir, err))
-					continue
-				}
-			}
 			dmap[int64(eid)] = e
-
+			errs = append(errs, cur_errs...)
-			if _, err := e.WipeTags(); err != nil {
-				errs = append(errs, fmt.Sprintf("%q: Unable to wipe tags: %s", edir, err))
-			}
-			for _, tag := range tags {
-				if _, err := e.AddTag(tag); err != nil {
-					errs = append(errs, fmt.Sprintf("%q: Unable to add tag: %s", edir, err))
-					continue
-				}
-			}
 		}
 
 		// Remove old exercices
@@ -167,7 +218,7 @@ func SyncExercices(i Importer, theme fic.Theme) []string {
 			}
 		}
 	}
-	return errs
+	return
 }
 
 // ApiListRemoteExercices is an accessor letting foreign packages to access remote exercices list.

admin/sync/markdown.go

@@ -3,7 +3,6 @@ package sync
 import (
 	"bufio"
 	"encoding/base32"
-	"log"
 	"os"
 	"path"
 	"regexp"
@@ -15,7 +14,7 @@ import (
 	"gopkg.in/russross/blackfriday.v2"
 )
 
-func ProcessMarkdown(i Importer, input string, rootDir string) (output string) {
+func ProcessMarkdown(i Importer, input string, rootDir string) (output string, err error) {
 	// Define the path where save linked files
 	hash := blake2b.Sum512([]byte(rootDir))
 	absPath := "$FILES$/" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:]))
@@ -32,9 +31,9 @@ func ProcessMarkdown(i Importer, input string, rootDir string) (output string) {
 	))
 
 	// Import files
-	re, err := regexp.Compile(strings.Replace(absPath, "$", "\\$", -1) + "/[^\"]+")
+	var re *regexp.Regexp
+	re, err = regexp.Compile(strings.Replace(absPath, "$", "\\$", -1) + "/[^\"]+")
 	if err != nil {
-		log.Println("Unable to compile regexp:", err)
 		return
 	}
 	files := re.FindAllString(output, -1)
@@ -43,20 +42,18 @@ func ProcessMarkdown(i Importer, input string, rootDir string) (output string) {
 		iPath := strings.TrimPrefix(filePath, absPath)
 		dPath := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), iPath)
 
-		if err := os.MkdirAll(path.Dir(dPath), 0755); err != nil {
+		if err = os.MkdirAll(path.Dir(dPath), 0755); err != nil {
-			log.Println("Unable to make directories:", err)
 			return
 		}
 
-		if fdto, err := os.Create(dPath); err != nil {
+		var fdto *os.File
-			log.Println("Unable to create destination file:", err)
+		if fdto, err = os.Create(dPath); err != nil {
 			return
 		} else {
 			defer fdto.Close()
 			writer := bufio.NewWriter(fdto)
-			if err := getFile(i, rootDir + iPath, writer); err != nil {
+			if err = getFile(i, rootDir + iPath, writer); err != nil {
 				os.Remove(dPath)
-				log.Println("Unable to create destination file:", err)
 				return
 			}
 		}

admin/sync/themes.go

@@ -99,7 +99,10 @@ func SyncThemes(i Importer) []string {
 			authors_str := strings.Join(authors, ", ")
 
 			// Format overview (markdown)
-			intro = ProcessMarkdown(i, intro, tdir)
+			intro, err = ProcessMarkdown(i, intro, tdir)
+			if err != nil {
+				errs = append(errs, fmt.Sprintf("%q: overview.txt: an error occurs during markdown formating: %s", tdir, err))
+			}
 			headline = string(blackfriday.Run([]byte(headline)))
 
 			if i.exists(path.Join(tdir, "heading.jpg")) {

libfic/exercice.go

@@ -2,6 +2,7 @@ package fic
 
 import (
 	"errors"
+	"fmt"
 	"time"
 )
 
@@ -122,25 +123,76 @@ func (t Theme) GetExercices() ([]Exercice, error) {
 	}
 }
 
-// AddExercice creates and fills a new struct Exercice and registers it into the database.
+// SaveNamedExercice looks for an exercice with the same title to update it, or create it if it doesn't exists yet.
-func (t Theme) AddExercice(title string, urlId string, path string, statement string, overview string, headline string, depend *Exercice, gain int64, videoURI string, finished string) (Exercice, error) {
+func (t Theme) SaveNamedExercice(e *Exercice) (err error) {
-	var dpd interface{}
+	var search Exercice
-	if depend == nil {
+	if search, err = t.GetExerciceByTitle(e.Title); err == nil {
-		dpd = nil
+		// Force ID
-	} else {
+		e.Id = search.Id
-		dpd = depend.Id
+
-	}
+		// Don't expect those values
-	if res, err := DBExec("INSERT INTO exercices (id_theme, title, url_id, path, statement, overview, headline, issue, depend, gain, video_uri, finished) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", t.Id, title, urlId, path, statement, overview, headline, "", dpd, gain, videoURI, finished); err != nil {
+		if e.Coefficient == 0 {
-		return Exercice{}, err
+			e.Coefficient = search.Coefficient
-	} else if eid, err := res.LastInsertId(); err != nil {
-		return Exercice{}, err
-	} else {
-		if depend == nil {
-			return Exercice{eid, title, urlId, path, statement, overview, headline, finished, "", "info", nil, gain, 1.0, videoURI}, nil
-		} else {
-			return Exercice{eid, title, urlId, path, statement, overview, headline, finished, "", "info", &depend.Id, gain, 1.0, videoURI}, nil
 		}
+		if len(e.Issue) == 0 {
+			e.Issue = search.Issue
+		}
+		if len(e.IssueKind) == 0 {
+			e.IssueKind = search.IssueKind
+		}
+
+		_, err = e.Update()
+	} else {
+		err = t.addExercice(e)
 	}
+	return
+}
+
+func (t Theme) addExercice(e *Exercice) (err error) {
+	var ik = "DEFAULT"
+	if len(e.IssueKind) > 0 {
+		ik = fmt.Sprintf("%q", e.IssueKind)
+	}
+
+	var cc = "DEFAULT"
+	if e.Coefficient != 0 {
+		cc = fmt.Sprintf("%f", e.Coefficient)
+	}
+
+	if res, err := DBExec("INSERT INTO exercices (id_theme, title, url_id, path, statement, overview, finished, headline, issue, depend, gain, video_uri, issue_kind, coefficient_cur) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + ik + ", " + cc + ")", t.Id, e.Title, e.URLId, e.Path, e.Statement, e.Overview, e.Finished, e.Headline, e.Issue, e.Depend, e.Gain, e.VideoURI); err != nil {
+		return err
+	} else if eid, err := res.LastInsertId(); err != nil {
+		return err
+	} else {
+		e.Id = eid
+
+		return nil
+	}
+}
+
+// AddExercice creates and fills a new struct Exercice and registers it into the database.
+func (t Theme) AddExercice(title string, urlId string, path string, statement string, overview string, headline string, depend *Exercice, gain int64, videoURI string, finished string) (e Exercice, err error) {
+	var dpd *int64 = nil
+	if depend != nil {
+		dpd = &depend.Id
+	}
+
+	e = Exercice{
+		Title: title,
+		URLId: urlId,
+		Path: path,
+		Statement: statement,
+		Overview: overview,
+		Headline: headline,
+		Depend: dpd,
+		Finished: finished,
+		Gain: gain,
+		VideoURI: videoURI,
+	}
+
+	err = t.addExercice(&e)
+
+	return
 }
 
 // Update applies modifications back to the database.