diff --git a/admin/api/sync.go b/admin/api/sync.go index 582c55e0..225b6e96 100644 --- a/admin/api/sync.go +++ b/admin/api/sync.go @@ -241,7 +241,7 @@ func declareSyncExercicesRoutes(router *gin.RouterGroup) { exceptions := sync.LoadExerciceException(sync.GlobalImporter, theme, exercice, nil) - c.JSON(http.StatusOK, flatifySyncErrors(sync.ImportExerciceFiles(sync.GlobalImporter, exercice, exceptions))) + c.JSON(http.StatusOK, flatifySyncErrors(sync.SyncExerciceFiles(sync.GlobalImporter, exercice, exceptions))) }) apiSyncExercicesRoutes.POST("/fixurlid", func(c *gin.Context) { exercice := c.MustGet("exercice").(*fic.Exercice) diff --git a/admin/sync/exercice_files.go b/admin/sync/exercice_files.go index 3ce45c80..11afe1ff 100644 --- a/admin/sync/exercice_files.go +++ b/admin/sync/exercice_files.go @@ -315,57 +315,9 @@ func DownloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice, return } -type importedFile struct { - file interface{} - Name string -} - -func SyncExerciceFiles(i Importer, exercice *fic.Exercice, paramsFiles map[string]ExerciceFile, actionAfterImport func(fname string, digests map[string][]byte, filePath, origin string) (interface{}, error)) (ret []*importedFile, errs error) { - files, digests, berrs := BuildFilesListInto(i, exercice, "files") - errs = multierr.Append(errs, berrs) - - // Import standard files - for _, fname := range files { - var f interface{} - var err error - - if pf, exists := paramsFiles[fname]; exists && pf.URL != "" && !i.Exists(path.Join(exercice.Path, "files", fname)) { - dest := GetDestinationFilePath(pf.URL, &pf.Filename) - - if _, err := os.Stat(dest); !os.IsNotExist(err) { - if d, err := actionAfterImport(fname, digests, dest, pf.URL); err == nil { - f = d - } - } - - if f == nil { - errs = multierr.Append(errs, DownloadExerciceFile(paramsFiles[fname], dest, exercice, false)) - - f, err = actionAfterImport(fname, digests, dest, pf.URL) - } - } else { - f, err = i.importFile(path.Join(exercice.Path, "files", fname), func(filePath, origin string) (interface{}, error) { - return actionAfterImport(fname, digests, filePath, origin) - }) - } - - if err != nil { - errs = multierr.Append(errs, NewFileError(exercice, fname, err)) - continue - } - - ret = append(ret, &importedFile{ - f, - fname, - }) - } - - return -} - -// ImportExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge. +// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge. // It takes care of DIGESTS.txt and ensure imported files match. -func ImportExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs error) { +func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs error) { if _, err := exercice.WipeFiles(); err != nil { errs = multierr.Append(errs, err) } @@ -376,41 +328,63 @@ func ImportExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckEx return } - actionAfterImport := func(fname string, digests map[string][]byte, filePath, origin string) (interface{}, error) { - var digest_shown []byte - if strings.HasSuffix(fname, ".gz") { - if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists { - digest_shown = d - } - } + files, digests, berrs := BuildFilesListInto(i, exercice, "files") + errs = multierr.Append(errs, berrs) - published := true - disclaimer := "" - if f, exists := paramsFiles[fname]; exists { - published = !f.Hidden - - // Call checks hooks - for _, hk := range hooks.mdTextHooks { - for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) { - errs = multierr.Append(errs, NewFileError(exercice, fname, err)) + // Import standard files + for _, fname := range files { + actionAfterImport := func(filePath string, origin string) (interface{}, error) { + var digest_shown []byte + if strings.HasSuffix(fname, ".gz") { + if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists { + digest_shown = d } } - if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil { - errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err))) + published := true + disclaimer := "" + if f, exists := paramsFiles[fname]; exists { + published = !f.Hidden + + // Call checks hooks + for _, hk := range hooks.mdTextHooks { + for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) { + errs = multierr.Append(errs, NewFileError(exercice, fname, err)) + } + } + + if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil { + errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err))) + } } + + return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published) } - return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published) - } + var f interface{} - files, berrs := SyncExerciceFiles(i, exercice, paramsFiles, actionAfterImport) - errs = multierr.Append(errs, berrs) + if pf, exists := paramsFiles[fname]; exists && pf.URL != "" && !i.Exists(path.Join(exercice.Path, "files", fname)) { + dest := GetDestinationFilePath(pf.URL, &pf.Filename) - // Import files in db - for _, file := range files { - fname := file.Name - f := file.file + if _, err := os.Stat(dest); !os.IsNotExist(err) { + if d, err := actionAfterImport(dest, pf.URL); err == nil { + f = d + } + } + + if f == nil { + errs = multierr.Append(errs, DownloadExerciceFile(paramsFiles[fname], dest, exercice, false)) + + f, err = actionAfterImport(dest, pf.URL) + } + } else { + f, err = i.importFile(path.Join(exercice.Path, "files", fname), actionAfterImport) + } + + if err != nil { + errs = multierr.Append(errs, NewFileError(exercice, fname, err)) + continue + } if f.(*fic.EFile).Size == 0 { errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!"))) diff --git a/admin/sync/exercices.go b/admin/sync/exercices.go index b452d4b6..49e1eef9 100644 --- a/admin/sync/exercices.go +++ b/admin/sync/exercices.go @@ -398,7 +398,7 @@ func SyncExercice(i Importer, theme *fic.Theme, epath string, dmap *map[int64]*f if len(e.Image) > 0 { if _, err := i.importFile(e.Image, func(filePath string, origin string) (interface{}, error) { - if err := resizePicture(i, origin, filePath, image.Rect(0, 0, 500, 300)); err != nil { + if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil { return nil, err } diff --git a/admin/sync/file.go b/admin/sync/file.go index 7efd47fa..39dc11e2 100644 --- a/admin/sync/file.go +++ b/admin/sync/file.go @@ -188,35 +188,26 @@ func GetDestinationFilePath(URI string, filename *string) string { return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), *filename) } -var fileWriter = fileWriterToFS - -func SetWriteFileFunc(writerFunc func(dest string) (io.WriteCloser, error)) { - fileWriter = writerFunc -} - -func fileWriterToFS(dest string) (io.WriteCloser, error) { +func importFile(i Importer, URI string, dest string) error { if err := os.MkdirAll(path.Dir(dest), 0751); err != nil { - return nil, err + return err } - return os.Create(dest) -} - -func importFile(i Importer, URI string, dest string) error { - if fdfrom, closer, err := GetFile(i, URI); err != nil { - os.Remove(dest) + // Write file + if fdto, err := os.Create(dest); err != nil { return err } else { - defer closer() - - fdto, err := fileWriter(dest) - if err != nil { - return err - } defer fdto.Close() - _, err = io.Copy(fdto, fdfrom) - return err + if fdfrom, closer, err := GetFile(i, URI); err != nil { + os.Remove(dest) + return err + } else { + defer closer() + + _, err = io.Copy(fdto, fdfrom) + return err + } } } diff --git a/admin/sync/full.go b/admin/sync/full.go index 1a64158c..275a1e83 100644 --- a/admin/sync/full.go +++ b/admin/sync/full.go @@ -246,7 +246,7 @@ func SyncThemeDeep(i Importer, theme *fic.Theme, tid int, themeStep uint8, excep log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path) DeepSyncProgress = 3 + uint8(tid)*themeStep + uint8(eid)*exerciceStep - errs = multierr.Append(errs, ImportExerciceFiles(i, exercice, ex_exceptions[eid])) + errs = multierr.Append(errs, SyncExerciceFiles(i, exercice, ex_exceptions[eid])) DeepSyncProgress += exerciceStep / 3 flagsBindings, ferrs := SyncExerciceFlags(i, exercice, ex_exceptions[eid]) diff --git a/admin/sync/markdown.go b/admin/sync/markdown.go index 5175b212..fe2eb12a 100644 --- a/admin/sync/markdown.go +++ b/admin/sync/markdown.go @@ -3,7 +3,9 @@ package sync import ( "bytes" "encoding/base32" + "io" "net/url" + "os" "path" "strings" @@ -87,10 +89,27 @@ func (t *imageImporterTransformer) Transform(doc *ast.Document, reader text.Read dPath := path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(t.hash[:])), iPath) child.Destination = []byte(path.Join(t.absPath, string(child.Destination))) - err := importFile(t.importer, path.Join(t.rootDir, iPath), dPath) - if err != nil { + if err := os.MkdirAll(path.Dir(dPath), 0755); err != nil { return ast.WalkStop, err } + + if fdto, err := os.Create(dPath); err != nil { + return ast.WalkStop, err + } else { + defer fdto.Close() + + if fd, closer, err := GetFile(t.importer, path.Join(t.rootDir, iPath)); err != nil { + os.Remove(dPath) + return ast.WalkStop, err + } else { + defer closer() + + _, err = io.Copy(fdto, fd) + if err != nil { + return ast.WalkStop, err + } + } + } } return ast.WalkContinue, nil diff --git a/admin/sync/themes.go b/admin/sync/themes.go index 8ac666d4..20b3c114 100644 --- a/admin/sync/themes.go +++ b/admin/sync/themes.go @@ -5,7 +5,6 @@ import ( "fmt" "image" "image/jpeg" - "io" "math/rand" "net/http" "os" @@ -41,19 +40,15 @@ func GetThemes(i Importer) (themes []string, err error) { } // resizePicture makes the given image just fill the given rectangle. -func resizePicture(i Importer, imgPath string, importedPath string, rect image.Rectangle) error { - if fl, err := i.GetFile(imgPath); err != nil { +func resizePicture(importedPath string, rect image.Rectangle) error { + if fl, err := os.Open(importedPath); err != nil { return err } else { if src, _, err := image.Decode(fl); err != nil { - if flc, ok := fl.(io.ReadCloser); ok { - flc.Close() - } + fl.Close() return err } else if src.Bounds().Max.X > rect.Max.X && src.Bounds().Max.Y > rect.Max.Y { - if flc, ok := fl.(io.ReadCloser); ok { - flc.Close() - } + fl.Close() mWidth := rect.Max.Y * src.Bounds().Max.X / src.Bounds().Max.Y mHeight := rect.Max.X * src.Bounds().Max.Y / src.Bounds().Max.X @@ -66,7 +61,7 @@ func resizePicture(i Importer, imgPath string, importedPath string, rect image.R dst := image.NewRGBA(rect) draw.CatmullRom.Scale(dst, rect, src, src.Bounds(), draw.Over, nil) - dstFile, err := fileWriter(strings.TrimSuffix(importedPath, ".jpg") + ".thumb.jpg") + dstFile, err := os.Create(strings.TrimSuffix(importedPath, ".jpg") + ".thumb.jpg") if err != nil { return err } @@ -76,7 +71,7 @@ func resizePicture(i Importer, imgPath string, importedPath string, rect image.R return err } } else { - dstFile, err := fileWriter(strings.TrimSuffix(importedPath, ".jpg") + ".thumb.jpg") + dstFile, err := os.Create(strings.TrimSuffix(importedPath, ".jpg") + ".thumb.jpg") if err != nil { return err } @@ -278,36 +273,6 @@ func BuildTheme(i Importer, tdir string) (th *fic.Theme, exceptions *CheckExcept return } -// SyncThemeFiles import all theme's related files -func SyncThemeFiles(i Importer, btheme *fic.Theme) (errs error) { - if len(btheme.Image) > 0 { - if _, err := i.importFile(btheme.Image, - func(filePath string, origin string) (interface{}, error) { - if err := resizePicture(i, origin, filePath, image.Rect(0, 0, 500, 300)); err != nil { - return nil, err - } - - btheme.Image = strings.TrimPrefix(filePath, fic.FilesDir) - btheme.BackgroundColor, _ = getBackgroundColor(filePath) - return nil, nil - }); err != nil { - errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err))) - } - } - - if len(btheme.PartnerImage) > 0 { - if _, err := i.importFile(btheme.PartnerImage, - func(filePath string, origin string) (interface{}, error) { - btheme.PartnerImage = strings.TrimPrefix(filePath, fic.FilesDir) - return nil, nil - }); err != nil { - errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import partner image: %w", err))) - } - } - - return -} - // SyncThemes imports new or updates existing themes. func SyncThemes(i Importer) (exceptions map[string]*CheckExceptions, errs error) { if themes, err := GetThemes(i); err != nil { @@ -329,9 +294,29 @@ func SyncThemes(i Importer) (exceptions map[string]*CheckExceptions, errs error) exceptions[tdir] = excepts - err = SyncThemeFiles(i, btheme) - if err != nil { - errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err))) + if len(btheme.Image) > 0 { + if _, err := i.importFile(btheme.Image, + func(filePath string, origin string) (interface{}, error) { + if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil { + return nil, err + } + + btheme.Image = strings.TrimPrefix(filePath, fic.FilesDir) + btheme.BackgroundColor, _ = getBackgroundColor(filePath) + return nil, nil + }); err != nil { + errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err))) + } + } + + if len(btheme.PartnerImage) > 0 { + if _, err := i.importFile(btheme.PartnerImage, + func(filePath string, origin string) (interface{}, error) { + btheme.PartnerImage = strings.TrimPrefix(filePath, fic.FilesDir) + return nil, nil + }); err != nil { + errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import partner image: %w", err))) + } } var theme *fic.Theme diff --git a/fileexporter/.gitignore b/fileexporter/.gitignore deleted file mode 100644 index cbf49db1..00000000 --- a/fileexporter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -fileexporter diff --git a/fileexporter/archive.go b/fileexporter/archive.go deleted file mode 100644 index 19544724..00000000 --- a/fileexporter/archive.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "archive/zip" - "errors" - "io" - "os" - "path" -) - -type archiveFileCreator interface { - Create(name string) (io.Writer, error) -} - -func init() { - OutputFormats["archive"] = func(args ...string) (func(string) (io.WriteCloser, error), error) { - if len(args) != 1 { - return nil, errors.New("archive has 1 required argument: [destination-file]") - } - - fd, err := os.Create(args[0]) - if err != nil { - return nil, err - } - - var w archiveFileCreator - if path.Ext(args[0]) == ".zip" { - w = zip.NewWriter(fd) - } else { - return nil, errors.New("destination file has to have .zip extension") - } - - return func(dest string) (io.WriteCloser, error) { - fw, err := w.Create(dest) - if err != nil { - return nil, err - } - - return NopCloser(fw), nil - }, nil - } -} diff --git a/fileexporter/copy.go b/fileexporter/copy.go deleted file mode 100644 index 6610b2c1..00000000 --- a/fileexporter/copy.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "errors" - "io" - - "srs.epita.fr/fic-server/libfic" -) - -func init() { - OutputFormats["copy"] = func(args ...string) (func(string) (io.WriteCloser, error), error) { - if len(args) > 1 { - return nil, errors.New("copy can only take 1 argument: [destination-folder]") - } - - if len(args) == 1 { - fic.FilesDir = args[0] - } - - return nil, nil - } -} diff --git a/fileexporter/main.go b/fileexporter/main.go deleted file mode 100644 index 7113c09d..00000000 --- a/fileexporter/main.go +++ /dev/null @@ -1,176 +0,0 @@ -package main - -import ( - "bytes" - "errors" - "flag" - "io" - "log" - "os" - "path" - "strings" - - "srs.epita.fr/fic-server/admin/sync" - "srs.epita.fr/fic-server/libfic" -) - -var OutputFormats = map[string]func(...string) (func(string) (io.WriteCloser, error), error){} - -func exportThemeFiles(tdir string) (errs error) { - theme, exceptions, err := sync.BuildTheme(sync.GlobalImporter, tdir) - errs = errors.Join(errs, err) - - err = sync.SyncThemeFiles(sync.GlobalImporter, theme) - if err != nil { - errs = errors.Join(errs, err) - } - - exercices, err := sync.GetExercices(sync.GlobalImporter, theme) - if err != nil { - log.Fatalf("Unable to list exercices for theme %q: %s", theme.Name, err) - } - - dmap := map[int64]*fic.Exercice{} - - for i, edir := range exercices { - log.Printf("In theme %s, doing exercice %d/%d: %s", tdir, i, len(exercices), tdir) - err = exportExerciceFiles(theme, edir, &dmap, exceptions) - errs = errors.Join(errs, err) - } - - return -} - -func exportExerciceFiles(theme *fic.Theme, edir string, dmap *map[int64]*fic.Exercice, exceptions *sync.CheckExceptions) (errs error) { - exercice, _, eid, exceptions, _, berrs := sync.BuildExercice(sync.GlobalImporter, theme, path.Join(theme.Path, edir), dmap, nil) - errs = errors.Join(errs, berrs) - - if exercice != nil { - paramsFiles, err := sync.GetExerciceFilesParams(sync.GlobalImporter, exercice) - if err != nil { - errs = errors.Join(errs, sync.NewChallengeTxtError(exercice, 0, err)) - return - } - - _, err = sync.SyncExerciceFiles(sync.GlobalImporter, exercice, paramsFiles, func(fname string, digests map[string][]byte, filePath, origin string) (interface{}, error) { - return nil, nil - }) - errs = errors.Join(errs, err) - - if dmap != nil { - (*dmap)[int64(eid)] = exercice - } - } - return -} - -type nopCloser struct { - w io.Writer -} - -func (nc *nopCloser) Close() error { - return nil -} - -func (nc *nopCloser) Write(p []byte) (int, error) { - return nc.w.Write(p) -} - -func NopCloser(w io.Writer) *nopCloser { - return &nopCloser{w} -} - -func writeFileToTar(dest string) (io.WriteCloser, error) { - log.Println("import2Tar", dest) - return NopCloser(bytes.NewBuffer([]byte{})), nil -} - -func main() { - cloudDAVBase := "" - cloudUsername := "fic" - cloudPassword := "" - localImporterDirectory := "" - - // Read paremeters from environment - if v, exists := os.LookupEnv("FICCLOUD_URL"); exists { - cloudDAVBase = v - } - if v, exists := os.LookupEnv("FICCLOUD_USER"); exists { - cloudUsername = v - } - if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists { - cloudPassword = v - } - - // Read parameters from command line - flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory, - "Base directory where to find challenges files to import, local part") - flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase, - "Base directory where to find challenges files to import, cloud part") - flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync") - flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync") - flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?") - flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required or is SHA-1 good enough?") - flag.Parse() - - // Do not display timestamp - log.SetFlags(0) - - // Instantiate importer - if localImporterDirectory != "" { - sync.GlobalImporter = sync.LocalImporter{Base: localImporterDirectory, Symlink: false} - } else if cloudDAVBase != "" { - sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword) - } - - if sync.GlobalImporter == nil { - log.Fatal("No importer configured!") - } - - log.Println("Using", sync.GlobalImporter.Kind()) - - // Configure destination - if flag.NArg() < 1 { - var formats []string - - for k := range OutputFormats { - formats = append(formats, k) - } - - log.Fatal("Please define wanted output format between [" + strings.Join(formats, " ") + "]") - } else if outputFormat, ok := OutputFormats[flag.Arg(0)]; !ok { - var formats []string - - for k := range OutputFormats { - formats = append(formats, k) - } - - log.Fatal("Please define wanted output format between [" + strings.Join(formats, " ") + "]") - } else { - fw, err := outputFormat(flag.Args()[1:]...) - if err != nil { - log.Fatal(err) - } else if fw != nil { - sync.SetWriteFileFunc(fw) - } - } - - themes, err := sync.GetThemes(sync.GlobalImporter) - if err != nil { - log.Fatal(err) - } - - hasError := false - for i, tdir := range themes { - log.Printf("Doing theme %d/%d: %s", i, len(themes), tdir) - err = exportThemeFiles(tdir) - if err != nil { - hasError = true - log.Println(err) - } - } - - if hasError { - os.Exit(1) - } -} diff --git a/repochecker/main.go b/repochecker/main.go index de6e1c57..ff49aacb 100644 --- a/repochecker/main.go +++ b/repochecker/main.go @@ -6,7 +6,7 @@ import ( "errors" "flag" "fmt" - "io" + "io/ioutil" "log" "os" "os/exec" @@ -194,8 +194,14 @@ func main() { regenImporter = true } - // Don't write any files - sync.SetWriteFileFunc(func(dest string) (io.WriteCloser, error) { return &nullFileWriter{}, nil }) + var err error + + // Create temporary directory for storing FILES/ content + fic.FilesDir, err = ioutil.TempDir("", "fic-repochecker.") + if err != nil { + + } + defer os.RemoveAll(fic.FilesDir) if sync.GlobalImporter != nil { log.Println("Using", sync.GlobalImporter.Kind()) diff --git a/repochecker/null.go b/repochecker/null.go deleted file mode 100644 index e618c541..00000000 --- a/repochecker/null.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -type nullFileWriter struct{} - -func (fw *nullFileWriter) Write(p []byte) (int, error) { - return len(p), nil -} - -func (fw *nullFileWriter) Close() error { - return nil -}