repochecker: add new option -skipfiledigests to speed up the checks and avoid downloading lots of content

admin/sync/exercice_files.go

@@ -11,7 +11,7 @@ import (
 	"srs.epita.fr/fic-server/libfic"
 )
 
-func buildFilesListInto(i Importer, exercice fic.Exercice, into string) (files []string, digests map[string][]byte, errs []string) {
+func BuildFilesListInto(i Importer, exercice fic.Exercice, into string) (files []string, digests map[string][]byte, errs []string) {
 	// If no files directory, don't display error
 	if !i.exists(path.Join(exercice.Path, into)) {
 		return
@@ -69,9 +69,33 @@ func buildFilesListInto(i Importer, exercice fic.Exercice, into string) (files [
 	return
 }
 
+// CheckExerciceFilesPresence limits remote checks to presence, don't get it to check digest.
+func CheckExerciceFilesPresence(i Importer, exercice fic.Exercice) (files []string, errs []string) {
+	flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
+	errs = append(errs, berrs...)
+
+	for _, fname := range flist {
+		if !i.exists(path.Join(exercice.Path, "files", fname)) {
+			errs = append(errs, fmt.Sprintf("%q: unable to read file %q: No such file or directory", path.Base(exercice.Path), fname))
+		} else if _, ok := digests[fname]; !ok {
+			errs = append(errs, fmt.Sprintf("%q: unable to import file %q: No digest given", path.Base(exercice.Path), fname))
+		} else {
+			files = append(files, fname)
+		}
+	}
+
+	for fname := range digests {
+		if !i.exists(path.Join(exercice.Path, "files", fname)) {
+			errs = append(errs, fmt.Sprintf("%q: unable to read file %q: No such file or directory. Check your DIGESTS.txt for legacy entries.", path.Base(exercice.Path), fname))
+		}
+	}
+
+	return
+}
+
 // CheckExerciceFiles checks that remote files have the right digest.
-func CheckExerciceFiles(i Importer, exercice fic.Exercice) (files []fic.EFile, errs []string) {
+func CheckExerciceFiles(i Importer, exercice fic.Exercice) (files []string, errs []string) {
-	flist, digests, berrs := buildFilesListInto(i, exercice, "files")
+	flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
 	errs = append(errs, berrs...)
 
 	for _, fname := range flist {
@@ -84,7 +108,7 @@ func CheckExerciceFiles(i Importer, exercice fic.Exercice) (files []fic.EFile, e
 			errs = append(errs, fmt.Sprintf("%q: %s: %s", path.Base(exercice.Path), fname, err))
 		}
 
-		files = append(files, fic.EFile{Name: fname})
+		files = append(files, fname)
 	}
 	return
 }
@@ -96,7 +120,7 @@ func SyncExerciceFiles(i Importer, exercice fic.Exercice) (errs []string) {
 		errs = append(errs, err.Error())
 	}
 
-	files, digests, berrs := buildFilesListInto(i, exercice, "files")
+	files, digests, berrs := BuildFilesListInto(i, exercice, "files")
 	errs = append(errs, berrs...)
 
 	// Import standard files

admin/sync/exercice_keys.go

@@ -341,7 +341,7 @@ func buildExerciceFlags(i Importer, exercice fic.Exercice) (flags map[int64]impo
 }
 
 // CheckExerciceFlags checks if all flags for the given challenge are correct.
-func CheckExerciceFlags(i Importer, exercice fic.Exercice, files []fic.EFile) (rf []fic.Flag, errs []string) {
+func CheckExerciceFlags(i Importer, exercice fic.Exercice, files []string) (rf []fic.Flag, errs []string) {
 	flags, flagsids, berrs := buildExerciceFlags(i, exercice)
 	errs = append(errs, berrs...)
 
@@ -358,7 +358,7 @@ func CheckExerciceFlags(i Importer, exercice fic.Exercice, files []fic.EFile) (r
 			for _, lf := range flag.FilesDeps {
 				found := false
 				for _, f := range files {
-					if f.Name == lf {
+					if f == lf {
 						found = true
 						break
 					}
@@ -452,7 +452,7 @@ func ApiGetRemoteExerciceFlags(ps httprouter.Params, _ []byte) (interface{}, err
 	if theme != nil {
 		exercice, _, _, _, errs := BuildExercice(GlobalImporter, *theme, path.Join(theme.Path, ps.ByName("exid")), nil)
 		if exercice != nil {
-			flags, errs := CheckExerciceFlags(GlobalImporter, *exercice, []fic.EFile{})
+			flags, errs := CheckExerciceFlags(GlobalImporter, *exercice, []string{})
 			if flags != nil {
 				return flags, nil
 			} else {

repochecker/main.go

@@ -13,15 +13,24 @@ import (
 	"srs.epita.fr/fic-server/libfic"
 )
 
+var skipFileChecks = false
+
 func checkExercice(theme fic.Theme, edir string, dmap *map[int64]fic.Exercice) (errs []string) {
 	e, _, eid, _, berrs := sync.BuildExercice(sync.GlobalImporter, theme, path.Join(theme.Path, edir), dmap)
 	errs = append(errs, berrs...)
 
 	if e != nil {
 		// Files
-		files, cerrs := sync.CheckExerciceFiles(sync.GlobalImporter, *e)
+		var files []string
+		var cerrs []string
+		if !skipFileChecks {
+			files, cerrs = sync.CheckExerciceFiles(sync.GlobalImporter, *e)
+			log.Printf("%d files checked.\n", len(files))
+		} else {
+			files, cerrs = sync.CheckExerciceFilesPresence(sync.GlobalImporter, *e)
+			log.Printf("%d files presents but not checked (please check digest yourself).\n", len(files))
+		}
 		errs = append(errs, cerrs...)
-		log.Printf("%d files checked.\n", len(files))
 
 		// Flags
 		flags, cerrs := sync.CheckExerciceFlags(sync.GlobalImporter, *e, files)
@@ -66,6 +75,7 @@ func main() {
 	flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync")
 	flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?")
 	flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required or is SHA-1 good enough?")
+	flag.BoolVar(&skipFileChecks, "skipfiledigests", skipFileChecks, "Don't perform DIGESTS checks on file to speed up the checks")
 	flag.Parse()
 
 	log.SetPrefix("[repochecker] ")