server/repochecker/main.go

package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"go.uber.org/multierr"

	"srs.epita.fr/fic-server/admin/sync"
	"srs.epita.fr/fic-server/libfic"
)

var (
	ignoreBinaryFileUnder = 1500000
	skipFileChecks        = false
	skipBinaryFileCheck   = false
	logMissingResolution  = false
)

func formatFileSize(size int) string {
	if size > 1000000000000 {
		return fmt.Sprintf("%.1f TiB", float64(size)/float64(1<<40))
	} else if size > 1000000000 {
		return fmt.Sprintf("%.1f GiB", float64(size)/float64(1<<30))
	} else if size > 1000000 {
		return fmt.Sprintf("%.1f MiB", float64(size)/float64(1<<20))
	} else if size > 1000 {
		return fmt.Sprintf("%.1f KiB", float64(size)/float64(1<<10))
	}
	return fmt.Sprintf("%d B", size)
}

func searchBinaryInGit(edir string) (ret []string) {
	// Check if git exists and if we are in a git repo
	err := exec.Command("git", "-C", edir, "remote").Run()

	if err == nil {
		cmd := exec.Command("git", "-C", edir, "log", "--all", "--numstat", "--no-renames", "-z")
		var out bytes.Buffer
		cmd.Stdout = &out
		err := cmd.Run()

		if err == nil {
			alreadySeen := map[string]string{}
			commit := ""

			scanner := bufio.NewScanner(&out)
			// Split on \n and \0 (-z option)
			scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
				for i := 0; i < len(data); i++ {
					if data[i] == '\n' || data[i] == '\000' {
						return i + 1, data[:i], nil
					}
				}
				if !atEOF {
					return 0, nil, nil
				}
				return 0, data, bufio.ErrFinalToken
			})

			for scanner.Scan() {
				if strings.HasPrefix(scanner.Text(), "commit ") {
					commit = strings.TrimPrefix(scanner.Text(), "commit ")
				} else if strings.HasPrefix(scanner.Text(), "-\t-\t") {
					fname := strings.TrimPrefix(scanner.Text(), "-\t-\t")

					cmdfile := exec.Command("git", "-C", edir, "ls-tree", "-r", "-l", commit, fname)
					var outfile bytes.Buffer
					cmdfile.Stdout = &outfile
					err = cmdfile.Run()
					var fsize int = -1024
					if err == nil {
						fields := strings.Fields(outfile.String())
						if len(fields) < 4 {
							// This should be a file deletion
							if _, ok := alreadySeen[fname]; !ok {
								alreadySeen[fname] = fmt.Sprintf("%s (commit %s) deleted", fname, commit[:7])
							}
							continue
						} else if fsize, err = strconv.Atoi(fields[3]); err == nil && fsize < ignoreBinaryFileUnder {
							if _, ok := alreadySeen[fname]; !ok || skipBinaryFileCheck {
								continue
							}
						} else if _, ok := alreadySeen[fname]; !ok && skipBinaryFileCheck {
							alreadySeen[fname] = fmt.Sprintf("%s (commit %s) (size %s)", fname, commit[:7], formatFileSize(fsize))
							continue
						}
					}
					if as, ok := alreadySeen[fname]; ok && as != "" {
						ret = append(ret, as)
						alreadySeen[fname] = ""
					}
					ret = append(ret, fmt.Sprintf("%s (commit %s) (size %s)", fname, commit[:7], formatFileSize(fsize)))
				}
			}
		}
	}

	return
}

func checkExercice(theme *fic.Theme, edir string, dmap *map[int64]*fic.Exercice, exceptions *sync.CheckExceptions) (errs error) {
	e, _, eid, exceptions, _, berrs := sync.BuildExercice(sync.GlobalImporter, theme, path.Join(theme.Path, edir), dmap, nil)
	errs = multierr.Append(errs, berrs)

	if e != nil {
		// Files
		var files []string
		var cerrs error
		if !skipFileChecks {
			files, cerrs = sync.CheckExerciceFiles(sync.GlobalImporter, e, exceptions)
			log.Printf("%d files checked.\n", len(files))
		} else {
			files, cerrs = sync.CheckExerciceFilesPresence(sync.GlobalImporter, e)
			log.Printf("%d files presents but not checked (please check digest yourself).\n", len(files))
		}
		errs = multierr.Append(errs, cerrs)

		// Flags
		flags, cerrs := sync.CheckExerciceFlags(sync.GlobalImporter, e, files, exceptions)
		errs = multierr.Append(errs, cerrs)
		log.Printf("%d flags checked.\n", len(flags))

		// Hints
		hints, cerrs := sync.CheckExerciceHints(sync.GlobalImporter, e, exceptions)
		errs = multierr.Append(errs, cerrs)
		log.Printf("%d hints checked.\n", len(hints))

		if dmap != nil {
			(*dmap)[int64(eid)] = e
		}
	}
	return
}

func main() {
	cloudDAVBase := ""
	cloudUsername := "fic"
	cloudPassword := ""
	localImporterDirectory := ""
	checkplugins := sync.CheckPluginList{}

	// Read paremeters from environment
	if v, exists := os.LookupEnv("FICCLOUD_URL"); exists {
		cloudDAVBase = v
	}
	if v, exists := os.LookupEnv("FICCLOUD_USER"); exists {
		cloudUsername = v
	}
	if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists {
		cloudPassword = v
	}

	// Read parameters from command line
	flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory,
		"Base directory where to find challenges files to import, local part")
	flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase,
		"Base directory where to find challenges files to import, cloud part")
	flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync")
	flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync")
	flag.BoolVar(&sync.AllowWIPExercice, "allow-wip-exercices", sync.AllowWIPExercice, "Are WIP exercice allowed?")
	flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?")
	flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required or is SHA-1 good enough?")
	flag.BoolVar(&skipFileChecks, "skipfiledigests", skipFileChecks, "Don't perform DIGESTS checks on file to speed up the checks")
	flag.BoolVar(&logMissingResolution, "skipresolution", logMissingResolution, "Don't fail if resolution.mp4 is absent")
	flag.BoolVar(&skipBinaryFileCheck, "skip-binary-file", skipBinaryFileCheck, "In Git-LFS check, don't warn files")
	flag.IntVar(&ignoreBinaryFileUnder, "skip-binary-files-under", ignoreBinaryFileUnder, "In Git-LFS check, don't warn files under this size")
	flag.Var(&checkplugins, "rules-plugins", "List of libraries containing others rules to checks")
	flag.Var(&sync.RemoteFileDomainWhitelist, "remote-file-domain-whitelist", "List of domains which are allowed to store remote files")
	flag.Parse()

	// Do not display timestamp
	log.SetFlags(0)

	// Instantiate importer
	regenImporter := false
	if localImporterDirectory != "" {
		sync.GlobalImporter = sync.LocalImporter{Base: localImporterDirectory, Symlink: true}
	} else if cloudDAVBase != "" {
		sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword)
	} else {
		// In this case, we want to treat the entier path given
		regenImporter = true
	}

	var err error

	// Create temporary directory for storing FILES/ content
	fic.FilesDir, err = ioutil.TempDir("", "fic-repochecker.")
	if err != nil {

	}
	defer os.RemoveAll(fic.FilesDir)

	if sync.GlobalImporter != nil {
		log.Println("Using", sync.GlobalImporter.Kind())

		if themes, err := sync.GetThemes(sync.GlobalImporter); err != nil {
			log.Fatal(err)
		} else if len(flag.Args()) == 0 {
			log.Println("Existing themes:")
			for _, th := range themes {
				log.Println("-", th)
			}
			os.Exit(1)
		}
	} else if len(flag.Args()) == 0 {
		log.Fatal("No importer nor path given!")
	}

	// Load rules plugins
	for _, p := range checkplugins {
		if err := sync.LoadChecksPlugin(p); err != nil {
			log.Fatalf("Unable to load rule plugin %q: %s", p, err.Error())
		} else {
			log.Printf("Rules plugin %q successfully loaded", p)
		}
	}

	// Variable that handles the exit status
	hasError := false

	for _, p := range flag.Args() {
		if regenImporter {
			var err error
			p, err = filepath.Abs(p)
			if err != nil {
				p = path.Clean(p)
			}
			sync.GlobalImporter = sync.LocalImporter{
				Base:    path.Dir(p),
				Symlink: true,
			}
			p = path.Base(p)
		}

		nberr := 0
		theme, exceptions, errs := sync.BuildTheme(sync.GlobalImporter, p)

		if theme != nil && !sync.GlobalImporter.Exists(path.Join(p, "challenge.toml")) && !sync.GlobalImporter.Exists(path.Join(p, "challenge.txt")) {
			thiserrors := multierr.Errors(errs)
			nberr += len(thiserrors)
			for _, err := range thiserrors {
				log.Println(err)
			}

			exercices, err := sync.GetExercices(sync.GlobalImporter, theme)
			if err != nil {
				nberr += 1
				log.Println(err)
				continue
			}

			dmap := map[int64]*fic.Exercice{}

			for _, edir := range exercices {
				ex_exceptions := exceptions.GetFileExceptions(edir)

				for _, err := range multierr.Errors(checkExercice(theme, edir, &dmap, ex_exceptions)) {
					log.Println(err.Error())

					if logMissingResolution {
						if e, ok := err.(*sync.ExerciceError); ok {
							if errors.Is(e.GetError(), sync.ErrResolutionNotFound) {
								continue
							}
						}
					}

					nberr += 1
				}
				log.Printf("================================== Exercice %q treated\n", edir)
			}

			bfile := searchBinaryInGit(path.Join(sync.GlobalImporter.(sync.LocalImporter).Base, p))
			if len(bfile) > 0 {
				fmt.Printf("\n")
				log.Println("There are some binary files in your git repository, they HAVE TO use LFS instead:")
				for _, f := range bfile {
					log.Println("  -", f)
				}
			}

			fmt.Printf("\n")
			log.Printf("Theme %q treated. %d error(s) reported.\n\n", p, nberr)
		} else {
			log.Printf("This is not a theme directory, run checks for exercice.\n\n")

			for _, err := range multierr.Errors(checkExercice(&fic.Theme{}, p, &map[int64]*fic.Exercice{}, nil)) {
				nberr += 1
				log.Println(err)
			}

			bfile := searchBinaryInGit(path.Join(sync.GlobalImporter.(sync.LocalImporter).Base, p))
			if len(bfile) > 0 {
				fmt.Printf("\n")
				log.Println("There are some binary files in your git repository, they HAVE TO use LFS instead:")
				for _, f := range bfile {
					if strings.HasPrefix(f, p) {
						log.Println("  -", f)
					}
				}
			}

			fmt.Printf("\n")
			log.Printf("Exercice %q treated. %d error(s) reported.\n", p, nberr)
		}

		if nberr > 0 {
			hasError = true
		}
	}

	if hasError {
		os.Exit(1)
	}
}