From 17dd69ac30d0fdba244c7e17b33f3f4c247f372f Mon Sep 17 00:00:00 2001 From: nemunaire Date: Tue, 12 Dec 2017 07:13:38 +0100 Subject: [PATCH] admin: localimporter can make symlink instead of copying whole files --- admin/main.go | 41 +++++++++++++++++++++++++--------- admin/sync/exercice_hints.go | 4 ++-- admin/sync/file.go | 5 +---- admin/sync/importer_cloud.go | 8 +++---- admin/sync/importer_localfs.go | 29 +++++++++++++++++++++--- 5 files changed, 63 insertions(+), 24 deletions(-) diff --git a/admin/main.go b/admin/main.go index c82fbf4c..0b2a93f9 100644 --- a/admin/main.go +++ b/admin/main.go @@ -62,15 +62,21 @@ func StripPrefix(prefix string, h http.Handler) http.Handler { } func main() { + cloudDAVBase := "" + cloudUsername := "fic" + cloudPassword := "" + localImporterDirectory := "" + localImporterSymlink := false + // Read paremeters from environment if v, exists := os.LookupEnv("FICCLOUD_URL"); exists { - sync.CloudDAVBase = v + cloudDAVBase = v } if v, exists := os.LookupEnv("FICCLOUD_USER"); exists { - sync.CloudUsername = v + cloudUsername = v } if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists { - sync.CloudPassword = v + cloudPassword = v } // Read parameters from command line @@ -82,19 +88,32 @@ func main() { flag.StringVar(&api.TeamsDir, "teams", "./TEAMS", "Base directory where save teams JSON files") flag.StringVar(&settings.SettingsDir, "settings", settings.SettingsDir, "Base directory where load and save settings") flag.StringVar(&fic.FilesDir, "files", fic.FilesDir, "Base directory where found challenges files, local part") - flag.StringVar(&sync.CloudDAVBase, "clouddav", sync.CloudDAVBase, - "Base directory where found challenges files, cloud part") - flag.StringVar(&sync.CloudUsername, "clouduser", sync.CloudUsername, "Username used to sync") - flag.StringVar(&sync.CloudPassword, "cloudpass", sync.CloudPassword, "Password used to sync") + flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory, + "Base directory where found challenges files to import, local part") + flag.BoolVar(&localImporterSymlink, "localimportsymlink", localImporterSymlink, + "Copy files or just create symlink?") + flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase, + "Base directory where found challenges files to import, cloud part") + flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync") + flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync") flag.BoolVar(&fic.OptionalDigest, "optionaldigest", fic.OptionalDigest, "Is the digest required when importing files?") - flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required instead of SHA-1 or BLAKE2b?") + flag.BoolVar(&fic.StrongDigest, "strongdigest", fic.StrongDigest, "Are BLAKE2b digests required or is SHA-1 good enough?") flag.Parse() log.SetPrefix("[admin] ") - //sync.GlobalImporter = sync.LocalImporter{"/mnt/fic/"} - sync.GlobalImporter, _ = sync.NewCloudImporter(sync.CloudDAVBase, sync.CloudUsername, sync.CloudPassword) - log.Println("Using", sync.GlobalImporter.Kind()) + // Instantiate importer + if localImporterDirectory != "" && cloudDAVBase != "" { + log.Fatal("Cannot have both --clouddav and --localimport defined.") + return + } else if localImporterDirectory != "" { + sync.GlobalImporter = sync.LocalImporter{localImporterDirectory, localImporterSymlink} + } else if cloudDAVBase != "" { + sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword) + } + if sync.GlobalImporter != nil { + log.Println("Using", sync.GlobalImporter.Kind()) + } // Sanitize options var err error diff --git a/admin/sync/exercice_hints.go b/admin/sync/exercice_hints.go index a7070089..2988e4fc 100644 --- a/admin/sync/exercice_hints.go +++ b/admin/sync/exercice_hints.go @@ -15,11 +15,11 @@ func SyncExerciceHints(i Importer, exercice fic.Exercice) []string { } else if hints, err := i.listDir(path.Join(exercice.Path, "hints")); err != nil { errs = append(errs, err.Error()) } else { - for _, hfile := range hints { + for n, hfile := range hints { if hint_cnt, err := getFileContent(i, path.Join(exercice.Path, "hints", hfile)); err != nil { errs = append(errs, fmt.Sprintf("%q: unable to read hint file %q: %s", path.Base(exercice.Path), hfile, err)) continue - } else if _, err := exercice.AddHint(hfile, hint_cnt, 1); err != nil { + } else if _, err := exercice.AddHint(fmt.Sprintf("Indice #%d", n + 1), hint_cnt, 1); err != nil { errs = append(errs, fmt.Sprintf("%q: unable to add hint %q: %s", path.Base(exercice.Path), hfile, err)) continue } diff --git a/admin/sync/file.go b/admin/sync/file.go index 46776b92..97cfa1a7 100644 --- a/admin/sync/file.go +++ b/admin/sync/file.go @@ -20,6 +20,7 @@ type Importer interface { Kind() string exists(filename string) bool toURL(filename string) string + importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) getFile(filename string, writer *bufio.Writer) error listDir(filename string) ([]string, error) } @@ -83,10 +84,6 @@ func ImportFile(i Importer, URI string, next func(string, string) (interface{}, if r, err := next(dest, URI); err == nil { return r, err } - - if err := os.Remove(dest); err != nil { - return nil, err - } } // Ensure no more file is registered with this path diff --git a/admin/sync/importer_cloud.go b/admin/sync/importer_cloud.go index 027147ce..b5381c0d 100644 --- a/admin/sync/importer_cloud.go +++ b/admin/sync/importer_cloud.go @@ -10,10 +10,6 @@ import ( "github.com/studio-b12/gowebdav" ) -var CloudDAVBase string = "https://srs.epita.fr/owncloud/remote.php/webdav/FIC 2018" -var CloudUsername string = "fic" -var CloudPassword string = "" - type CloudImporter struct { baseDAV url.URL username string @@ -55,6 +51,10 @@ func (i CloudImporter) toURL(filename string) string { return fullURL.String() } +func (i CloudImporter) importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) { + return ImportFile(i, URI, next) +} + func (i CloudImporter) getFile(filename string, writer *bufio.Writer) error { fullURL := i.baseDAV fullURL.Path = path.Join(fullURL.Path, filename) diff --git a/admin/sync/importer_localfs.go b/admin/sync/importer_localfs.go index a281a8ee..4317ec72 100644 --- a/admin/sync/importer_localfs.go +++ b/admin/sync/importer_localfs.go @@ -8,15 +8,20 @@ import ( ) type LocalImporter struct { - Base string + Base string + Symlink bool } func (i LocalImporter) Kind() string { - return "local file importer: " + i.Base + if i.Symlink { + return "local file importer (through symlink): " + i.Base + } else { + return "local file importer: " + i.Base + } } func (i LocalImporter) exists(filename string) bool { - _, err := os.Stat(path.Join(i.Base, filename)) + _, err := os.Stat(i.toURL(filename)) return !os.IsNotExist(err) } @@ -24,6 +29,24 @@ func (i LocalImporter) toURL(filename string) string { return path.Join(i.Base, filename) } +func (i LocalImporter) importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) { + if i.Symlink { + dest := getDestinationFilePath(URI) + + if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { + return nil, err + } + + if i.exists(URI) { + os.Symlink(i.toURL(URI), dest) + } else { + os.Symlink(i.toURL(URI) + "_MERGED", dest) + } + } + + return ImportFile(i, URI, next) +} + func (i LocalImporter) getFile(filename string, writer *bufio.Writer) error { if fd, err := os.Open(path.Join(i.Base, filename)); err != nil { return err