Introducing get-remote-files to download remote files

.drone-manifest-fic-get-remote-files.yml

@@ -0,0 +1,22 @@
+image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
+  - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      architecture: arm
+      os: linux
+      variant: v7

.drone.yml

@@ -24,7 +24,6 @@ steps:
     commands:
       - cd qa/ui
       - npm install --network-timeout=100000
-      - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
       - npm run build
       - tar chjf ../../deploy/htdocs-qa.tar.bz2 build
 
@@ -98,7 +97,6 @@ steps:
     commands:
       - cd frontend/fic
       - npm install --network-timeout=100000
-      - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
       - npm run build
       - tar chjf ../../deploy/htdocs-frontend-fic.tar.bz2 build
     when:
@@ -327,6 +325,21 @@ steps:
       branch:
         - master
 
+  - name: docker fic-get-remote-files
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      repo: nemunaire/fic-get-remote-files
+      auto_tag: true
+      auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
+      dockerfile: Dockerfile-get-remote-files
+    when:
+      branch:
+        - master
+
   - name: docker fickit-deploy
     image: plugins/docker
     settings:
@@ -430,7 +443,6 @@ steps:
     commands:
       - cd frontend/fic
       - npm install --network-timeout=100000
-      - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
       - npm run build
     when:
       branch:
@@ -479,7 +491,6 @@ steps:
     commands:
       - cd qa/ui
       - npm install --network-timeout=100000
-      - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
       - npm run build
       - tar chjf ../../deploy/htdocs-qa.tar.bz2 build
     when:
@@ -742,6 +753,21 @@ steps:
       password:
         from_secret: docker_password
 
+  - name: docker fic-get-remote-files
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      repo: nemunaire/fic-get-remote-files
+      auto_tag: true
+      auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
+      dockerfile: Dockerfile-get-remote-files
+    when:
+      branch:
+        - master
+
   - name: publish fickit-deploy
     image: plugins/manifest
     settings:

.gitignore

@@ -34,6 +34,7 @@ result
 started
 
 # Standalone binaries
+admin/get-remote-files/get-remote-files
 fic-admin
 fic-backend
 fic-dashboard

.gitlab-ci/image.yml

@@ -50,6 +50,12 @@ fickit-deploy:
   variables:
     DOCKERFILE: Dockerfile-deploy
 
+get-remote-files:
+  extends:
+    - .push
+  variables:
+    DOCKERFILE: Dockerfile-get-remote-files
+
 evdist:
   extends:
     - .push

Dockerfile-get-remote-files

@@ -0,0 +1,27 @@
+FROM golang:1-alpine as gobuild
+
+RUN apk add --no-cache git
+
+WORKDIR /go/src/srs.epita.fr/fic-server/
+
+RUN apk add --no-cache build-base
+
+COPY go.mod go.sum ./
+COPY settings settings/
+COPY libfic ./libfic/
+COPY admin ./admin/
+
+RUN go get -d -v ./admin && \
+    go build -v -o get-remote-files ./admin/get-remote-files
+
+
+FROM alpine:3.18
+
+RUN apk add --no-cache \
+        ca-certificates
+
+WORKDIR /srv
+
+ENTRYPOINT ["/srv/get-remote-files", "/mnt/fic/"]
+
+COPY --from=gobuild /go/src/srs.epita.fr/fic-server/get-remote-files /srv/get-remote-files

admin/get-remote-files/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+
+	"srs.epita.fr/fic-server/admin/sync"
+	"srs.epita.fr/fic-server/libfic"
+)
+
+func main() {
+	cloudDAVBase := ""
+	cloudUsername := "fic"
+	cloudPassword := ""
+	localImporterDirectory := ""
+
+	// Read paremeters from environment
+	if v, exists := os.LookupEnv("FICCLOUD_URL"); exists {
+		cloudDAVBase = v
+	}
+	if v, exists := os.LookupEnv("FICCLOUD_USER"); exists {
+		cloudUsername = v
+	}
+	if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists {
+		cloudPassword = v
+	}
+
+	// Read parameters from command line
+	flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory,
+		"Base directory where to find challenges files to import, local part")
+	flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase,
+		"Base directory where to find challenges files to import, cloud part")
+	flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync")
+	flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync")
+	flag.Var(&sync.RemoteFileDomainWhitelist, "remote-file-domain-whitelist", "List of domains which are allowed to store remote files")
+	flag.Parse()
+
+	// Do not display timestamp
+	log.SetFlags(0)
+
+	// Instantiate importer
+	regenImporter := false
+	if localImporterDirectory != "" {
+		sync.GlobalImporter = sync.LocalImporter{Base: localImporterDirectory, Symlink: true}
+	} else if cloudDAVBase != "" {
+		sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword)
+	} else {
+		// In this case, we want to treat the entier path given
+		regenImporter = true
+	}
+
+	for _, p := range flag.Args() {
+		if regenImporter {
+			var err error
+			p, err = filepath.Abs(p)
+			if err != nil {
+				p = path.Clean(p)
+			}
+			sync.GlobalImporter = sync.LocalImporter{
+				Base:    p,
+				Symlink: true,
+			}
+		}
+
+		// Find all challenge.toml or challenge.txt
+		treatDir("")
+	}
+}
+
+func treatDir(p string) {
+	var expath string
+
+	for _, f := range []string{"challenge.txt", "challenge.toml"} {
+		if sync.GlobalImporter.Exists(path.Join(p, f)) {
+			expath = p
+			break
+		}
+	}
+
+	if expath != "" {
+		treatExercice(expath)
+	} else {
+		files, err := sync.GlobalImporter.ListDir(p)
+		if err != nil {
+			log.Printf("Unable to readdir at %s: %s", p, err.Error())
+			return
+		}
+
+		for _, f := range files {
+			st, err := sync.GlobalImporter.Stat(path.Join(p, f))
+			if err == nil && st.IsDir() {
+				treatDir(path.Join(p, f))
+			}
+		}
+	}
+}
+
+func treatExercice(expath string) {
+	// Load exercice
+	exercice, _, _, _, _, err := sync.BuildExercice(sync.GlobalImporter, &fic.Theme{}, expath, nil, nil)
+	if exercice == nil {
+		log.Printf("Unable to treat exercice %q: %s", expath, err.Error())
+		return
+	}
+
+	paramsFiles, err := sync.GetExerciceFilesParams(sync.GlobalImporter, exercice)
+	if err != nil {
+		log.Printf("Unable to read challenge.txt %q: %s", expath, err.Error())
+		return
+	}
+
+	for fname, pf := range paramsFiles {
+		if pf.URL == "" {
+			continue
+		}
+
+		dest := path.Join(exercice.Path, "files", fname)
+
+		log.Printf("Downloading %s...", fname)
+		if li, ok := sync.GlobalImporter.(sync.LocalImporter); ok {
+			err = sync.DownloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false)
+		} else {
+			err = sync.DownloadExerciceFile(paramsFiles[fname], dest, exercice, false)
+		}
+
+		if err != nil {
+			log.Println("DownloadExerciceFile error:", err.Error())
+		}
+	}
+}