diff --git a/.drone-manifest-fic-get-remote-files.yml b/.drone-manifest-fic-get-remote-files.yml new file mode 100644 index 00000000..0f64b8c1 --- /dev/null +++ b/.drone-manifest-fic-get-remote-files.yml @@ -0,0 +1,22 @@ +image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/.drone.yml b/.drone.yml index 0294e4bc..6faf2d5c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,6 @@ steps: commands: - cd qa/ui - npm install --network-timeout=100000 - - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte - npm run build - tar chjf ../../deploy/htdocs-qa.tar.bz2 build @@ -98,7 +97,6 @@ steps: commands: - cd frontend/fic - npm install --network-timeout=100000 - - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte - npm run build - tar chjf ../../deploy/htdocs-frontend-fic.tar.bz2 build when: @@ -327,6 +325,21 @@ steps: branch: - master + - name: docker fic-get-remote-files + image: plugins/docker + settings: + username: + from_secret: docker_username + password: + from_secret: docker_password + repo: nemunaire/fic-get-remote-files + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile-get-remote-files + when: + branch: + - master + - name: docker fickit-deploy image: plugins/docker settings: @@ -430,7 +443,6 @@ steps: commands: - cd frontend/fic - npm install --network-timeout=100000 - - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte - npm run build when: branch: @@ -479,7 +491,6 @@ steps: commands: - cd qa/ui - npm install --network-timeout=100000 - - sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte - npm run build - tar chjf ../../deploy/htdocs-qa.tar.bz2 build when: @@ -742,6 +753,21 @@ steps: password: from_secret: docker_password + - name: docker fic-get-remote-files + image: plugins/docker + settings: + username: + from_secret: docker_username + password: + from_secret: docker_password + repo: nemunaire/fic-get-remote-files + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile-get-remote-files + when: + branch: + - master + - name: publish fickit-deploy image: plugins/manifest settings: diff --git a/.gitignore b/.gitignore index e80fe652..4327ccd0 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ result started # Standalone binaries +admin/get-remote-files/get-remote-files fic-admin fic-backend fic-dashboard diff --git a/.gitlab-ci/image.yml b/.gitlab-ci/image.yml index 2e68093c..5ad48de5 100644 --- a/.gitlab-ci/image.yml +++ b/.gitlab-ci/image.yml @@ -50,6 +50,12 @@ fickit-deploy: variables: DOCKERFILE: Dockerfile-deploy +get-remote-files: + extends: + - .push + variables: + DOCKERFILE: Dockerfile-get-remote-files + evdist: extends: - .push diff --git a/Dockerfile-get-remote-files b/Dockerfile-get-remote-files new file mode 100644 index 00000000..c383bcb9 --- /dev/null +++ b/Dockerfile-get-remote-files @@ -0,0 +1,27 @@ +FROM golang:1-alpine as gobuild + +RUN apk add --no-cache git + +WORKDIR /go/src/srs.epita.fr/fic-server/ + +RUN apk add --no-cache build-base + +COPY go.mod go.sum ./ +COPY settings settings/ +COPY libfic ./libfic/ +COPY admin ./admin/ + +RUN go get -d -v ./admin && \ + go build -v -o get-remote-files ./admin/get-remote-files + + +FROM alpine:3.18 + +RUN apk add --no-cache \ + ca-certificates + +WORKDIR /srv + +ENTRYPOINT ["/srv/get-remote-files", "/mnt/fic/"] + +COPY --from=gobuild /go/src/srs.epita.fr/fic-server/get-remote-files /srv/get-remote-files diff --git a/admin/get-remote-files/main.go b/admin/get-remote-files/main.go new file mode 100644 index 00000000..1260f40c --- /dev/null +++ b/admin/get-remote-files/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "flag" + "log" + "os" + "path" + "path/filepath" + + "srs.epita.fr/fic-server/admin/sync" + "srs.epita.fr/fic-server/libfic" +) + +func main() { + cloudDAVBase := "" + cloudUsername := "fic" + cloudPassword := "" + localImporterDirectory := "" + + // Read paremeters from environment + if v, exists := os.LookupEnv("FICCLOUD_URL"); exists { + cloudDAVBase = v + } + if v, exists := os.LookupEnv("FICCLOUD_USER"); exists { + cloudUsername = v + } + if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists { + cloudPassword = v + } + + // Read parameters from command line + flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory, + "Base directory where to find challenges files to import, local part") + flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase, + "Base directory where to find challenges files to import, cloud part") + flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync") + flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync") + flag.Var(&sync.RemoteFileDomainWhitelist, "remote-file-domain-whitelist", "List of domains which are allowed to store remote files") + flag.Parse() + + // Do not display timestamp + log.SetFlags(0) + + // Instantiate importer + regenImporter := false + if localImporterDirectory != "" { + sync.GlobalImporter = sync.LocalImporter{Base: localImporterDirectory, Symlink: true} + } else if cloudDAVBase != "" { + sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword) + } else { + // In this case, we want to treat the entier path given + regenImporter = true + } + + for _, p := range flag.Args() { + if regenImporter { + var err error + p, err = filepath.Abs(p) + if err != nil { + p = path.Clean(p) + } + sync.GlobalImporter = sync.LocalImporter{ + Base: p, + Symlink: true, + } + } + + // Find all challenge.toml or challenge.txt + treatDir("") + } +} + +func treatDir(p string) { + var expath string + + for _, f := range []string{"challenge.txt", "challenge.toml"} { + if sync.GlobalImporter.Exists(path.Join(p, f)) { + expath = p + break + } + } + + if expath != "" { + treatExercice(expath) + } else { + files, err := sync.GlobalImporter.ListDir(p) + if err != nil { + log.Printf("Unable to readdir at %s: %s", p, err.Error()) + return + } + + for _, f := range files { + st, err := sync.GlobalImporter.Stat(path.Join(p, f)) + if err == nil && st.IsDir() { + treatDir(path.Join(p, f)) + } + } + } +} + +func treatExercice(expath string) { + // Load exercice + exercice, _, _, _, _, err := sync.BuildExercice(sync.GlobalImporter, &fic.Theme{}, expath, nil, nil) + if exercice == nil { + log.Printf("Unable to treat exercice %q: %s", expath, err.Error()) + return + } + + paramsFiles, err := sync.GetExerciceFilesParams(sync.GlobalImporter, exercice) + if err != nil { + log.Printf("Unable to read challenge.txt %q: %s", expath, err.Error()) + return + } + + for fname, pf := range paramsFiles { + if pf.URL == "" { + continue + } + + dest := path.Join(exercice.Path, "files", fname) + + log.Printf("Downloading %s...", fname) + if li, ok := sync.GlobalImporter.(sync.LocalImporter); ok { + err = sync.DownloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false) + } else { + err = sync.DownloadExerciceFile(paramsFiles[fname], dest, exercice, false) + } + + if err != nil { + log.Println("DownloadExerciceFile error:", err.Error()) + } + } +}