Introducing get-remote-files to download remote files

This commit is contained in:
nemunaire 2023-11-25 19:05:29 +01:00
parent c3353572e6
commit c9e3a9ac3c
6 changed files with 219 additions and 4 deletions

View File

@ -0,0 +1,22 @@
image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
- image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: nemunaire/fic-get-remote-files:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

View File

@ -24,7 +24,6 @@ steps:
commands: commands:
- cd qa/ui - cd qa/ui
- npm install --network-timeout=100000 - npm install --network-timeout=100000
- sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
- npm run build - npm run build
- tar chjf ../../deploy/htdocs-qa.tar.bz2 build - tar chjf ../../deploy/htdocs-qa.tar.bz2 build
@ -98,7 +97,6 @@ steps:
commands: commands:
- cd frontend/fic - cd frontend/fic
- npm install --network-timeout=100000 - npm install --network-timeout=100000
- sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
- npm run build - npm run build
- tar chjf ../../deploy/htdocs-frontend-fic.tar.bz2 build - tar chjf ../../deploy/htdocs-frontend-fic.tar.bz2 build
when: when:
@ -327,6 +325,21 @@ steps:
branch: branch:
- master - master
- name: docker fic-get-remote-files
image: plugins/docker
settings:
username:
from_secret: docker_username
password:
from_secret: docker_password
repo: nemunaire/fic-get-remote-files
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile-get-remote-files
when:
branch:
- master
- name: docker fickit-deploy - name: docker fickit-deploy
image: plugins/docker image: plugins/docker
settings: settings:
@ -430,7 +443,6 @@ steps:
commands: commands:
- cd frontend/fic - cd frontend/fic
- npm install --network-timeout=100000 - npm install --network-timeout=100000
- sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
- npm run build - npm run build
when: when:
branch: branch:
@ -479,7 +491,6 @@ steps:
commands: commands:
- cd qa/ui - cd qa/ui
- npm install --network-timeout=100000 - npm install --network-timeout=100000
- sed -i 's!@popperjs/core/dist/esm/popper!@popperjs/core!' node_modules/sveltestrap/src/*.js node_modules/sveltestrap/src/*.svelte
- npm run build - npm run build
- tar chjf ../../deploy/htdocs-qa.tar.bz2 build - tar chjf ../../deploy/htdocs-qa.tar.bz2 build
when: when:
@ -742,6 +753,21 @@ steps:
password: password:
from_secret: docker_password from_secret: docker_password
- name: docker fic-get-remote-files
image: plugins/docker
settings:
username:
from_secret: docker_username
password:
from_secret: docker_password
repo: nemunaire/fic-get-remote-files
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile-get-remote-files
when:
branch:
- master
- name: publish fickit-deploy - name: publish fickit-deploy
image: plugins/manifest image: plugins/manifest
settings: settings:

1
.gitignore vendored
View File

@ -34,6 +34,7 @@ result
started started
# Standalone binaries # Standalone binaries
admin/get-remote-files/get-remote-files
fic-admin fic-admin
fic-backend fic-backend
fic-dashboard fic-dashboard

View File

@ -50,6 +50,12 @@ fickit-deploy:
variables: variables:
DOCKERFILE: Dockerfile-deploy DOCKERFILE: Dockerfile-deploy
get-remote-files:
extends:
- .push
variables:
DOCKERFILE: Dockerfile-get-remote-files
evdist: evdist:
extends: extends:
- .push - .push

View File

@ -0,0 +1,27 @@
FROM golang:1-alpine as gobuild
RUN apk add --no-cache git
WORKDIR /go/src/srs.epita.fr/fic-server/
RUN apk add --no-cache build-base
COPY go.mod go.sum ./
COPY settings settings/
COPY libfic ./libfic/
COPY admin ./admin/
RUN go get -d -v ./admin && \
go build -v -o get-remote-files ./admin/get-remote-files
FROM alpine:3.18
RUN apk add --no-cache \
ca-certificates
WORKDIR /srv
ENTRYPOINT ["/srv/get-remote-files", "/mnt/fic/"]
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/get-remote-files /srv/get-remote-files

View File

@ -0,0 +1,133 @@
package main
import (
"flag"
"log"
"os"
"path"
"path/filepath"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
)
func main() {
cloudDAVBase := ""
cloudUsername := "fic"
cloudPassword := ""
localImporterDirectory := ""
// Read paremeters from environment
if v, exists := os.LookupEnv("FICCLOUD_URL"); exists {
cloudDAVBase = v
}
if v, exists := os.LookupEnv("FICCLOUD_USER"); exists {
cloudUsername = v
}
if v, exists := os.LookupEnv("FICCLOUD_PASS"); exists {
cloudPassword = v
}
// Read parameters from command line
flag.StringVar(&localImporterDirectory, "localimport", localImporterDirectory,
"Base directory where to find challenges files to import, local part")
flag.StringVar(&cloudDAVBase, "clouddav", cloudDAVBase,
"Base directory where to find challenges files to import, cloud part")
flag.StringVar(&cloudUsername, "clouduser", cloudUsername, "Username used to sync")
flag.StringVar(&cloudPassword, "cloudpass", cloudPassword, "Password used to sync")
flag.Var(&sync.RemoteFileDomainWhitelist, "remote-file-domain-whitelist", "List of domains which are allowed to store remote files")
flag.Parse()
// Do not display timestamp
log.SetFlags(0)
// Instantiate importer
regenImporter := false
if localImporterDirectory != "" {
sync.GlobalImporter = sync.LocalImporter{Base: localImporterDirectory, Symlink: true}
} else if cloudDAVBase != "" {
sync.GlobalImporter, _ = sync.NewCloudImporter(cloudDAVBase, cloudUsername, cloudPassword)
} else {
// In this case, we want to treat the entier path given
regenImporter = true
}
for _, p := range flag.Args() {
if regenImporter {
var err error
p, err = filepath.Abs(p)
if err != nil {
p = path.Clean(p)
}
sync.GlobalImporter = sync.LocalImporter{
Base: p,
Symlink: true,
}
}
// Find all challenge.toml or challenge.txt
treatDir("")
}
}
func treatDir(p string) {
var expath string
for _, f := range []string{"challenge.txt", "challenge.toml"} {
if sync.GlobalImporter.Exists(path.Join(p, f)) {
expath = p
break
}
}
if expath != "" {
treatExercice(expath)
} else {
files, err := sync.GlobalImporter.ListDir(p)
if err != nil {
log.Printf("Unable to readdir at %s: %s", p, err.Error())
return
}
for _, f := range files {
st, err := sync.GlobalImporter.Stat(path.Join(p, f))
if err == nil && st.IsDir() {
treatDir(path.Join(p, f))
}
}
}
}
func treatExercice(expath string) {
// Load exercice
exercice, _, _, _, _, err := sync.BuildExercice(sync.GlobalImporter, &fic.Theme{}, expath, nil, nil)
if exercice == nil {
log.Printf("Unable to treat exercice %q: %s", expath, err.Error())
return
}
paramsFiles, err := sync.GetExerciceFilesParams(sync.GlobalImporter, exercice)
if err != nil {
log.Printf("Unable to read challenge.txt %q: %s", expath, err.Error())
return
}
for fname, pf := range paramsFiles {
if pf.URL == "" {
continue
}
dest := path.Join(exercice.Path, "files", fname)
log.Printf("Downloading %s...", fname)
if li, ok := sync.GlobalImporter.(sync.LocalImporter); ok {
err = sync.DownloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false)
} else {
err = sync.DownloadExerciceFile(paramsFiles[fname], dest, exercice, false)
}
if err != nil {
log.Println("DownloadExerciceFile error:", err.Error())
}
}
}