From 64eb8f28526b7b7a09de4bc7624e09699a7da679 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 15:00:51 +0100 Subject: [PATCH 1/9] ui: Add a button to refresh grades --- ui/src/routes/works/[wid]/+page.svelte | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ui/src/routes/works/[wid]/+page.svelte b/ui/src/routes/works/[wid]/+page.svelte index af8dbbc..cf07c02 100644 --- a/ui/src/routes/works/[wid]/+page.svelte +++ b/ui/src/routes/works/[wid]/+page.svelte @@ -60,7 +60,15 @@ {/if}
-

Notes

+
+

Notes

+ +
{#await gradesP}
From ce0c8e00256adbab53575bd4e917f17bf564118d Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 17:05:01 +0100 Subject: [PATCH 2/9] Update SharingTime to 20 min as 15 is not enough --- repositories.go | 2 +- submissions.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/repositories.go b/repositories.go index 855397e..d54063d 100644 --- a/repositories.go +++ b/repositories.go @@ -576,7 +576,7 @@ func TriggerTests(c *gin.Context, work *Work, repo *Repository, u *User) { Key: aws.String(filepath.Join(fmt.Sprintf("%d", work.Id), fmt.Sprintf("rendu-%s.tar.xz", login))), }) - url, err := req.Presign(SharingTime) + url, err := req.Presign(SharingTime * 20) if err != nil { log.Println("Unable to create presign URL:", err) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": "Something goes wrong when creating the presigned URL."}) diff --git a/submissions.go b/submissions.go index b5f0da1..8f7aa59 100644 --- a/submissions.go +++ b/submissions.go @@ -17,7 +17,7 @@ import ( "github.com/gin-gonic/gin" ) -const SharingTime = 15 * time.Minute +const SharingTime = 10 * time.Minute var ( s3_endpoint string From 5020f378c8b71b4a2c15534e18c51aed46d7b0b9 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 17:05:17 +0100 Subject: [PATCH 3/9] Only consider the last log line --- repositories.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/repositories.go b/repositories.go index d54063d..4447dad 100644 --- a/repositories.go +++ b/repositories.go @@ -746,14 +746,17 @@ func (r *Repository) fetchRepoTests(build_number int) error { continue } - for _, line := range logs { - if strings.HasPrefix(line.Message, "grade:") { - g, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line.Message, "grade:")), 64) - if err == nil { - grade += g - } else { - log.Println("Unable to parse grade:", err.Error()) - } + if len(logs) < 1 { + continue + } + + line := logs[len(logs)-1] + if strings.HasPrefix(line.Message, "grade:") { + g, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line.Message, "grade:")), 64) + if err == nil { + grade += g + } else { + log.Println("Unable to parse grade:", err.Error()) } } } From 906501cc7bf38e81b439d0670da317f4eb3b9440 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 17:43:03 +0100 Subject: [PATCH 4/9] ui: Use ScoreBadge component --- ui/src/lib/components/ScoreBadge.svelte | 14 ++++++++++++++ ui/src/lib/components/SurveyList.svelte | 12 ++---------- ui/src/routes/works/+page.svelte | 12 ++---------- ui/src/routes/works/[wid]/+page.svelte | 3 ++- 4 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 ui/src/lib/components/ScoreBadge.svelte diff --git a/ui/src/lib/components/ScoreBadge.svelte b/ui/src/lib/components/ScoreBadge.svelte new file mode 100644 index 0000000..f170afe --- /dev/null +++ b/ui/src/lib/components/ScoreBadge.svelte @@ -0,0 +1,14 @@ + + += 18} + class:bg-info={score < 18 && score >= 15} + class:bg-warning={score < 15 && score >= 9} + class:bg-danger={score < 9} + class:bg-dark={score == "N/A"} +> + {score} + diff --git a/ui/src/lib/components/SurveyList.svelte b/ui/src/lib/components/SurveyList.svelte index 6b97e43..5ca0c68 100644 --- a/ui/src/lib/components/SurveyList.svelte +++ b/ui/src/lib/components/SurveyList.svelte @@ -4,6 +4,7 @@ import { user } from '$lib/stores/user'; import DateFormat from '$lib/components/DateFormat.svelte'; import SurveyBadge from '$lib/components/SurveyBadge.svelte'; + import ScoreBadge from '$lib/components/ScoreBadge.svelte'; import SubmissionStatus from '$lib/components/SubmissionStatus.svelte'; import { getCategories } from '$lib/categories'; import { getSurveys } from '$lib/surveys'; @@ -144,16 +145,7 @@ {/if} {:else} - = 18} - class:bg-info={score.score < 18 && score.score >= 15} - class:bg-warning={score.score < 15 && score.score >= 9} - class:bg-danger={score.score < 9} - class:bg-dark={score.score == "N/A"} - > - {score.score} - + {/if} {:catch error} diff --git a/ui/src/routes/works/+page.svelte b/ui/src/routes/works/+page.svelte index 722a343..d248212 100644 --- a/ui/src/routes/works/+page.svelte +++ b/ui/src/routes/works/+page.svelte @@ -3,6 +3,7 @@ import { user } from '$lib/stores/user'; import DateFormat from '$lib/components/DateFormat.svelte'; + import ScoreBadge from '$lib/components/ScoreBadge.svelte'; import SurveyBadge from '$lib/components/SurveyBadge.svelte'; import SubmissionStatus from '$lib/components/SubmissionStatus.svelte'; import { getCategories } from '$lib/categories'; @@ -120,16 +121,7 @@ {#await getScore(work)} {:then score} - = 18} - class:bg-info={score.score < 18 && score.score >= 15} - class:bg-warning={score.score < 15 && score.score >= 9} - class:bg-danger={score.score < 9} - class:bg-dark={score.score == "N/A"} - > - {score.score} - + {:catch error} {/await} diff --git a/ui/src/routes/works/[wid]/+page.svelte b/ui/src/routes/works/[wid]/+page.svelte index cf07c02..d79e6bb 100644 --- a/ui/src/routes/works/[wid]/+page.svelte +++ b/ui/src/routes/works/[wid]/+page.svelte @@ -3,6 +3,7 @@ import { user } from '$lib/stores/user'; import DateFormat from '$lib/components/DateFormat.svelte'; + import ScoreBadge from '$lib/components/ScoreBadge.svelte'; import SubmissionStatus from '$lib/components/SubmissionStatus.svelte'; import SurveyBadge from '$lib/components/SurveyBadge.svelte'; import WorkAdmin from '$lib/components/WorkAdmin.svelte'; @@ -99,7 +100,7 @@ {grade.login} - {grade.score} + {#if grade.comment}{grade.comment}{:else}-{/if} {grade.date} From a78de73671d8131901681ceeece4c452d6b2703a Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 17:43:14 +0100 Subject: [PATCH 5/9] ui: Display a spinner during gradation launch --- ui/src/routes/works/[wid]/rendus/+page.svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ui/src/routes/works/[wid]/rendus/+page.svelte b/ui/src/routes/works/[wid]/rendus/+page.svelte index 23c7597..818735d 100644 --- a/ui/src/routes/works/[wid]/rendus/+page.svelte +++ b/ui/src/routes/works/[wid]/rendus/+page.svelte @@ -32,16 +32,21 @@ repositoriesP[userid] = getRepositories(wid, userid); } + let gradation_preflight = false; async function runGradations() { + gradation_preflight = true; for (const user of await usersP) { if (repositoriesP[user.id]) { try { for (const repo of await repositoriesP[user.id]) { repo.runGradation(); } - } catch {} + } catch (err) { + ToastsStore.addToast({color: "danger", title: "Impossible de lancer la notation de " + user.login, msg: err}) + } } } + gradation_preflight = false; } let nb_rendus = 0; @@ -66,8 +71,13 @@ class="btn btn-sm btn-success mr-1" title="Relancer les tests" on:click={runGradations} + disabled={gradation_preflight} > - + {#if gradation_preflight} + + {:else} + + {/if}
From 8b8f3947f859b20b08aa0caed4dfd43be4b8fe12 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 17:54:51 +0100 Subject: [PATCH 6/9] Refactor gradation + add way to give point for a succeeded task --- gradation.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ repositories.go | 63 +---------------------------------------- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/gradation.go b/gradation.go index 8e762e9..4acf203 100644 --- a/gradation.go +++ b/gradation.go @@ -1,8 +1,11 @@ package main import ( + "fmt" "log" "net/http" + "strconv" + "strings" "github.com/drone/drone-go/drone" "github.com/gin-gonic/gin" @@ -30,3 +33,74 @@ func declareAPIAdminGradationRoutes(router *gin.RouterGroup) { c.JSON(http.StatusOK, result) }) } + +type TestsWebhook struct { + Login string `json:"login"` + RepositoryId int `json:"repository_id"` + BuildNumber int `json:"build_number"` + Steps map[string]float64 `json:"steps,omitempty"` +} + +func (r *Repository) fetchRepoTests(build_number int, steps map[string]float64) error { + tmp := strings.Split(r.TestsRef, "/") + if len(tmp) < 3 { + return fmt.Errorf("This repository tests reference is not filled properly.") + } + + work, err := getWork(r.IdWork) + if err != nil { + return fmt.Errorf("Unable to retrieve the related work: %w", err) + } + + client := drone.NewClient(droneEndpoint, droneConfig) + result, err := client.Build(tmp[0], tmp[1], build_number) + if err != nil { + return fmt.Errorf("Unable to find the referenced build (%d): %w", build_number, err) + } + + if result.Finished > 0 { + return fmt.Errorf("The test phase is not finished") + } + + var grade float64 + for _, stage := range result.Stages { + for _, step := range stage.Steps { + if g, ok := steps[fmt.Sprintf("%d", step.Number)]; ok { + log.Printf("Step %q (%d) in status %q", step.Name, step.Number, step.Status) + // Give the point if it succeed + if step.Status == "success" { + grade += g + } + continue + } + + logs, err := client.Logs(tmp[0], tmp[1], build_number, stage.Number, step.Number) + if err != nil { + log.Printf("Unable to retrieve build logs %s/%s/%d/%d/%d: %s", tmp[0], tmp[1], build_number, stage.Number, step.Number, err.Error()) + continue + } + + if len(logs) < 1 { + continue + } + + line := logs[len(logs)-1] + if strings.HasPrefix(line.Message, "grade:") { + g, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line.Message, "grade:")), 64) + if err == nil { + grade += g + } else { + log.Println("Unable to parse grade:", err.Error()) + } + } + } + } + + work.AddGrade(WorkGrade{ + IdUser: r.IdUser, + IdWork: work.Id, + Grade: grade, + }) + + return nil +} diff --git a/repositories.go b/repositories.go index 4447dad..49e8d48 100644 --- a/repositories.go +++ b/repositories.go @@ -334,12 +334,6 @@ type GitLabWebhook struct { Repository GitLabRepository } -type TestsWebhook struct { - Login string `json:"login"` - RepositoryId int `json:"repository_id"` - BuildNumber int `json:"build_number"` -} - func declareCallbacksRoutes(router *gin.RouterGroup) { router.POST("/callbacks/trigger.json", func(c *gin.Context) { // Check event type @@ -446,7 +440,7 @@ func declareCallbacksRoutes(router *gin.RouterGroup) { return } - err = repo.fetchRepoTests(hook.BuildNumber) + err = repo.fetchRepoTests(hook.BuildNumber, hook.Steps) if err != nil { log.Printf("Unable to fetchRepoTests(%d): %s", hook.RepositoryId, err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": "Unable to fetch tests results."}) @@ -716,61 +710,6 @@ func (r Repository) Delete() (int64, error) { } } -func (r *Repository) fetchRepoTests(build_number int) error { - tmp := strings.Split(r.TestsRef, "/") - if len(tmp) < 3 { - return fmt.Errorf("This repository tests reference is not filled properly.") - } - - work, err := getWork(r.IdWork) - if err != nil { - return fmt.Errorf("Unable to retrieve the related work: %w", err) - } - - client := drone.NewClient(droneEndpoint, droneConfig) - result, err := client.Build(tmp[0], tmp[1], build_number) - if err != nil { - return fmt.Errorf("Unable to find the referenced build (%d): %w", build_number, err) - } - - if result.Finished > 0 { - return fmt.Errorf("The test phase is not finished") - } - - var grade float64 - for _, stage := range result.Stages { - for _, step := range stage.Steps { - logs, err := client.Logs(tmp[0], tmp[1], build_number, stage.Number, step.Number) - if err != nil { - log.Printf("Unable to retrieve build logs %s/%s/%d/%d/%d: %s", tmp[0], tmp[1], build_number, stage.Number, step.Number, err.Error()) - continue - } - - if len(logs) < 1 { - continue - } - - line := logs[len(logs)-1] - if strings.HasPrefix(line.Message, "grade:") { - g, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line.Message, "grade:")), 64) - if err == nil { - grade += g - } else { - log.Println("Unable to parse grade:", err.Error()) - } - } - } - } - - work.AddGrade(WorkGrade{ - IdUser: r.IdUser, - IdWork: work.Id, - Grade: grade, - }) - - return nil -} - func ClearRepositories() (int64, error) { if res, err := DBExec("DELETE FROM user_work_repositories"); err != nil { return 0, err From bcf76a2c86d15ca4885350823c1de5eeb89af63e Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 5 Mar 2023 18:16:38 +0100 Subject: [PATCH 7/9] New route to stop all running/pending tests for a given work --- repositories.go | 57 +++++++++++++++++++ ui/src/lib/works.js | 14 +++++ ui/src/routes/works/[wid]/rendus/+page.svelte | 7 +++ works.go | 12 ++++ 4 files changed, 90 insertions(+) diff --git a/repositories.go b/repositories.go index 49e8d48..221991c 100644 --- a/repositories.go +++ b/repositories.go @@ -603,6 +603,42 @@ func TriggerTests(c *gin.Context, work *Work, repo *Repository, u *User) { c.JSON(http.StatusOK, repo) } +func (w *Work) stopTests() error { + repos, err := w.GetRepositories() + if err != nil { + return err + } + + client := drone.NewClient(droneEndpoint, droneConfig) + for _, repo := range repos { + slug := strings.Split(repo.TestsRef, "/") + if len(slug) < 3 { + continue + } + + buildn, err := strconv.ParseInt(slug[2], 10, 32) + if err != nil { + continue + } + + build, err := client.Build(slug[0], slug[1], int(buildn)) + if err != nil { + log.Println("Unable to communicate with Drone:", err.Error()) + continue + } + + if build.Status == "pending" { + err := client.BuildCancel(slug[0], slug[1], int(buildn)) + if err != nil { + log.Println("Unable to cancel the build:", err.Error()) + continue + } + } + } + + return nil +} + type Repository struct { Id int64 `json:"id"` IdUser int64 `json:"id_user"` @@ -637,6 +673,27 @@ func (u *User) GetRepositories() (repositories []*Repository, err error) { } } +func (w *Work) GetRepositories() (repositories []*Repository, err error) { + if rows, errr := DBQuery("SELECT id_repository, id_user, id_work, uri, secret, last_check, droneref, last_tests, testsref FROM user_work_repositories WHERE id_work=?", w.Id); errr != nil { + return nil, errr + } else { + defer rows.Close() + + for rows.Next() { + var repo Repository + if err = rows.Scan(&repo.Id, &repo.IdUser, &repo.IdWork, &repo.URI, &repo.Secret, &repo.LastCheck, &repo.DroneRef, &repo.LastTests, &repo.TestsRef); err != nil { + return + } + repositories = append(repositories, &repo) + } + if err = rows.Err(); err != nil { + return + } + + return + } +} + func getRepositoriesByURI(uri string) (repositories []*Repository, err error) { if rows, errr := DBQuery("SELECT id_repository, id_user, id_work, uri, secret, last_check, droneref, last_tests, testsref FROM user_work_repositories WHERE uri=?", uri); errr != nil { return nil, errr diff --git a/ui/src/lib/works.js b/ui/src/lib/works.js index 9d83b98..1d5976c 100644 --- a/ui/src/lib/works.js +++ b/ui/src/lib/works.js @@ -95,6 +95,20 @@ export class Work { } } + async stopTests() { + if (this.id) { + const res = await fetch(`api/works/${this.id}/tests`, { + method: 'DELETE', + headers: {'Accept': 'application/json'}, + }); + if (res.status == 200) { + return true; + } else { + throw new Error((await res.json()).errmsg); + } + } + } + async getSubmission(uid) { const res = await fetch(uid?`api/users/${uid}/works/${this.id}/submission`:`api/works/${this.id}/submission`, { headers: {'Accept': 'application/json'} diff --git a/ui/src/routes/works/[wid]/rendus/+page.svelte b/ui/src/routes/works/[wid]/rendus/+page.svelte index 818735d..a7d0574 100644 --- a/ui/src/routes/works/[wid]/rendus/+page.svelte +++ b/ui/src/routes/works/[wid]/rendus/+page.svelte @@ -79,6 +79,13 @@ {/if} + diff --git a/works.go b/works.go index e3c4940..1ce2e81 100644 --- a/works.go +++ b/works.go @@ -153,6 +153,18 @@ func declareAPIAdminWorksRoutes(router *gin.RouterGroup) { c.JSON(http.StatusOK, nil) }) + worksRoutes.DELETE("/tests", func(c *gin.Context) { + w := c.MustGet("work").(*Work) + + err := w.stopTests() + if err != nil { + log.Println("Unable to stop tests:", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"errmsg": "An error occurs during test stop."}) + return + } + + c.JSON(http.StatusOK, true) + }) // Grades related to works worksRoutes.GET("/grades", func(c *gin.Context) { From 9a145965fb0cea9c541abf57cccc00dccf2c8b00 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 6 Mar 2023 01:45:50 +0100 Subject: [PATCH 8/9] ui: Change Id to ensure they are all differents --- ui/src/lib/components/WorkRepository.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/lib/components/WorkRepository.svelte b/ui/src/lib/components/WorkRepository.svelte index 950d827..79b5069 100644 --- a/ui/src/lib/components/WorkRepository.svelte +++ b/ui/src/lib/components/WorkRepository.svelte @@ -160,7 +160,7 @@ {:then rrepos} From 1515140c099b57413fc9e7ea8581a7f1d098ae06 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 6 Mar 2023 01:46:38 +0100 Subject: [PATCH 9/9] Add a default value to testsref --- repositories.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repositories.go b/repositories.go index 221991c..d06d8fa 100644 --- a/repositories.go +++ b/repositories.go @@ -740,7 +740,7 @@ func (u *User) NewRepository(w *Work, uri string) (*Repository, error) { return nil, err } - if res, err := DBExec("INSERT INTO user_work_repositories (id_user, id_work, uri, secret, droneref) VALUES (?, ?, ?, ?, ?)", u.Id, w.Id, uri, secret, ""); err != nil { + if res, err := DBExec("INSERT INTO user_work_repositories (id_user, id_work, uri, secret, droneref, testsref) VALUES (?, ?, ?, ?, ?, '')", u.Id, w.Id, uri, secret, ""); err != nil { return nil, err } else if rid, err := res.LastInsertId(); err != nil { return nil, err