repochecker/videos: Also check grammar in subtitles

Dockerfile-admin

@@ -14,7 +14,8 @@ RUN apk add --no-cache build-base && \
     go get -d -v ./admin && \
     go build -v -buildvcs=false -o admin/admin ./admin && \
     go build -v -buildmode=plugin -o repochecker/epita-rules.so ./repochecker/epita && \
-    go build -v -buildmode=plugin -o repochecker/grammalecte-rules.so ./repochecker/grammalecte
+    go build -v -buildmode=plugin -o repochecker/grammalecte-rules.so ./repochecker/grammalecte && \
+    go build -v -buildmode=plugin -o repochecker/videos-rules.so ./repochecker/videos
 
 
 FROM alpine:3.16
@@ -35,3 +36,4 @@ ENTRYPOINT ["/srv/admin", "-bind=:8081", "-baseurl=/admin/"]
 COPY --from=gobuild /go/src/srs.epita.fr/fic-server/admin/admin /srv/admin
 COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/epita-rules.so /srv/epita-rules.so
 COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/grammalecte-rules.so /usr/lib/grammalecte-rules.so
+COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/videos-rules.so /usr/lib/videos-rules.so

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/BurntSushi/toml v1.2.1
+	github.com/asticode/go-astisub v0.21.0
 	github.com/gin-gonic/gin v1.8.1
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/go-sql-driver/mysql v1.6.0
@@ -22,6 +23,8 @@ require (
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/asticode/go-astikit v0.20.0 // indirect
+	github.com/asticode/go-astits v1.8.0 // indirect
 	github.com/aws/aws-sdk-go v1.38.20 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect

go.sum

@@ -11,6 +11,12 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=
+github.com/asticode/go-astikit v0.20.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0=
+github.com/asticode/go-astisub v0.21.0 h1:xaCx7SnqblsR7ZqFbo9wq/JYwen7IAG2AVIcfecLxNI=
+github.com/asticode/go-astisub v0.21.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8=
+github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg=
+github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ=
 github.com/aws/aws-sdk-go v1.38.20 h1:QbzNx/tdfATbdKfubBpkt84OM6oBkxQZRw6+bW2GyeA=
 github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -113,6 +119,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -165,6 +172,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
@@ -181,6 +189,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

repochecker/videos/main.go

@@ -1,120 +1,24 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"log"
-	"net/url"
-	"strconv"
-
-	ffmpeg "github.com/u2takey/ffmpeg-go"
 
 	"srs.epita.fr/fic-server/admin/sync"
 	"srs.epita.fr/fic-server/libfic"
 )
 
-type VideoInfo struct {
+var hooks *sync.CheckHooks
-	Streams []struct {
+
-		CodecType     string `json:"codec_type"`
+func RegisterChecksHooks(h *sync.CheckHooks) {
-		CodecName     string `json:"codec_name"`
+	hooks = h
-		CodecLongName string `json:"codec_long_name"`
+
-		Duration      string
+	h.RegisterExerciceHook(CheckResolutionVideo)
-		NbFrames      string `json:"nb_frames"`
-		Width         int
-		Height        int
-	} `json:"streams"`
 }
 
 func CheckResolutionVideo(e *fic.Exercice, exceptions *sync.CheckExceptions) (errs []error) {
-	i, ok := sync.GlobalImporter.(sync.LocalImporter)
+	for _, err := range checkResolutionVideo(e, exceptions) {
-	if !ok {
+		errs = append(errs, fmt.Errorf("resolution.mp4: %w", err))
-		log.Printf("Unable to load `videos-rules.so` as the current Importer is not a LocalImporter (%T).", sync.GlobalImporter)
-		return
-	}
-
-	if len(e.VideoURI) == 0 {
-		return
-	}
-
-	path, err := url.PathUnescape(e.VideoURI[9:])
-	if err != nil {
-		path = e.VideoURI
-	}
-
-	data, err := ffmpeg.Probe(i.GetLocalPath(path))
-	if err != nil {
-		errs = append(errs, fmt.Errorf("unable to open %q: %w", path, err))
-		return
-	}
-
-	vInfo := &VideoInfo{}
-	err = json.Unmarshal([]byte(data), vInfo)
-	if err != nil {
-		panic(err)
-	}
-
-	video_seen := false
-	subtitles_seen := false
-	for _, s := range vInfo.Streams {
-		if s.CodecType == "video" {
-			video_seen = true
-			if (s.Width > 1920 || s.Height > 1080) && !exceptions.HasException(":size:above_maximum") {
-				errs = append(errs, fmt.Errorf("video track is too wide: %dx%d (maximum allowed: 1920x1080)", s.Width, s.Height))
-			}
-
-			if s.CodecName != "h264" {
-				errs = append(errs, fmt.Errorf("video codec has to be H264 (currently: %s)", s.CodecLongName))
-			}
-
-			duration, err := strconv.ParseFloat(s.Duration, 64)
-			if err == nil {
-				if duration < 45 && !exceptions.HasException(":duration:too_short") {
-					errs = append(errs, fmt.Errorf("video is too short"))
-				}
-				if duration > 450 && !exceptions.HasException(":duration:too_long") {
-					errs = append(errs, fmt.Errorf("video is too long"))
-				}
-			} else {
-				errs = append(errs, fmt.Errorf("invalid track duration: %q", s.Duration))
-			}
-		} else if s.CodecType == "subtitle" {
-			subtitles_seen = true
-
-			if s.CodecName != "mov_text" {
-				errs = append(errs, fmt.Errorf("subtitle format has to be MOV text/3GPP Timed Text (currently: %s)", s.CodecLongName))
-			}
-
-			nbframes, err := strconv.ParseInt(s.NbFrames, 10, 64)
-			if err == nil {
-				if nbframes < 5 && !exceptions.HasException(":subtitle:tiny") {
-					errs = append(errs, fmt.Errorf("too few subtitles"))
-				}
-			} else {
-				errs = append(errs, fmt.Errorf("invalid number of frame: %q", s.NbFrames))
-			}
-		} else if s.CodecType == "audio" {
-			if !exceptions.HasException(":audio:allowed") {
-				errs = append(errs, fmt.Errorf("an audio track is present, use subtitle for explainations"))
-			}
-
-			if s.CodecName != "aac" {
-				errs = append(errs, fmt.Errorf("audio codec has to be AAC (Advanced Audio Coding) (currently: %s)", s.CodecLongName))
-			}
-		} else {
-			errs = append(errs, fmt.Errorf("unknown track found of type %q", s.CodecType))
-		}
-	}
-
-	if !video_seen {
-		errs = append(errs, fmt.Errorf("no video track found"))
-	}
-	if !subtitles_seen && !exceptions.HasException(":subtitle:no_track") {
-		errs = append(errs, fmt.Errorf("no subtitles track found"))
 	}
 
 	return
 }
-
-func RegisterChecksHooks(h *sync.CheckHooks) {
-	h.RegisterExerciceHook(CheckResolutionVideo)
-}

repochecker/videos/subtitles.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/asticode/go-astisub"
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+
+	"srs.epita.fr/fic-server/admin/sync"
+)
+
+func CheckGrammarSubtitleTrack(path string, exceptions *sync.CheckExceptions) (errs []error) {
+	tmpfile, err := ioutil.TempFile("", "resolution-*.srt")
+	if err != nil {
+		errs = append(errs, fmt.Errorf("unable to create a temporary file: %w", err))
+		return
+	}
+	defer os.Remove(tmpfile.Name())
+
+	// Extract subtitles
+	err = ffmpeg.Input(path).
+		Output(tmpfile.Name(), ffmpeg.KwArgs{"map": "0:s:0"}).
+		OverWriteOutput().Run()
+	if err != nil {
+		errs = append(errs, fmt.Errorf("ffmpeg returns an error when extracting subtitles track: %w", err))
+	}
+
+	subtitles, err := astisub.OpenFile(tmpfile.Name())
+	if err != nil {
+		log.Println("Unable to open subtitles file:", err)
+		return
+	}
+
+	var lines []string
+	for _, item := range subtitles.Items {
+		lines = append(lines, item.String())
+	}
+	for _, e := range hooks.CallCustomHook("CheckGrammar", strings.Join(lines, "\n"), exceptions) {
+		errs = append(errs, fmt.Errorf("subtitle-track: %w", e))
+	}
+
+	return
+}

repochecker/videos/video.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/url"
+	"strconv"
+
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+
+	"srs.epita.fr/fic-server/admin/sync"
+	"srs.epita.fr/fic-server/libfic"
+)
+
+type VideoInfo struct {
+	Streams []struct {
+		CodecType     string `json:"codec_type"`
+		CodecName     string `json:"codec_name"`
+		CodecLongName string `json:"codec_long_name"`
+		Duration      string
+		NbFrames      string `json:"nb_frames"`
+		Width         int
+		Height        int
+	} `json:"streams"`
+}
+
+func checkResolutionVideo(e *fic.Exercice, exceptions *sync.CheckExceptions) (errs []error) {
+	i, ok := sync.GlobalImporter.(sync.LocalImporter)
+	if !ok {
+		log.Printf("Unable to load `videos-rules.so` as the current Importer is not a LocalImporter (%T).", sync.GlobalImporter)
+		return
+	}
+
+	if len(e.VideoURI) == 0 {
+		return
+	}
+
+	// Filter exceptions to only keep related to resolution.mp4
+	exceptions = exceptions.GetFileExceptions("resolution.mp4")
+
+	path, err := url.PathUnescape(e.VideoURI[9:])
+	if err != nil {
+		path = e.VideoURI
+	}
+	path = i.GetLocalPath(path)
+
+	data, err := ffmpeg.Probe(path)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("unable to open %q: %w", path, err))
+		return
+	}
+
+	vInfo := &VideoInfo{}
+	err = json.Unmarshal([]byte(data), vInfo)
+	if err != nil {
+		panic(err)
+	}
+
+	video_seen := false
+	subtitles_seen := false
+	for _, s := range vInfo.Streams {
+		if s.CodecType == "video" {
+			video_seen = true
+			if (s.Width > 1920 || s.Height > 1080) && !exceptions.HasException(":size:above_maximum") {
+				errs = append(errs, fmt.Errorf("video track is too wide: %dx%d (maximum allowed: 1920x1080)", s.Width, s.Height))
+			}
+
+			if s.CodecName != "h264" {
+				errs = append(errs, fmt.Errorf("video codec has to be H264 (currently: %s)", s.CodecLongName))
+			}
+
+			duration, err := strconv.ParseFloat(s.Duration, 64)
+			if err == nil {
+				if duration < 45 && !exceptions.HasException(":duration:too_short") {
+					errs = append(errs, fmt.Errorf("video is too short"))
+				}
+				if duration > 450 && !exceptions.HasException(":duration:too_long") {
+					errs = append(errs, fmt.Errorf("video is too long"))
+				}
+			} else {
+				errs = append(errs, fmt.Errorf("invalid track duration: %q", s.Duration))
+			}
+		} else if s.CodecType == "subtitle" {
+			subtitles_seen = true
+
+			if s.CodecName != "mov_text" {
+				errs = append(errs, fmt.Errorf("subtitle format has to be MOV text/3GPP Timed Text (currently: %s)", s.CodecLongName))
+			}
+
+			nbframes, err := strconv.ParseInt(s.NbFrames, 10, 64)
+			if err == nil {
+				if nbframes < 5 && !exceptions.HasException(":subtitle:tiny") {
+					errs = append(errs, fmt.Errorf("too few subtitles"))
+				}
+			} else {
+				errs = append(errs, fmt.Errorf("invalid number of frame: %q", s.NbFrames))
+			}
+		} else if s.CodecType == "audio" {
+			if !exceptions.HasException(":audio:allowed") {
+				errs = append(errs, fmt.Errorf("an audio track is present, use subtitle for explainations"))
+			}
+
+			if s.CodecName != "aac" {
+				errs = append(errs, fmt.Errorf("audio codec has to be AAC (Advanced Audio Coding) (currently: %s)", s.CodecLongName))
+			}
+		} else {
+			errs = append(errs, fmt.Errorf("unknown track found of type %q", s.CodecType))
+		}
+	}
+
+	if !video_seen {
+		errs = append(errs, fmt.Errorf("no video track found"))
+	}
+	if !subtitles_seen && !exceptions.HasException(":subtitle:no_track") {
+		errs = append(errs, fmt.Errorf("no subtitles track found"))
+	} else if subtitles_seen {
+		errs = append(errs, CheckGrammarSubtitleTrack(path, exceptions)...)
+	}
+
+	return
+}