fic
/
server
2
0
Fork 0
Code Issues Pull Requests 10 Releases Wiki Activity

repochecker/videos: Also check grammar in subtitles

This commit is contained in:
nemunaire 2022-10-31 17:00:37 +01:00
parent acdf0a6261
commit 5d716106c4
6 changed files with 192 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile-admin
View File

@ -14,7 +14,8 @@ RUN apk add --no-cache build-base && \
go get -d -v ./admin && \
go build -v -buildvcs=false -o admin/admin ./admin && \
go build -v -buildmode=plugin -o repochecker/epita-rules.so ./repochecker/epita && \
go build -v -buildmode=plugin -o repochecker/grammalecte-rules.so ./repochecker/grammalecte
go build -v -buildmode=plugin -o repochecker/grammalecte-rules.so ./repochecker/grammalecte && \
go build -v -buildmode=plugin -o repochecker/videos-rules.so ./repochecker/videos
FROM alpine:3.16
@ -35,3 +36,4 @@ ENTRYPOINT ["/srv/admin", "-bind=:8081", "-baseurl=/admin/"]
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/admin/admin /srv/admin
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/epita-rules.so /srv/epita-rules.so
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/grammalecte-rules.so /usr/lib/grammalecte-rules.so
COPY --from=gobuild /go/src/srs.epita.fr/fic-server/repochecker/videos-rules.so /usr/lib/videos-rules.so

3
go.mod
View File

@ -4,6 +4,7 @@ go 1.18
require (
github.com/BurntSushi/toml v1.2.1
github.com/asticode/go-astisub v0.21.0
github.com/gin-gonic/gin v1.8.1
github.com/go-git/go-git/v5 v5.4.2
github.com/go-sql-driver/mysql v1.6.0
@ -22,6 +23,8 @@ require (
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/asticode/go-astikit v0.20.0 // indirect
github.com/asticode/go-astits v1.8.0 // indirect
github.com/aws/aws-sdk-go v1.38.20 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect

9
go.sum
View File

@ -11,6 +11,12 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=
github.com/asticode/go-astikit v0.20.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0=
github.com/asticode/go-astisub v0.21.0 h1:xaCx7SnqblsR7ZqFbo9wq/JYwen7IAG2AVIcfecLxNI=
github.com/asticode/go-astisub v0.21.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8=
github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg=
github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ=
github.com/aws/aws-sdk-go v1.38.20 h1:QbzNx/tdfATbdKfubBpkt84OM6oBkxQZRw6+bW2GyeA=
github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@ -113,6 +119,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@ -165,6 +172,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
@ -181,6 +189,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

112
repochecker/videos/main.go
View File

@ -1,120 +1,24 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/url"
"strconv"
ffmpeg "github.com/u2takey/ffmpeg-go"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
)
type VideoInfo struct {
Streams []struct {
CodecType string `json:"codec_type"`
CodecName string `json:"codec_name"`
CodecLongName string `json:"codec_long_name"`
Duration string
NbFrames string `json:"nb_frames"`
Width int
Height int
} `json:"streams"`
var hooks *sync.CheckHooks
func RegisterChecksHooks(h *sync.CheckHooks) {
hooks = h
h.RegisterExerciceHook(CheckResolutionVideo)
}
func CheckResolutionVideo(e *fic.Exercice, exceptions *sync.CheckExceptions) (errs []error) {
i, ok := sync.GlobalImporter.(sync.LocalImporter)
if !ok {
log.Printf("Unable to load `videos-rules.so` as the current Importer is not a LocalImporter (%T).", sync.GlobalImporter)
return
}
if len(e.VideoURI) == 0 {
return
}
path, err := url.PathUnescape(e.VideoURI[9:])
if err != nil {
path = e.VideoURI
}
data, err := ffmpeg.Probe(i.GetLocalPath(path))
if err != nil {
errs = append(errs, fmt.Errorf("unable to open %q: %w", path, err))
return
}
vInfo := &VideoInfo{}
err = json.Unmarshal([]byte(data), vInfo)
if err != nil {
panic(err)
}
video_seen := false
subtitles_seen := false
for _, s := range vInfo.Streams {
if s.CodecType == "video" {
video_seen = true
if (s.Width > 1920 || s.Height > 1080) && !exceptions.HasException(":size:above_maximum") {
errs = append(errs, fmt.Errorf("video track is too wide: %dx%d (maximum allowed: 1920x1080)", s.Width, s.Height))
}
if s.CodecName != "h264" {
errs = append(errs, fmt.Errorf("video codec has to be H264 (currently: %s)", s.CodecLongName))
}
duration, err := strconv.ParseFloat(s.Duration, 64)
if err == nil {
if duration < 45 && !exceptions.HasException(":duration:too_short") {
errs = append(errs, fmt.Errorf("video is too short"))
}
if duration > 450 && !exceptions.HasException(":duration:too_long") {
errs = append(errs, fmt.Errorf("video is too long"))
}
} else {
errs = append(errs, fmt.Errorf("invalid track duration: %q", s.Duration))
}
} else if s.CodecType == "subtitle" {
subtitles_seen = true
if s.CodecName != "mov_text" {
errs = append(errs, fmt.Errorf("subtitle format has to be MOV text/3GPP Timed Text (currently: %s)", s.CodecLongName))
}
nbframes, err := strconv.ParseInt(s.NbFrames, 10, 64)
if err == nil {
if nbframes < 5 && !exceptions.HasException(":subtitle:tiny") {
errs = append(errs, fmt.Errorf("too few subtitles"))
}
} else {
errs = append(errs, fmt.Errorf("invalid number of frame: %q", s.NbFrames))
}
} else if s.CodecType == "audio" {
if !exceptions.HasException(":audio:allowed") {
errs = append(errs, fmt.Errorf("an audio track is present, use subtitle for explainations"))
}
if s.CodecName != "aac" {
errs = append(errs, fmt.Errorf("audio codec has to be AAC (Advanced Audio Coding) (currently: %s)", s.CodecLongName))
}
} else {
errs = append(errs, fmt.Errorf("unknown track found of type %q", s.CodecType))
}
}
if !video_seen {
errs = append(errs, fmt.Errorf("no video track found"))
}
if !subtitles_seen && !exceptions.HasException(":subtitle:no_track") {
errs = append(errs, fmt.Errorf("no subtitles track found"))
for _, err := range checkResolutionVideo(e, exceptions) {
errs = append(errs, fmt.Errorf("resolution.mp4: %w", err))
}
return
}
func RegisterChecksHooks(h *sync.CheckHooks) {
h.RegisterExerciceHook(CheckResolutionVideo)
}

47
repochecker/videos/subtitles.go Normal file
View File

@ -0,0 +1,47 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"github.com/asticode/go-astisub"
ffmpeg "github.com/u2takey/ffmpeg-go"
"srs.epita.fr/fic-server/admin/sync"
)
func CheckGrammarSubtitleTrack(path string, exceptions *sync.CheckExceptions) (errs []error) {
tmpfile, err := ioutil.TempFile("", "resolution-*.srt")
if err != nil {
errs = append(errs, fmt.Errorf("unable to create a temporary file: %w", err))
return
}
defer os.Remove(tmpfile.Name())
// Extract subtitles
err = ffmpeg.Input(path).
Output(tmpfile.Name(), ffmpeg.KwArgs{"map": "0:s:0"}).
OverWriteOutput().Run()
if err != nil {
errs = append(errs, fmt.Errorf("ffmpeg returns an error when extracting subtitles track: %w", err))
}
subtitles, err := astisub.OpenFile(tmpfile.Name())
if err != nil {
log.Println("Unable to open subtitles file:", err)
return
}
var lines []string
for _, item := range subtitles.Items {
lines = append(lines, item.String())
}
for _, e := range hooks.CallCustomHook("CheckGrammar", strings.Join(lines, "\n"), exceptions) {
errs = append(errs, fmt.Errorf("subtitle-track: %w", e))
}
return
}

122
repochecker/videos/video.go Normal file
View File

@ -0,0 +1,122 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/url"
"strconv"
ffmpeg "github.com/u2takey/ffmpeg-go"
"srs.epita.fr/fic-server/admin/sync"
"srs.epita.fr/fic-server/libfic"
)
type VideoInfo struct {
Streams []struct {
CodecType string `json:"codec_type"`
CodecName string `json:"codec_name"`
CodecLongName string `json:"codec_long_name"`
Duration string
NbFrames string `json:"nb_frames"`
Width int
Height int
} `json:"streams"`
}
func checkResolutionVideo(e *fic.Exercice, exceptions *sync.CheckExceptions) (errs []error) {
i, ok := sync.GlobalImporter.(sync.LocalImporter)
if !ok {
log.Printf("Unable to load `videos-rules.so` as the current Importer is not a LocalImporter (%T).", sync.GlobalImporter)
return
}
if len(e.VideoURI) == 0 {
return
}
// Filter exceptions to only keep related to resolution.mp4
exceptions = exceptions.GetFileExceptions("resolution.mp4")
path, err := url.PathUnescape(e.VideoURI[9:])
if err != nil {
path = e.VideoURI
}
path = i.GetLocalPath(path)
data, err := ffmpeg.Probe(path)
if err != nil {
errs = append(errs, fmt.Errorf("unable to open %q: %w", path, err))
return
}
vInfo := &VideoInfo{}
err = json.Unmarshal([]byte(data), vInfo)
if err != nil {
panic(err)
}
video_seen := false
subtitles_seen := false
for _, s := range vInfo.Streams {
if s.CodecType == "video" {
video_seen = true
if (s.Width > 1920 || s.Height > 1080) && !exceptions.HasException(":size:above_maximum") {
errs = append(errs, fmt.Errorf("video track is too wide: %dx%d (maximum allowed: 1920x1080)", s.Width, s.Height))
}
if s.CodecName != "h264" {
errs = append(errs, fmt.Errorf("video codec has to be H264 (currently: %s)", s.CodecLongName))
}
duration, err := strconv.ParseFloat(s.Duration, 64)
if err == nil {
if duration < 45 && !exceptions.HasException(":duration:too_short") {
errs = append(errs, fmt.Errorf("video is too short"))
}
if duration > 450 && !exceptions.HasException(":duration:too_long") {
errs = append(errs, fmt.Errorf("video is too long"))
}
} else {
errs = append(errs, fmt.Errorf("invalid track duration: %q", s.Duration))
}
} else if s.CodecType == "subtitle" {
subtitles_seen = true
if s.CodecName != "mov_text" {
errs = append(errs, fmt.Errorf("subtitle format has to be MOV text/3GPP Timed Text (currently: %s)", s.CodecLongName))
}
nbframes, err := strconv.ParseInt(s.NbFrames, 10, 64)
if err == nil {
if nbframes < 5 && !exceptions.HasException(":subtitle:tiny") {
errs = append(errs, fmt.Errorf("too few subtitles"))
}
} else {
errs = append(errs, fmt.Errorf("invalid number of frame: %q", s.NbFrames))
}
} else if s.CodecType == "audio" {
if !exceptions.HasException(":audio:allowed") {
errs = append(errs, fmt.Errorf("an audio track is present, use subtitle for explainations"))
}
if s.CodecName != "aac" {
errs = append(errs, fmt.Errorf("audio codec has to be AAC (Advanced Audio Coding) (currently: %s)", s.CodecLongName))
}
} else {
errs = append(errs, fmt.Errorf("unknown track found of type %q", s.CodecType))
}
}
if !video_seen {
errs = append(errs, fmt.Errorf("no video track found"))
}
if !subtitles_seen && !exceptions.HasException(":subtitle:no_track") {
errs = append(errs, fmt.Errorf("no subtitles track found"))
} else if subtitles_seen {
errs = append(errs, CheckGrammarSubtitleTrack(path, exceptions)...)
}
return
}