From b4fa17918074c290818d8d6ca45fe9c3a403b350 Mon Sep 17 00:00:00 2001 From: Thomas Boerger Date: Fri, 15 May 2020 02:13:51 +0200 Subject: [PATCH] Compile Hugo on our own, fix extended version and refactoring Since the extended version is only available for amd64 I have started to build Hugo on our own in a multi stage Dockerfile. Now we are installing both binaries, regular hugo and extended hugo, bundled into the Docker image. Beside that the download for regular hugo and extended hugo should also be more solid now. After that I have also added more aliases for the environment variables. As mentioned on some GitHub issue we are also installing more required packages for a downloaded extended Hugo version as this relies on libc6-compat and libstdc++. --- docker/Dockerfile.linux.amd64 | 20 ++++-- docker/Dockerfile.linux.arm | 20 ++++-- docker/Dockerfile.linux.arm64 | 20 ++++-- download/download.go | 126 ++++++++++++++++++++-------------- main.go | 49 ++++++------- plugin.go | 114 +++++++++++++++++------------- 6 files changed, 205 insertions(+), 144 deletions(-) diff --git a/docker/Dockerfile.linux.amd64 b/docker/Dockerfile.linux.amd64 index 3540007..b105e29 100644 --- a/docker/Dockerfile.linux.amd64 +++ b/docker/Dockerfile.linux.amd64 @@ -1,3 +1,11 @@ +FROM amd64/golang:1.11-alpine AS build + +RUN apk add --no-cache git build-base && \ + git clone --branch v0.58.3 https://github.com/gohugoio/hugo.git && \ + cd hugo/ && \ + CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo . && \ + CGO_ENABLED=1 go build -tags extended -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo-extended + FROM plugins/base:linux-amd64 LABEL maintainer="Drone.IO Community " \ @@ -5,13 +13,13 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.vendor="Drone.IO Community" \ org.label-schema.schema-version="1.0" -ENV HUGO_VERSION=0.58.3 -ENV HUGO_ARCH=64bit -ENV PLUGIN_HUGO_ARCH=$HUGO_ARCH -ENV PLUGIN_HUGO_SHIPPED_VERSION=$HUGO_VERSION +RUN apk --no-cache add git libc6-compat libstdc++ -RUN apk --no-cache add git && \ - wget -O- https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-${HUGO_ARCH}.tar.gz | tar xz -C /bin +ENV PLUGIN_HUGO_ARCH=64bit +ENV PLUGIN_HUGO_SHIPPED_VERSION=0.58.3 + +COPY --from=build /tmp/hugo /bin/hugo +COPY --from=build /tmp/hugo-extended /bin/hugo-extended ADD release/linux/amd64/drone-hugo /bin/ ENTRYPOINT ["/bin/drone-hugo"] diff --git a/docker/Dockerfile.linux.arm b/docker/Dockerfile.linux.arm index 06fd571..2da12ca 100644 --- a/docker/Dockerfile.linux.arm +++ b/docker/Dockerfile.linux.arm @@ -1,3 +1,11 @@ +FROM arm32v6/golang:1.11-alpine AS build + +RUN apk add --no-cache git build-base && \ + git clone --branch v0.58.3 https://github.com/gohugoio/hugo.git && \ + cd hugo/ && \ + CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo . && \ + CGO_ENABLED=1 go build -tags extended -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo-extended + FROM plugins/base:linux-arm LABEL maintainer="Drone.IO Community " \ @@ -5,13 +13,13 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.vendor="Drone.IO Community" \ org.label-schema.schema-version="1.0" -ENV HUGO_VERSION=0.58.3 -ENV HUGO_ARCH=arm -ENV PLUGIN_HUGO_ARCH=$HUGO_ARCH -ENV PLUGIN_HUGO_SHIPPED_VERSION=$HUGO_VERSION +RUN apk --no-cache add git libc6-compat libstdc++ -RUN apk --no-cache add git && \ - wget -O- https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-${HUGO_ARCH}.tar.gz | tar xz -C /bin +ENV PLUGIN_HUGO_ARCH=arm +ENV PLUGIN_HUGO_SHIPPED_VERSION=0.58.3 + +COPY --from=build /tmp/hugo /bin/hugo +COPY --from=build /tmp/hugo-extended /bin/hugo-extended ADD release/linux/arm/drone-hugo /bin/ ENTRYPOINT ["/bin/drone-hugo"] diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 index d22e7bc..f5e767c 100644 --- a/docker/Dockerfile.linux.arm64 +++ b/docker/Dockerfile.linux.arm64 @@ -1,3 +1,11 @@ +FROM arm64v8/golang:1.11-alpine AS build + +RUN apk add --no-cache git build-base && \ + git clone --branch v0.58.3 https://github.com/gohugoio/hugo.git && \ + cd hugo/ && \ + CGO_ENABLED=0 go build -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo . && \ + CGO_ENABLED=1 go build -tags extended -ldflags "-s -w -X github.com/gohugoio/hugo/common/hugo.buildDate=$(date +%Y-%m-%dT%H:%M:%SZ) -X github.com/gohugoio/hugo/common/hugo.commitHash=$(git rev-parse --short HEAD)" -o /tmp/hugo-extended + FROM plugins/base:linux-arm64 LABEL maintainer="Drone.IO Community " \ @@ -5,13 +13,13 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.vendor="Drone.IO Community" \ org.label-schema.schema-version="1.0" -ENV HUGO_VERSION=0.58.3 -ENV HUGO_ARCH=arm64 -ENV PLUGIN_HUGO_ARCH=$HUGO_ARCH -ENV PLUGIN_HUGO_SHIPPED_VERSION=$HUGO_VERSION +RUN apk --no-cache add git libc6-compat libstdc++ -RUN apk --no-cache add git && \ - wget -O- https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-${HUGO_ARCH}.tar.gz | tar xz -C /bin +ENV PLUGIN_HUGO_ARCH=arm64 +ENV PLUGIN_HUGO_SHIPPED_VERSION=0.58.3 + +COPY --from=build /tmp/hugo /bin/hugo +COPY --from=build /tmp/hugo-extended /bin/hugo-extended ADD release/linux/arm64/drone-hugo /bin/ ENTRYPOINT ["/bin/drone-hugo"] diff --git a/download/download.go b/download/download.go index 2fe8cab..2c043fb 100644 --- a/download/download.go +++ b/download/download.go @@ -6,29 +6,87 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "os" "runtime" "strings" - - "github.com/pkg/errors" ) var ( - _downloadURL = "https://github.com/gohugoio/hugo/releases/download/v%s/%s_%s_Linux-%s.tar.gz" + url = "https://github.com/gohugoio/hugo/releases/download/v%s/%s_%s_%s-%s.tar.gz" ) -func downloadURL(version string, extended bool) string { - var archType string - var binary string - +// Get will download the specified hugo verion +func Get(version string, extended bool) (string, error) { + resp, err := http.Get(download(version, extended)) + + if err != nil { + return "", err + } + + defer resp.Body.Close() + gz, err := gzip.NewReader(resp.Body) + + if err != nil { + return "", err + } + + defer gz.Close() + targz := tar.NewReader(gz) + + hugoPath, hugoBin, err := tempfile() + + if err != nil { + return "", err + } + + defer hugoBin.Close() + + for { + h, err := targz.Next() + + if err == io.EOF { + return "", fmt.Errorf("no hugo binary found") + } + + if err != nil { + return "", err + } + + if strings.HasSuffix(h.Name, "hugo") { + io.Copy(hugoBin, targz) + + if err := os.Chmod(hugoPath, 0755); err != nil { + return "", err + } + + return hugoPath, nil + } + } +} + +func download(version string, extended bool) string { + var ( + binary string + osName string + archType string + ) + if extended { binary = "hugo_extended" } else { binary = "hugo" } - + + switch runtime.GOOS { + case "linux": + osName = "Linux" + case "windows": + osName = "Windows" + default: + osName = "unsupported" + } + switch runtime.GOARCH { case "amd64": archType = "64bit" @@ -41,55 +99,17 @@ func downloadURL(version string, extended bool) string { default: archType = "unsupported" } - return fmt.Sprintf(_downloadURL, version, binary, version, archType) + + return fmt.Sprintf(url, version, binary, version, osName, archType) } -func getTempFile() (string, io.WriteCloser, error) { +func tempfile() (string, io.WriteCloser, error) { d, err := ioutil.TempDir("", "") + if err != nil { - return "", nil, errors.Wrap(err, "") + return "", nil, err } + f, err := ioutil.TempFile(d, "") return f.Name(), f, err } - -// Get will download the specified hugo verion -func Get(version string, extended bool) (string, error) { - resp, err := http.Get(downloadURL(version, extended)) - if err != nil { - return "", errors.Wrap(err, "") - } - defer resp.Body.Close() - gz, err := gzip.NewReader(resp.Body) - if err != nil { - return "", errors.Wrap(err, "") - } - defer gz.Close() - targz := tar.NewReader(gz) - - hugoPath, hugoBin, err := getTempFile() - if err != nil { - log.Printf("ERROR: %s", err) - return "", errors.Wrap(err, "") - } - defer hugoBin.Close() - - for { - h, err := targz.Next() - if err == io.EOF { - return "", errors.New("no hugo binary found") - } - if err != nil { - return "", errors.Wrap(err, "") - } - if strings.HasSuffix(h.Name, "hugo") { - io.Copy(hugoBin, targz) - - if err := os.Chmod(hugoPath, 0755); err != nil { - log.Fatal(err) - } - - return hugoPath, nil - } - } -} diff --git a/main.go b/main.go index bbb9011..2c52800 100644 --- a/main.go +++ b/main.go @@ -19,24 +19,24 @@ func main() { app.Version = version app.Flags = []cli.Flag{ cli.BoolFlag{ - Name: "buildDrafts", + Name: "drafts", Usage: " include content marked as draft", - EnvVar: "PLUGIN_BUILDDRAFTS", + EnvVar: "PLUGIN_BUILDDRAFTS,PLUGIN_DRAFTS", }, cli.BoolFlag{ - Name: "buildExpired", + Name: "expired", Usage: "include expired content", - EnvVar: "PLUGIN_BUILDEXPIRED", + EnvVar: "PLUGIN_BUILDEXPIRED,PLUGIN_EXPIRED", }, cli.BoolFlag{ - Name: "buildFuture", + Name: "future", Usage: "include content with publishdate in the future", - EnvVar: "PLUGIN_BUILDFUTURE", + EnvVar: "PLUGIN_BUILDFUTURE,PLUGIN_FUTURE", }, cli.StringFlag{ - Name: "cacheDir", + Name: "cache", Usage: "change cache directory (useful when using caching plugins)", - EnvVar: "PLUGIN_CACHEDIR", + EnvVar: "PLUGIN_CACHEDIR,PLUGIN_CACHE", Value: "", }, cli.StringFlag{ @@ -87,13 +87,13 @@ func main() { EnvVar: "PLUGIN_VALIDATE", }, cli.StringFlag{ - Name: "hugoVersion", + Name: "hugoversion", Usage: "the hugo version to be used", - EnvVar: "PLUGIN_HUGO_VERSION", + EnvVar: "PLUGIN_HUGO_VERSION,PLUGIN_VERSION", Value: "", }, cli.BoolFlag{ - Name: "hugoExtended", + Name: "extended", Usage: "If the hugo extended package should be used", EnvVar: "PLUGIN_EXTENDED", }, @@ -106,21 +106,22 @@ func main() { func run(c *cli.Context) error { plugin := Plugin{ Config: Config{ - HugoVersion: c.String("hugoVersion"), - HugoExtended: c.Bool("hugoExtended"), - BuildDrafts: c.Bool("buildDrafts"), - BuildExpired: c.Bool("buildExpired"), - BuildFuture: c.Bool("buildFuture"), - Validate: c.Bool("validate"), - Config: c.String("config"), - Content: c.String("content"), - Layout: c.String("layout"), - Output: c.String("output"), - Source: c.String("source"), - Theme: c.String("theme"), - Url: c.String("url"), + URL: c.String("url"), + Drafts: c.Bool("drafts"), + Expired: c.Bool("expired"), + Future: c.Bool("future"), + Validate: c.Bool("validate"), + Config: c.String("config"), + Content: c.String("content"), + Layout: c.String("layout"), + Output: c.String("output"), + Source: c.String("source"), + Theme: c.String("theme"), + Version: c.String("hugoversion"), + Extended: c.Bool("extended"), }, BuildInVersion: os.Getenv("PLUGIN_HUGO_SHIPPED_VERSION"), } + return plugin.Exec() } diff --git a/plugin.go b/plugin.go index 6168467..97340fd 100644 --- a/plugin.go +++ b/plugin.go @@ -16,43 +16,53 @@ type ( } Config struct { - BuildDrafts bool - BuildExpired bool - BuildFuture bool - CacheDir string - Config string - Content string - Layout string - Output string - Source string - Theme string - Url string - HugoVersion string - HugoExtended bool - Validate bool + URL string + Drafts bool + Expired bool + Future bool + Validate bool + Cache string + Config string + Content string + Layout string + Output string + Source string + Theme string + Version string + Extended bool } ) -var hugoExecutable = "hugo" +var ( + hugoExecutable = "hugo" +) // Exec executes the plugins functionality func (p Plugin) Exec() error { var cmds = make([]*exec.Cmd, 0) + if p.Config.Extended { + hugoExecutable = "hugo-extended" + } + // Check if buildIn plugin version equals // plugin version declared in drone.yml - if !versionsEqual(p.BuildInVersion, p.Config.HugoVersion, p.Config.HugoExtended) { - hugoVersion := p.Config.HugoVersion - if hugoVersion == "" { - hugoVersion = p.BuildInVersion + if !versionsEqual(p.BuildInVersion, p.Config.Version) { + version := p.Config.Version + + if version == "" { + version = p.BuildInVersion } - hugoPath, err := download.Get(hugoVersion, p.Config.HugoExtended) + executable, err := download.Get(version, p.Config.Extended) + if err != nil { return err } - hugoExecutable = hugoPath + + hugoExecutable = executable } + if p.Config.Validate { cmds = append(cmds, commandValidate(p.Config)) } @@ -63,75 +73,66 @@ func (p Plugin) Exec() error { func commandValidate(config Config) *exec.Cmd { args := []string{"check"} + if config.Config != "" { args = append(args, "--config", config.Config) } + return exec.Command(hugoExecutable, args...) } func commandBuild(config Config) *exec.Cmd { var args = make([]string, 0) - // add bool args - if config.BuildDrafts { + if config.Drafts { args = append(args, "-D") } - if config.BuildExpired { + + if config.Expired { args = append(args, "-E") } - if config.BuildFuture { + + if config.Future { args = append(args, "-F") } - // add string args - if config.CacheDir != "" { - args = append(args, "--cacheDir", config.CacheDir) + + if config.Cache != "" { + args = append(args, "--cacheDir", config.Cache) } + if config.Config != "" { args = append(args, "--config", config.Config) } + if config.Content != "" { args = append(args, "--contentDir", config.Content) } + if config.Layout != "" { args = append(args, "--layoutDir", config.Layout) } + if config.Output != "" { args = append(args, "--destination", config.Output) } + if config.Source != "" { args = append(args, "--source", config.Source) } + if config.Theme != "" { args = append(args, "--theme", config.Theme) } - if config.Url != "" { - args = append(args, "--baseURL", config.Url) + + if config.URL != "" { + args = append(args, "--baseURL", config.URL) } return exec.Command(hugoExecutable, args...) } -// trace writes each command to stdout with the command wrapped in an xml -// tag so that it can be extracted and displayed in the logs. -func trace(cmd *exec.Cmd) { - fmt.Fprintf(os.Stdout, "+ %s\n", strings.Join(cmd.Args, " ")) -} - -func versionsEqual(version string, toCompare string, extended bool) bool { - if extended { - return false - } - - if toCompare == version || toCompare == "" { - return true - } else { - return false - } -} - // execAll executes a slice of commands as a batch job func execAll(cmds []*exec.Cmd) error { - // Execute all commands for _, cmd := range cmds { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -141,5 +142,20 @@ func execAll(cmds []*exec.Cmd) error { return err } } + return nil } + +func versionsEqual(version string, toCompare string) bool { + if toCompare == version || toCompare == "" { + return true + } + + return false +} + +// trace writes each command to stdout with the command wrapped in an xml +// tag so that it can be extracted and displayed in the logs. +func trace(cmd *exec.Cmd) { + fmt.Fprintf(os.Stdout, "+ %s\n", strings.Join(cmd.Args, " ")) +}