init

.dockerignore

@@ -0,0 +1 @@
+downloads/

.gitignore

@@ -0,0 +1,2 @@
+.idea/
+downloads/

Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.13.6-alpine3.11 as builder
+
+RUN apk add --no-cache curl
+
+WORKDIR /app
+
+COPY src src
+COPY templates templates
+COPY go.mod go.mod
+
+RUN go mod download
+RUN go build -x -o media-roller ./src
+
+# youtube-dl needs python
+FROM python:3.8.1-alpine3.11
+RUN apk add --no-cache ffmpeg \
+    curl && \
+    ffmpeg -version
+
+COPY --from=builder /app/media-roller /app/media-roller
+COPY templates /app/templates
+
+WORKDIR /app
+
+RUN curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl && \
+   chmod a+rx /usr/local/bin/youtube-dl && \
+   youtube-dl --version
+
+CMD /app/media-roller

build.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+go build -x -o media-roller ./src

docker-build.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+docker build -f Dockerfile -t media-roller .

docker-run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+docker run -p 3000:3000 media-roller

go.mod

@@ -0,0 +1,10 @@
+module media-roller
+
+go 1.13
+
+require (
+	github.com/go-chi/chi v4.0.3+incompatible
+	github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4
+	github.com/google/uuid v1.1.1
+	github.com/rs/zerolog v1.17.2
+)

go.sum

@@ -0,0 +1,19 @@
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY=
+github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 h1:JYZmrkBDj6LwUbsRysF9tpLnz59npoZSI3KG2XHqvHw=
+github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4/go.mod h1:F4ZINQr5T71wO1JOmdQsGTBew+njUAXn65LLGjuagwY=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.17.2 h1:RMRHFw2+wF7LO0QqtELQwo8hqSmqISyCJeFeAAuWcRo=
+github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+go run ./src

src/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/valve"
+	"github.com/rs/zerolog/log"
+	"media-roller/src/media"
+	"net/http"
+	"os"
+	"os/signal"
+	"time"
+)
+
+func main() {
+	// Setup routes
+	r := chi.NewRouter()
+	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("Welcome!"))
+	})
+	r.Route("/media", func(r chi.Router) {
+		r.Get("/", media.Index)
+		r.Get("/fetch", media.FetchMedia)
+		r.Get("/download", media.ServeMedia)
+	})
+
+	// Print out all routes
+	walkFunc := func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error {
+		log.Info().Msgf("%s %s", method, route)
+		return nil
+	}
+	// Panic if there is an error
+	if err := chi.Walk(r, walkFunc); err != nil {
+		log.Panic().Msgf("%s\n", err.Error())
+	}
+
+	valv := valve.New()
+	baseCtx := valv.Context()
+	srv := http.Server{Addr: ":3000", Handler: chi.ServerBaseContext(baseCtx, r)}
+
+	// Create a shutdown hook for graceful shutdowns
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for range c {
+			// sig is a ^C, handle it
+			log.Info().Msgf("Shutting down...")
+
+			// first valv
+			_ = valv.Shutdown(20 * time.Second)
+
+			// create context with timeout
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+			defer cancel()
+
+			// start http shutdown
+			_ = srv.Shutdown(ctx)
+
+			// verify, in worst case call cancel via defer
+			select {
+			case <-time.After(21 * time.Second):
+				log.Error().Msgf("Not all connections done")
+			case <-ctx.Done():
+			}
+		}
+	}()
+
+	// Start the listener
+	err := srv.ListenAndServe()
+	if err != nil {
+		log.Info().Msg(err.Error())
+	}
+	log.Info().Msgf("Shutdown complete")
+}

src/media/fetch.go

@@ -0,0 +1,121 @@
+package media
+
+import (
+	"html/template"
+	"net/http"
+)
+
+/**
+This file will download the media from a URL and save it to disk.
+*/
+
+import (
+	"bytes"
+	"github.com/google/uuid"
+	"github.com/rs/zerolog/log"
+	"io"
+	"os"
+	"os/exec"
+	"sync"
+)
+
+const downloadDir = "downloads/"
+
+type ResponseData struct {
+	Id string
+}
+
+var fetchResponseTmpl = template.Must(template.ParseFiles("templates/media/response.html"))
+var fetchIndexTmpl = template.Must(template.ParseFiles("templates/media/index.html"))
+
+func Index(w http.ResponseWriter, _ *http.Request) {
+	if err := fetchIndexTmpl.Execute(w, nil); err != nil {
+		log.Error().Msgf("Error rendering template: %v", err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+	}
+}
+
+func FetchMedia(w http.ResponseWriter, r *http.Request) {
+	url := r.URL.Query().Get("url")
+	if url == "" {
+		http.Error(w, "Missing URL", http.StatusBadRequest)
+		return
+	}
+
+	id, err := fetch(url)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	data := ResponseData{
+		Id: id,
+	}
+	if err := fetchResponseTmpl.Execute(w, data); err != nil {
+		log.Error().Msgf("Error rendering template: %v", err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+	}
+}
+
+// returns the ID of the file
+func fetch(url string) (string, error) {
+	// This will be the output file name
+	id := uuid.New().String()
+	// youtube-dl will add the extension as needed
+	name := getFilenameWithoutExtensionById(id)
+
+	log.Info().Msgf("Downloading %s to %s", url, id)
+
+	cmd := exec.Command("youtube-dl", "-f", "bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4/", "-o", name, url)
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+	stdoutIn, _ := cmd.StdoutPipe()
+	stderrIn, _ := cmd.StderrPipe()
+
+	var errStdout, errStderr error
+	stdout := io.MultiWriter(os.Stdout, &stdoutBuf)
+	stderr := io.MultiWriter(os.Stderr, &stderrBuf)
+
+	err := cmd.Start()
+	if err != nil {
+		log.Error().Msgf("Error starting command: %v", err)
+		return "", err
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		_, errStdout = io.Copy(stdout, stdoutIn)
+		wg.Done()
+	}()
+
+	_, errStderr = io.Copy(stderr, stderrIn)
+	wg.Wait()
+	log.Info().Msgf("Done with %s", id)
+
+	err = cmd.Wait()
+	if err != nil {
+		log.Error().Msgf("cmd.Run() failed with %s", err)
+		return "", err
+	} else if errStdout != nil {
+		log.Error().Msgf("failed to capture stdout: %v", errStdout)
+	} else if errStderr != nil {
+		log.Error().Msgf("failed to capture stderr: %v", errStderr)
+	}
+
+	return id, nil
+}
+
+// Returns the relative filename without the extension. Example:
+// downloads/b541cc43-9833-4146-ab19-71334484c0c1/media
+// where media can be media.mp4
+func getFilenameWithoutExtensionById(id string) string {
+	return getMediaDirectory(id) + "media"
+}
+
+// Returns the relative directory containing the media file, with a trailing slash
+// Id is expected to be pre validated
+func getMediaDirectory(id string) string {
+	return downloadDir + id + "/"
+}

src/media/serve.go

@@ -0,0 +1,105 @@
+package media
+
+import (
+	"github.com/rs/zerolog/log"
+	"net/http"
+)
+
+/**
+This will serve the fetched files to the client
+*/
+
+import (
+	"errors"
+	"github.com/google/uuid"
+	"io"
+	"os"
+	"strconv"
+)
+
+func ServeMedia(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("id")
+	log.Info().Msgf("Serving file %s", id)
+	if id == "" {
+		http.Error(w, "Missing file ID", http.StatusBadRequest)
+		return
+	} else if _, err := uuid.Parse(id); err != nil {
+		// Try to parse it just to avoid any type of directory traversal attacks
+		http.Error(w, "Invalid file ID", http.StatusBadRequest)
+		return
+	}
+
+	filename, err := getFileFromId(id)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	streamFileToClient(w, filename)
+}
+
+// id is expected to be validated prior to calling this func
+func getFileFromId(id string) (string, error) {
+	root := getMediaDirectory(id)
+	file, err := os.Open(root)
+	if err != nil {
+		return "", err
+	}
+	files, _ := file.Readdirnames(0) // 0 to read all files and folders
+	if len(files) == 0 {
+		return "", errors.New("ID not found")
+	} else if len(files) > 1 {
+		// We should only have 1 media file produced
+		return "", errors.New("internal error")
+	}
+
+	return root + files[0], nil
+}
+
+func streamFileToClient(writer http.ResponseWriter, filename string) {
+	// Check if file exists and open
+	Openfile, err := os.Open(filename)
+	defer Openfile.Close() //Close after function return
+	if err != nil {
+		//File not found, send 404
+		http.Error(writer, "File not found.", 404)
+		return
+	}
+
+	// Get the Content-Type of the file
+	// Create a buffer to store the header of the file in
+	fileHeader := make([]byte, 100)
+	//Copy the headers into the FileHeader buffer
+	if _, err = Openfile.Read(fileHeader); err != nil {
+		log.Error().Msgf("File not found, couldn't open for reading at %s %v", filename, err)
+		http.Error(writer, "File not found", 404)
+		return
+	}
+
+	// Get content type of file
+	fileContentType := http.DetectContentType(fileHeader)
+
+	// Get the file size as a string
+	fileStat, _ := Openfile.Stat()
+	fileSize := strconv.FormatInt(fileStat.Size(), 10)
+
+	// Send the headers
+	// Set the following if you want to force the client to download the file
+	// writer.Header().Set("Content-Disposition", "attachment; filename="+filepath.Base(filename))
+	writer.Header().Set("Content-Type", fileContentType)
+	writer.Header().Set("Content-Length", fileSize)
+
+	// Send the file
+	// We read n bytes from the file already, so we reset the offset back to 0
+	if _, err = Openfile.Seek(0, 0); err != nil {
+		log.Error().Msgf("Error seeking into file %s %v", filename, err)
+		http.Error(writer, "File not found", 404)
+		return
+	}
+
+	// Copy the file to the client
+	if _, err = io.Copy(writer, Openfile); err != nil {
+		log.Error().Msgf("Error copying file %s %v", filename, err)
+		http.Error(writer, "Couldn't copy file", 404)
+		return
+	}
+}

templates/media/index.html

@@ -0,0 +1,7 @@
+<html lang="">
+<head><title>media-roller</title></head>
+<h1>Download media file</h1>
+<form action="media/fetch">
+    <input type="text" name="url"><input type="submit" value="Fetch"/>
+</form>
+</html>

templates/media/response.html

@@ -0,0 +1,5 @@
+<html lang="">
+<head></head>
+<h1>Download media file</h1>
+<a href="download?id={{.Id}}">download</a>
+</html>