From 13cce6cd87147638d3d25941b55832d5e98ee375 Mon Sep 17 00:00:00 2001 From: Ronnie Roller Date: Mon, 3 Feb 2020 09:03:14 -0800 Subject: [PATCH] tweaks --- go.mod | 1 + go.sum | 2 + src/main.go | 2 +- src/media/fetch.go | 120 ++++++++++++++++++++++++++++++---- src/media/serve.go | 40 ++---------- templates/media/index.html | 41 ++++++++++-- templates/media/response.html | 36 ++++++++-- 7 files changed, 183 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 2659b3b..cb12df0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module media-roller go 1.13 require ( + github.com/dustin/go-humanize v1.0.0 github.com/go-chi/chi v4.0.3+incompatible github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 github.com/google/uuid v1.1.1 diff --git a/go.sum b/go.sum index 1a42e57..202dade 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY= github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 h1:JYZmrkBDj6LwUbsRysF9tpLnz59npoZSI3KG2XHqvHw= diff --git a/src/main.go b/src/main.go index 5578f74..c9e2e4b 100644 --- a/src/main.go +++ b/src/main.go @@ -16,7 +16,7 @@ func main() { // Setup routes r := chi.NewRouter() r.Get("/", func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("Welcome!")) + _, _ = w.Write([]byte("media roller")) }) r.Route("/media", func(r chi.Router) { r.Get("/", media.Index) diff --git a/src/media/fetch.go b/src/media/fetch.go index b01d507..fa71d67 100644 --- a/src/media/fetch.go +++ b/src/media/fetch.go @@ -1,8 +1,14 @@ package media import ( + "crypto/md5" + "errors" + "fmt" + "github.com/dustin/go-humanize" "html/template" "net/http" + "path/filepath" + "strings" ) /** @@ -11,7 +17,6 @@ This file will download the media from a URL and save it to disk. import ( "bytes" - "github.com/google/uuid" "github.com/rs/zerolog/log" "io" "os" @@ -21,8 +26,15 @@ import ( const downloadDir = "downloads/" -type ResponseData struct { - Id string +type Media struct { + Id string + Name string + SizeInBytes int64 + HumanSize string +} + +type MediaResults struct { + Medias []Media } // TODO: Use something better than this. It's too tedious to map @@ -43,16 +55,32 @@ func FetchMedia(w http.ResponseWriter, r *http.Request) { return } - id, err := fetch(url) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + // NOTE: This system is for a simple use case, meant to run at home. This is not a great design for a robust system. + // We are hashing the URL here and writing files to disk to a consistent directory based on the ID. You can imagine + // concurrent users would break this for the same URL. That's fine given this is for a simple home system. + // Future work can make this more sophisticated. + id := GetMD5Hash(url) + // Look to see if we already have the media on disk + medias, err := getAllFilesForId(id) + if len(medias) == 0 { + // We don't, so go fetch it + id, err = fetch(url) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + medias, err = getAllFilesForId(id) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } } - data := ResponseData{ - Id: id, + response := MediaResults{ + Medias: medias, } - if err := fetchResponseTmpl.Execute(w, data); err != nil { + + if err := fetchResponseTmpl.Execute(w, response); err != nil { log.Error().Msgf("Error rendering template: %v", err) http.Error(w, "Internal error", http.StatusInternalServerError) } @@ -61,7 +89,7 @@ func FetchMedia(w http.ResponseWriter, r *http.Request) { // returns the ID of the file func fetch(url string) (string, error) { // The id will be used as the name of the parent directory of the output files - id := uuid.New().String() + id := GetMD5Hash(url) name := getMediaDirectory(id) + "%(title)s.%(ext)s" log.Info().Msgf("Downloading %s to %s", url, id) @@ -118,3 +146,73 @@ func fetch(url string) (string, error) { func getMediaDirectory(id string) string { return downloadDir + id + "/" } + +// id is expected to be validated prior to calling this func +func getAllFilesForId(id string) ([]Media, error) { + root := getMediaDirectory(id) + file, err := os.Open(root) + if err != nil { + return nil, err + } + files, _ := file.Readdirnames(0) // 0 to read all files and folders + if len(files) == 0 { + return nil, errors.New("ID not found") + } + + var medias []Media + + // We expect two files to be produced for each video, a json manifest and an mp4. + for _, f := range files { + if !strings.HasSuffix(f, ".json") { + fi, err := os.Stat(root + f) + var size int64 = 0 + if err == nil { + size = fi.Size() + } + + media := Media{ + Id: id, + Name: filepath.Base(f), + SizeInBytes: size, + HumanSize: humanize.Bytes(uint64(size)), + } + medias = append(medias, media) + } + } + + return medias, nil +} + +// id is expected to be validated prior to calling this func +// TODO: This needs to handle multiple files in the directory +func getFileFromId(id string) (string, error) { + root := getMediaDirectory(id) + file, err := os.Open(root) + if err != nil { + return "", err + } + files, _ := file.Readdirnames(0) // 0 to read all files and folders + if len(files) == 0 { + return "", errors.New("ID not found") + } + + // We expect two files to be produced, a json manifest and an mp4. We want to return the mp4 + // Sometimes the video file might not have an mp4 extension, so filter out the json file + for _, f := range files { + if !strings.HasSuffix(f, ".json") { + // TODO: This is just returning the first file found. We need to handle multiple + return root + f, nil + } + } + + return "", errors.New("unable to find file") +} + +func GetMD5Hash(url string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(url))) +} + +func isValidId(id string) bool { + // TODO: Finish this + return true +} diff --git a/src/media/serve.go b/src/media/serve.go index b8f83ed..b48cfa9 100644 --- a/src/media/serve.go +++ b/src/media/serve.go @@ -2,30 +2,24 @@ package media import ( "github.com/rs/zerolog/log" + "io" "net/http" + "os" "path/filepath" - "strings" + "strconv" ) /** This will serve the fetched files to the client */ -import ( - "errors" - "github.com/google/uuid" - "io" - "os" - "strconv" -) - func ServeMedia(w http.ResponseWriter, r *http.Request) { id := r.URL.Query().Get("id") log.Info().Msgf("Serving file %s", id) if id == "" { http.Error(w, "Missing file ID", http.StatusBadRequest) return - } else if _, err := uuid.Parse(id); err != nil { + } else if !isValidId(id) { // Try to parse it just to avoid any type of directory traversal attacks http.Error(w, "Invalid file ID", http.StatusBadRequest) return @@ -39,32 +33,6 @@ func ServeMedia(w http.ResponseWriter, r *http.Request) { streamFileToClient(w, filename) } -// id is expected to be validated prior to calling this func -func getFileFromId(id string) (string, error) { - root := getMediaDirectory(id) - file, err := os.Open(root) - if err != nil { - return "", err - } - files, _ := file.Readdirnames(0) // 0 to read all files and folders - if len(files) == 0 { - return "", errors.New("ID not found") - } else if len(files) > 2 { - // We should only have 2 media file produced, the mp4 and the json file - return "", errors.New("internal error") - } - - // We expect two files to be produced, a json manifest and an mp4. We want to return the mp4 - // Sometimes the video file might not have an mp4 extension, so filter out the json file - for _, f := range files { - if !strings.HasSuffix(f, ".json") { - return root + f, nil - } - } - - return "", errors.New("unable to find file") -} - func streamFileToClient(writer http.ResponseWriter, filename string) { // Check if file exists and open Openfile, err := os.Open(filename) diff --git a/templates/media/index.html b/templates/media/index.html index 576e1d5..962f60e 100644 --- a/templates/media/index.html +++ b/templates/media/index.html @@ -1,7 +1,36 @@ - -media-roller -

Download media file

-
- -
+ + + + media-roller + + + +
+
+
+

media roller

+

+ Mobile friendly tool for downloading videos from social media +

+
+
+
+ +
+ +
+
+
+
+
+ +
+ diff --git a/templates/media/response.html b/templates/media/response.html index 3951089..794a647 100644 --- a/templates/media/response.html +++ b/templates/media/response.html @@ -1,5 +1,31 @@ - - -

Download media file

-download - + + + + media-roller + + + +
+
+
+

media roller

+

+ Mobile friendly tool for downloading videos from social media +

+
+

Done!

+ {{range .Medias}} + {{.Name}} {{.HumanSize}} + {{ end }} +
+
+ +
+ + \ No newline at end of file