vndangkhoa/kv-download
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tweaks

Browse source
This commit is contained in:
Ronnie Roller 2020-02-03 09:03:14 -08:00
parent 7e35d6a68e
commit 13cce6cd87
7 changed files with 183 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -3,6 +3,7 @@ module media-roller
go 1.13
require (
github.com/dustin/go-humanize v1.0.0
github.com/go-chi/chi v4.0.3+incompatible
github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4
github.com/google/uuid v1.1.1

2
go.sum
View file

@ -1,4 +1,6 @@
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY=
github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 h1:JYZmrkBDj6LwUbsRysF9tpLnz59npoZSI3KG2XHqvHw=

2
src/main.go
View file

@ -16,7 +16,7 @@ func main() {
// Setup routes
r := chi.NewRouter()
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("Welcome!"))
_, _ = w.Write([]byte("<a href='/media'>media roller</a>"))
})
r.Route("/media", func(r chi.Router) {
r.Get("/", media.Index)

120
src/media/fetch.go
View file

@ -1,8 +1,14 @@
package media
import (
"crypto/md5"
"errors"
"fmt"
"github.com/dustin/go-humanize"
"html/template"
"net/http"
"path/filepath"
"strings"
)
/**
@ -11,7 +17,6 @@ This file will download the media from a URL and save it to disk.
import (
"bytes"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"io"
"os"
@ -21,8 +26,15 @@ import (
const downloadDir = "downloads/"
type ResponseData struct {
Id string
type Media struct {
Id string
Name string
SizeInBytes int64
HumanSize string
}
type MediaResults struct {
Medias []Media
}
// TODO: Use something better than this. It's too tedious to map
@ -43,16 +55,32 @@ func FetchMedia(w http.ResponseWriter, r *http.Request) {
return
}
id, err := fetch(url)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
// NOTE: This system is for a simple use case, meant to run at home. This is not a great design for a robust system.
// We are hashing the URL here and writing files to disk to a consistent directory based on the ID. You can imagine
// concurrent users would break this for the same URL. That's fine given this is for a simple home system.
// Future work can make this more sophisticated.
id := GetMD5Hash(url)
// Look to see if we already have the media on disk
medias, err := getAllFilesForId(id)
if len(medias) == 0 {
// We don't, so go fetch it
id, err = fetch(url)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
medias, err = getAllFilesForId(id)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
data := ResponseData{
Id: id,
response := MediaResults{
Medias: medias,
}
if err := fetchResponseTmpl.Execute(w, data); err != nil {
if err := fetchResponseTmpl.Execute(w, response); err != nil {
log.Error().Msgf("Error rendering template: %v", err)
http.Error(w, "Internal error", http.StatusInternalServerError)
}
@ -61,7 +89,7 @@ func FetchMedia(w http.ResponseWriter, r *http.Request) {
// returns the ID of the file
func fetch(url string) (string, error) {
// The id will be used as the name of the parent directory of the output files
id := uuid.New().String()
id := GetMD5Hash(url)
name := getMediaDirectory(id) + "%(title)s.%(ext)s"
log.Info().Msgf("Downloading %s to %s", url, id)
@ -118,3 +146,73 @@ func fetch(url string) (string, error) {
func getMediaDirectory(id string) string {
return downloadDir + id + "/"
}
// id is expected to be validated prior to calling this func
func getAllFilesForId(id string) ([]Media, error) {
root := getMediaDirectory(id)
file, err := os.Open(root)
if err != nil {
return nil, err
}
files, _ := file.Readdirnames(0) // 0 to read all files and folders
if len(files) == 0 {
return nil, errors.New("ID not found")
}
var medias []Media
// We expect two files to be produced for each video, a json manifest and an mp4.
for _, f := range files {
if !strings.HasSuffix(f, ".json") {
fi, err := os.Stat(root + f)
var size int64 = 0
if err == nil {
size = fi.Size()
}
media := Media{
Id: id,
Name: filepath.Base(f),
SizeInBytes: size,
HumanSize: humanize.Bytes(uint64(size)),
}
medias = append(medias, media)
}
}
return medias, nil
}
// id is expected to be validated prior to calling this func
// TODO: This needs to handle multiple files in the directory
func getFileFromId(id string) (string, error) {
root := getMediaDirectory(id)
file, err := os.Open(root)
if err != nil {
return "", err
}
files, _ := file.Readdirnames(0) // 0 to read all files and folders
if len(files) == 0 {
return "", errors.New("ID not found")
}
// We expect two files to be produced, a json manifest and an mp4. We want to return the mp4
// Sometimes the video file might not have an mp4 extension, so filter out the json file
for _, f := range files {
if !strings.HasSuffix(f, ".json") {
// TODO: This is just returning the first file found. We need to handle multiple
return root + f, nil
}
}
return "", errors.New("unable to find file")
}
func GetMD5Hash(url string) string {
return fmt.Sprintf("%x", md5.Sum([]byte(url)))
}
func isValidId(id string) bool {
// TODO: Finish this
return true
}

40
src/media/serve.go
View file

@ -2,30 +2,24 @@ package media
import (
"github.com/rs/zerolog/log"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"strconv"
)
/**
This will serve the fetched files to the client
*/
import (
"errors"
"github.com/google/uuid"
"io"
"os"
"strconv"
)
func ServeMedia(w http.ResponseWriter, r *http.Request) {
id := r.URL.Query().Get("id")
log.Info().Msgf("Serving file %s", id)
if id == "" {
http.Error(w, "Missing file ID", http.StatusBadRequest)
return
} else if _, err := uuid.Parse(id); err != nil {
} else if !isValidId(id) {
// Try to parse it just to avoid any type of directory traversal attacks
http.Error(w, "Invalid file ID", http.StatusBadRequest)
return
@ -39,32 +33,6 @@ func ServeMedia(w http.ResponseWriter, r *http.Request) {
streamFileToClient(w, filename)
}
// id is expected to be validated prior to calling this func
func getFileFromId(id string) (string, error) {
root := getMediaDirectory(id)
file, err := os.Open(root)
if err != nil {
return "", err
}
files, _ := file.Readdirnames(0) // 0 to read all files and folders
if len(files) == 0 {
return "", errors.New("ID not found")
} else if len(files) > 2 {
// We should only have 2 media file produced, the mp4 and the json file
return "", errors.New("internal error")
}
// We expect two files to be produced, a json manifest and an mp4. We want to return the mp4
// Sometimes the video file might not have an mp4 extension, so filter out the json file
for _, f := range files {
if !strings.HasSuffix(f, ".json") {
return root + f, nil
}
}
return "", errors.New("unable to find file")
}
func streamFileToClient(writer http.ResponseWriter, filename string) {
// Check if file exists and open
Openfile, err := os.Open(filename)

41
templates/media/index.html
View file

@ -1,7 +1,36 @@
<html lang="">
<head><title>media-roller</title></head>
<h1>Download media file</h1>
<form action="media/fetch">
<input type="text" name="url"><input type="submit" value="Fetch"/>
</form>
<!doctype html>
<html lang="en">
<head>
<title>media-roller</title>
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">
</head>
<body style="background-color: #43464a">
<div class="container d-flex flex-column text-light text-center">
<div class="flex-grow-1"></div>
<div class="jumbotron bg-transparent flex-grow-1">
<h1 class="display-4"><a class="text-light" href="/media">media roller</a></h1>
<p>
Mobile friendly tool for downloading videos from social media
</p>
<div>
<form action="media/fetch" method="GET">
<div class="input-group">
<input name="url" type="url" class="form-control" placeholder="URL" aria-label="URL"
aria-describedby="button-submit" autofocus>
<div class="input-group-append">
<button class="btn btn-primary" type="submit" id="button-submit">Submit</button>
</div>
</div>
</form>
</div>
</div>
<footer>
<div>
<p class="text-muted">Source at <a class="text-light" href="https://github.com/rroller/media-roller">https://github.com/rroller/media-roller</a>
</p>
</div>
</footer>
</div>
</body>
</html>

36
templates/media/response.html
View file

@ -1,5 +1,31 @@
<html lang="">
<head></head>
<h1>Download media file</h1>
<a href="download?id={{.Id}}">download</a>
</html>
<!doctype html>
<html lang="en">
<head>
<title>media-roller</title>
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">
</head>
<body style="background-color: #43464a">
<div class="container d-flex flex-column text-light text-center">
<div class="flex-grow-1"></div>
<div class="jumbotron bg-transparent flex-grow-1">
<h1 class="display-4"><a class="text-light" href="/media">media roller</a></h1>
<p>
Mobile friendly tool for downloading videos from social media
</p>
<div>
<h2>Done!</h2>
{{range .Medias}}
<a style="color: dodgerblue" href="download?id={{.Id}}">{{.Name}}</a> <small>{{.HumanSize}}</small>
{{ end }}
</div>
</div>
<footer>
<div>
<p class="text-muted">Source at <a class="text-light" href="https://github.com/rroller/media-roller">https://github.com/rroller/media-roller</a>
</p>
</div>
</footer>
</div>
</body>
</html>