From 3afc39fed3d712f97f88e1a023528f86639981be Mon Sep 17 00:00:00 2001 From: rroller Date: Sat, 19 Oct 2024 13:40:18 -0700 Subject: [PATCH] Switch from youtube-dl to yt-dlp. Upgrade to go 1.23.2, ffmpeg 7.1, alpine 3.20 --- .gitignore | 4 +- Dockerfile | 110 +++++---------------------------------------- docker-run.sh | 2 +- go.mod | 16 ++++--- go.sum | 40 ++++++++--------- src/main.go | 59 ++++++++++++------------ src/media/fetch.go | 2 +- 7 files changed, 77 insertions(+), 156 deletions(-) diff --git a/.gitignore b/.gitignore index 0d2a8e2..3ea9910 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .idea/ -downloads/ +download/ +.DS_Store +media-roller \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index da569ff..8062245 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,124 +1,38 @@ -FROM golang:1.13.6-alpine3.11 as builder +FROM golang:1.23.2-alpine3.20 AS builder RUN apk add --no-cache curl -# ffmpeg source - https://github.com/alfg/docker-ffmpeg -ARG FFMPEG_VERSION=6.0 -ARG PREFIX=/opt/ffmpeg -ARG LD_LIBRARY_PATH=/opt/ffmpeg/lib -ARG MAKEFLAGS="-j4" - -# FFmpeg build dependencies. -RUN apk update && apk add --update \ - build-base \ - coreutils \ - freetype-dev \ - gcc \ - lame-dev \ - openssl-dev \ - libogg-dev \ - libass \ - libass-dev \ - libvpx-dev \ - libvorbis-dev \ - libwebp-dev \ - libtheora-dev \ - opus-dev \ - pkgconf \ - pkgconfig \ - rtmpdump-dev \ - wget \ - x264-dev \ - x265-dev \ - yasm - -# Get fdk-aac from community. -RUN echo http://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ - apk add --update fdk-aac-dev - -# Get ffmpeg source. -RUN cd /tmp/ && \ - wget http://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ - tar zxf ffmpeg-${FFMPEG_VERSION}.tar.gz && rm ffmpeg-${FFMPEG_VERSION}.tar.gz - -# Compile ffmpeg. -RUN cd /tmp/ffmpeg-${FFMPEG_VERSION} && \ - ./configure \ - --enable-version3 \ - --enable-gpl \ - --enable-nonfree \ - --enable-small \ - --enable-libmp3lame \ - --enable-openssl \ - --enable-libx264 \ - --enable-libx265 \ - --enable-libvpx \ - --enable-libtheora \ - --enable-libvorbis \ - --enable-libopus \ - --enable-libfdk-aac \ - --enable-libass \ - --enable-libwebp \ - --enable-librtmp \ - --enable-postproc \ - --enable-avresample \ - --enable-libfreetype \ - --disable-debug \ - --disable-doc \ - --disable-ffplay \ - --extra-cflags="-I${PREFIX}/include" \ - --extra-ldflags="-L${PREFIX}/lib" \ - --extra-libs="-lpthread -lm" \ - --prefix="${PREFIX}" && \ - make && make install && make distclean - -# Cleanup. -RUN rm -rf /var/cache/apk/* /tmp/* - WORKDIR /app COPY src src COPY templates templates COPY go.mod go.mod +COPY go.sum go.sum RUN go mod download RUN go build -x -o media-roller ./src -# youtube-dl needs python -FROM python:3.8.1-alpine3.11 +# yt-dlp needs python +FROM python:3.13.0-alpine3.20 # This is where the downloaded files will be saved in the container. ENV MR_DOWNLOAD_DIR="/download" -ENV PATH=/opt/ffmpeg/bin:$PATH RUN apk add --update --no-cache \ - curl \ - ca-certificates \ - openssl \ - pcre \ - lame \ - libogg \ - libass \ - libvpx \ - libvorbis \ - libwebp \ - libtheora \ - opus \ - rtmpdump \ - x264-dev \ - x265-dev + curl COPY --from=builder /app/media-roller /app/media-roller -COPY --from=builder /opt/ffmpeg /opt/ffmpeg -COPY --from=builder /usr/lib/libfdk-aac.so.2 /usr/lib/libfdk-aac.so.2 +COPY --from=mwader/static-ffmpeg:7.1 /ffmpeg /usr/local/bin/ COPY templates /app/templates COPY static /app/static WORKDIR /app -RUN curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl && \ - chmod a+rx /usr/local/bin/youtube-dl && \ - youtube-dl --version && \ - ffmpeg -version +RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod a+rx /usr/local/bin/yt-dlp + +# Sanity check +RUN yt-dlp --version && \ + ffmpeg -version CMD /app/media-roller diff --git a/docker-run.sh b/docker-run.sh index 2910a25..9edf236 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -docker run -p 3000:3000 media-roller +docker run -p 3000:3000 -v $(pwd)/download:/download media-roller diff --git a/go.mod b/go.mod index cb12df0..d65c3f5 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,15 @@ module media-roller -go 1.13 +go 1.23 require ( - github.com/dustin/go-humanize v1.0.0 - github.com/go-chi/chi v4.0.3+incompatible - github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 - github.com/google/uuid v1.1.1 - github.com/rs/zerolog v1.17.2 + github.com/dustin/go-humanize v1.0.1 + github.com/go-chi/chi/v5 v5.1.0 + github.com/rs/zerolog v1.33.0 +) + +require ( + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + golang.org/x/sys v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 202dade..e1c7422 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,19 @@ -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY= -github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4 h1:JYZmrkBDj6LwUbsRysF9tpLnz59npoZSI3KG2XHqvHw= -github.com/go-chi/valve v0.0.0-20170920024740-9e45288364f4/go.mod h1:F4ZINQr5T71wO1JOmdQsGTBew+njUAXn65LLGjuagwY= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.17.2 h1:RMRHFw2+wF7LO0QqtELQwo8hqSmqISyCJeFeAAuWcRo= -github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/src/main.go b/src/main.go index 92a2ff0..1e1708c 100644 --- a/src/main.go +++ b/src/main.go @@ -2,8 +2,7 @@ package main import ( "context" - "github.com/go-chi/chi" - "github.com/go-chi/valve" + "github.com/go-chi/chi/v5" "github.com/rs/zerolog/log" "media-roller/src/media" "net/http" @@ -12,6 +11,7 @@ import ( "path" "path/filepath" "strings" + "syscall" "time" ) @@ -36,41 +36,44 @@ func main() { log.Panic().Msgf("%s\n", err.Error()) } - valv := valve.New() - baseCtx := valv.Context() - srv := http.Server{Addr: ":3000", Handler: chi.ServerBaseContext(baseCtx, router)} + // The HTTP Server + server := &http.Server{Addr: ":3000", Handler: router} - // Create a shutdown hook for graceful shutdowns - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) + // Server run context + serverCtx, serverStopCtx := context.WithCancel(context.Background()) + + // Listen for syscall signals for process to interrupt/quit + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { - for range c { - // sig is a ^C, handle it - log.Info().Msgf("Shutting down...") + <-sig - // first valv - _ = valv.Shutdown(20 * time.Second) + // Shutdown signal with grace period of 30 seconds + shutdownCtx, _ := context.WithTimeout(serverCtx, 30*time.Second) - // create context with timeout - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - // start http shutdown - _ = srv.Shutdown(ctx) - - // verify, in worst case call cancel via defer - select { - case <-time.After(21 * time.Second): - log.Error().Msgf("Not all connections done") - case <-ctx.Done(): + go func() { + <-shutdownCtx.Done() + if shutdownCtx.Err() == context.DeadlineExceeded { + log.Fatal().Msg("graceful shutdown timed out.. forcing exit.") } + }() + + // Trigger graceful shutdown + err := server.Shutdown(shutdownCtx) + if err != nil { + log.Fatal().Err(err) } + serverStopCtx() }() - err := srv.ListenAndServe() - if err != nil { - log.Info().Msg(err.Error()) + // Run the server + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + log.Fatal().Err(err) } + + // Wait for server context to be stopped + <-serverCtx.Done() log.Info().Msgf("Shutdown complete") } diff --git a/src/media/fetch.go b/src/media/fetch.go index 9bf3828..77277b2 100644 --- a/src/media/fetch.go +++ b/src/media/fetch.go @@ -117,7 +117,7 @@ func downloadMedia(url string) (string, error) { log.Info().Msgf("Downloading %s to %s", url, id) - cmd := exec.Command("youtube-dl", + cmd := exec.Command("yt-dlp", "--format", "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", "--merge-output-format", "mp4", "--restrict-filenames",