Merge pull request #3 from klahaha/tests

Some refactoring

.gitignore

@@ -1,2 +1,5 @@
-bin/
-streams.yaml
+config.yml
+config.yaml
+go-transcode
+bin
+.env

Dockerfile

@@ -17,9 +17,9 @@ RUN set -eux; apt update; \
 COPY . .
 
 RUN go get -v -t -d .; \
-    ./build
+    go build -o bin/go-transcode
 
 ENV TRANSCODE_BIND=:8080
 
-ENTRYPOINT [ "bin/transcode" ]
+ENTRYPOINT [ "bin/go-transcode" ]
 CMD [ "serve" ]

Dockerfile.nvidia

@@ -14,7 +14,7 @@ RUN groupadd --gid $USER_GID $USERNAME; \
     useradd --uid $USER_UID --gid $USERNAME --shell /bin/bash --create-home $USERNAME;
 
 COPY --from=build /app/bin bin
-COPY profiles_nvidia profiles
+COPY profiles profiles
 COPY data data
 
 ENV USER=$USERNAME

README.md

@@ -1,53 +1,65 @@
-# Go live HTTP on-demand transcoding
-Transcoding is expensive and resource consuming operation on CPU and GPU. For big companies with thousands of customers it is essential, to have a dedicated 24/7 transcoding servers. But we, single sporadic users of transcoding, need to have different approach. Transcoding should be done only when its output is really needed. This tool is trying to solve this problem by offering transcoding on demand.
+# go-transcode HTTP on-demand transcoding API
 
-This tool is intended to be used with live streams only. Seeking is not supported, yet.
+## Why
+
+Transcoding is expensive and resource consuming operation on CPU and GPU. For big companies with thousands of customers it is essential, to have a dedicated 24/7 transcoding servers which can store all the transcoded versions.
+
+For the rest of us who don't have infinite resources and cannot have 3 times bigger media library because of transcoding, we should only transcode when it is needed. This tool is trying to solve this problem by offering transcoding on demand.
+
+This feature is common in media centers (plex, jellyfin) but there was no simple transcoding server without all other media center features. Now there is one! go-transcode is simple and extensible, and will probably not add features unrelated to transcoding.
+
+## Features
+
+Sources:
+- [x] Live streams
+- [ ] Static files (basic support)
+- [x] Any codec/container supported by ffmpeg
+
+Outputs:
+- [x] Basic MP4 over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]`
+- [x] Basic HLS over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]/index.m3u8`
+- [x] Demo HTML player (for HLS) : `http://go-transcode/[profile]/[stream-id]/play.html`
+
+Features:
+- [ ] Seeking for static files (index)
+- [ ] Audio/Subtitles tracks
+- [ ] Private mode (serve users authenticated by reverse proxy)
 
 ## Config
-Specify streams as object in yaml file.
 
-### Streams
-Create `streams.yaml` file, with your streams:
+Place your config file in `./config.yaml` (or `/etc/transcode/config.yaml`). The streams are defined like this:
 
 ```yaml
 streams:
   <stream-id>: <stream-url>
 ```
 
-Example:
+Full configuration example:
+
 ```yaml
+# allow debug outputs
+debug: true
+
+# bind server to IP:PORT (use :8888 for all connections)
+bind: localhost:8888
+
+# serve static files from this directory (optional)
+static: /var/www/html
+
+# TODO: issue #4
+proxy: true
+
 streams:
   cam: rtmp://localhost/live/cam
   ch1_hd: http://192.168.1.34:9981/stream/channelid/85
   ch2_hd: http://192.168.1.34:9981/stream/channelid/43
 ```
 
-HTTP streaming is accessible via:
-- `http://localhost:8080/<profile>/<stream-id>`
-
-HLS is accessible via:
-- `http://localhost:8080/<profile>/<stream-id>/index.m3u8`
-- `http://localhost:8080/<profile>/<stream-id>/play.html`
-
-## CPU Profiles
-Profiles (HTTP and HLS) with CPU transcoding can be found in `profiles`:
+## Transcoding profiles
 
-* h264_360p
-* h264_540p
-* h264_720p
-* h264_1080p
+go-transcode supports any formats that ffmpeg likes. We provide profiles out-of-the-box for h264+aac (mp4 container) for 360p, 540p, 720p and 1080p resolutions: `h264_360p`, `h264_540p`, `h264_720p` and `h264_1080p`. Profiles can have any name, but must match regex: `^[0-9A-Za-z_-]+$`
 
-Profile names must match flowing regex: `^[0-9A-Za-z_-]+$`
-
-## GPU Profiles
-Profiles (HTTP and HLS) with GPU transcoding can be found in `profiles_nvidia`:
-
-* h264_360p
-* h264_540p
-* h264_720p
-* h264_1080p
-
-Profile names must match flowing regex: `^[0-9A-Za-z_-]+$`
+In these profile directories, actual profiles are located in `hls/` and `http/`, depending on the output format requested. The profiles scripts detect hardware support by running ffmpeg. No special config needed to use hardware acceleration.
 
 ## Docker
 
@@ -63,7 +75,7 @@ docker build -t go-transcode:latest .
 docker run --rm -d \
   --name="go-transcode" \
   -p "8080:8080" \
-  -v "${PWD}/streams.yaml:/app/streams.yaml" go-transcode:latest
+  -v "${PWD}/config.yaml:/app/config.yaml" go-transcode:latest
 ```
 
 ## Nvidia GPU support (docker)
@@ -85,7 +97,7 @@ docker run --rm -d \
   --gpus=all \
   --name="go-transcode-nvidia" \
   -p "8080:8080" \
-  -v "${PWD}/streams.yaml:/app/streams.yaml" go-transcode-nvidia:latest
+  -v "${PWD}/config.yaml:/app/config.yaml" go-transcode-nvidia:latest
 ```
 
 ### Supported inputs
@@ -103,3 +115,34 @@ Input codec will be automatically determined from given stream. Please check you
 | vc1        | vc1_cuvid   | SMPTE VC-1                                |
 | vp8        | vp8_cuvid   | On2 VP8                                   |
 | vp9        | vp9_cuvid   | Google VP9                                |
+
+## Alternatives
+
+- [nginx-vod-module](https://github.com/kaltura/nginx-vod-module): Only supports MP4 sources.
+- [tvheadend](https://tvheadend.org/): Intended for various live sources (IPTV or DVB), not media library - although it can record TV. Supports Nvidia acceleration, but it is hard to compile.
+- [jellyfin](https://github.com/jellyfin/jellyfin): Supports live TV sources, although does not work realiably. Cannot run standalone transcoding service (without media library).
+- Any suggestions?
+
+## Contribute
+
+Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) (or the [#go-transcode-general](https://matrix.to/#/#go-transcode-general:proxychat.net) room directly) or [via XMPP bridge](xmpp:#go-transcode-general#[email protected]).
+
+## Architecture
+
+The source code is in the following files/folders:
+
+- `cmd/` and `main.go`: source for the command-line interface
+- `hls/`: process runner for HLS transcoding
+- `internal/`: actual source code logic
+
+*TODO: document different modules/packages and dependencies*
+
+Other files/folders in the repositories are:
+
+- `data/`: files used/served by go-transcode
+- `dev/`: some docker helper scripts
+- `profiles/`: the ffmpeg profiles for transcoding
+- `tests/`: some tests for the project
+- `Dockerfile`, `Dockerfile.nvidia` and `docker-compose.yaml`: for the docker lovers
+- `god.mod` and `go.sum`: golang dependencies/modules tracking
+- `LICENSE`: licensing information (Apache 2.0)

cmd/root.go

@@ -4,12 +4,13 @@ import (
 	"os"
 	"runtime"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/m1k1o/go-transcode"
+	transcode "github.com/m1k1o/go-transcode/internal"
 )
 
 func Execute() error {
@@ -24,59 +25,59 @@ var root = &cobra.Command{
 
 func init() {
 	cobra.OnInitialize(func() {
+		config := transcode.Service.RootConfig
+		config.Set()
+
 		//////
 		// logs
 		//////
-		zerolog.TimeFieldFormat = ""
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+		zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout})
 
-		if viper.GetBool("debug") {
+		if config.Debug {
 			zerolog.SetGlobalLevel(zerolog.DebugLevel)
+		} else {
+			zerolog.SetGlobalLevel(zerolog.InfoLevel)
 		}
 
-		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout})
-
 		//////
 		// configs
 		//////
-		config := viper.GetString("config")
-		if config != "" {
-			viper.SetConfigFile(config) // Use config file from the flag.
+		if config.CfgFile != "" {
+			viper.SetConfigFile(config.CfgFile) // use config file from the flag
 		} else {
 			if runtime.GOOS == "linux" {
 				viper.AddConfigPath("/etc/transcode/")
 			}
 
 			viper.AddConfigPath(".")
-			viper.SetConfigName("transcode")
+			viper.SetConfigName("config")
 		}
 
 		viper.SetEnvPrefix("transcode")
 		viper.AutomaticEnv() // read in environment variables that match
 
-		if err := viper.ReadInConfig(); err != nil {
-			if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
-				log.Error().Err(err)
-			}
-			if config != "" {
-				log.Error().Err(err)
-			}
+		err := viper.ReadInConfig()
+		if err != nil && config.CfgFile != "" {
+			log.Err(err)
 		}
 
-		file := viper.ConfigFileUsed()
 		logger := log.With().
-			Bool("debug", viper.GetBool("debug")).
-			Str("logging", viper.GetString("logs")).
-			Str("config", file).
+			Bool("debug", config.Debug).
 			Logger()
 
-		if file == "" {
-			logger.Warn().Msg("preflight complete without config file")
+		file := viper.ConfigFileUsed()
+		if file != "" {
+			viper.OnConfigChange(func(e fsnotify.Event) {
+				log.Info().Msg("config file reloaded")
+				transcode.Service.ConfigReload()
+			})
+
+			viper.WatchConfig()
+			logger.Info().Str("config", file).Msg("preflight complete with config file")
 		} else {
-			logger.Info().Msg("preflight complete")
+			logger.Warn().Msg("preflight complete without config file")
 		}
-
-		transcode.Service.RootConfig.Set()
 	})
 
 	if err := transcode.Service.RootConfig.Init(root); err != nil {

cmd/serve.go

@@ -4,8 +4,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
-	"github.com/m1k1o/go-transcode"
-	"github.com/m1k1o/go-transcode/internal/config"
+	"github.com/m1k1o/go-transcode/internal"
 )
 
 func init() {
@@ -16,21 +15,13 @@ func init() {
 		Run:   transcode.Service.ServeCommand,
 	}
 
-	configs := []config.Config{
-		transcode.Service.ServerConfig,
-	}
-
 	cobra.OnInitialize(func() {
-		for _, cfg := range configs {
-			cfg.Set()
-		}
+		transcode.Service.ServerConfig.Set()
 		transcode.Service.Preflight()
 	})
 
-	for _, cfg := range configs {
-		if err := cfg.Init(command); err != nil {
-			log.Panic().Err(err).Msg("unable to run serve command")
-		}
+	if err := transcode.Service.ServerConfig.Init(command); err != nil {
+		log.Panic().Err(err).Msg("unable to run serve command")
 	}
 
 	root.AddCommand(command)

dev/start

@@ -6,4 +6,5 @@ docker run --rm -it \
     -p "3005:8080" \
     -v "${PWD}/../:/app" \
     --entrypoint="/bin/bash" \
-    transcode_server_img -c '/app/build && ./bin/transcode serve --bind :8080';
+    --workdir="/app" \
+    transcode_server_img -c 'go build && ./go-transcode serve --bind :8080';

docker-compose.yaml

@@ -8,5 +8,5 @@ services:
     ports:
       - "8080:8080"
     volumes:
-      - ./streams.yaml:/app/streams.yaml
+      - ./config.yaml:/app/config.yaml
     command: serve -d

go.mod

@@ -3,27 +3,29 @@ module github.com/m1k1o/go-transcode
 go 1.17
 
 require (
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1
 	github.com/go-chi/chi v1.5.4
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/rs/zerolog v1.24.0
+	github.com/rs/zerolog v1.25.0
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.8.1
-	golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 // indirect
+	github.com/spf13/viper v1.9.0
+	golang.org/x/sys v0.0.0-20210925032602-92d5a993a665 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/ini.v1 v1.63.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/ini.v1 v1.63.2 // indirect
 )
 
 require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )