sheepflix/patches/go-transcode-vaapi.patch

diff --git a/hlsvod/manager.go b/hlsvod/manager.go
index 662c41d..51460f8 100644
--- a/hlsvod/manager.go
+++ b/hlsvod/manager.go
@@ -378,12 +378,17 @@ func (m *ManagerCtx) waitForSegment(index int) (chan struct{}, bool) {
 
 func (m *ManagerCtx) transcodeSegments(offset, limit int) error {
 	logger := m.logger.With().Int("offset", offset).Int("limit", limit).Logger()
+	inputCodec := ""
+	if m.metadata.Video != nil {
+		inputCodec = m.metadata.Video.CodecName
+	}
 
 	segmentTimes := m.breakpoints[offset : offset+limit+1]
 	logger.Info().Interface("segments-times", segmentTimes).Msg("transcoding segments")
 
 	segments, err := TranscodeSegments(m.ctx, m.config.FFmpegBinary, TranscodeConfig{
 		InputFilePath: m.config.MediaPath,
+		InputCodec:    inputCodec,
 		OutputDirPath: m.config.TranscodeDir,
 		SegmentPrefix: m.config.SegmentPrefix, // This does not need to match.
 
diff --git a/hlsvod/probe.go b/hlsvod/probe.go
index c39461c..58c3edd 100644
--- a/hlsvod/probe.go
+++ b/hlsvod/probe.go
@@ -83,6 +83,7 @@ func ProbeMedia(ctx context.Context, ffprobeBinary string, inputFilePath string)
 			}
 
 			data.Video = &ProbeVideoData{
+				CodecName: stream.CodecName,
 				Width:    stream.Width,
 				Height:   stream.Height,
 				Duration: duration,
@@ -118,6 +119,7 @@ func ProbeMedia(ctx context.Context, ffprobeBinary string, inputFilePath string)
 }
 
 type ProbeVideoData struct {
+	CodecName  string
 	Width      int
 	Height     int
 	Duration   time.Duration
diff --git a/hlsvod/transcode.go b/hlsvod/transcode.go
index 9bbe037..c0297fe 100644
--- a/hlsvod/transcode.go
+++ b/hlsvod/transcode.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
 	"os/exec"
 	"path"
 	"strings"
@@ -13,6 +14,7 @@ import (
 
 type TranscodeConfig struct {
 	InputFilePath string // Transcoded video input.
+	InputCodec    string // ffprobe codec name, e.g. h264 or hevc.
 	OutputDirPath string // Segments output path.
 	SegmentPrefix string // e.g. prefix-000001.ts
 	SegmentOffset int    // Start segment number.
@@ -69,6 +71,21 @@ func TranscodeSegments(ctx context.Context, ffmpegBinary string, config Transcod
 		}...)
 	}
 
+	// Haswell can decode H.264 in hardware, but not HEVC Main 10. Use full
+	// hardware transcoding for H.264 and software decode plus VAAPI upload for
+	// other codecs. Scaling and H.264 encoding always remain on the GPU.
+	vaapiDevice := os.Getenv("TRANSCODE_VAAPI_DEVICE")
+	hardwareDecode := vaapiDevice != "" && config.InputCodec == "h264"
+	if vaapiDevice != "" {
+		args = append(args, []string{
+			"-init_hw_device", "vaapi=va:" + vaapiDevice,
+			"-filter_hw_device", "va",
+		}...)
+		if hardwareDecode {
+			args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi")
+		}
+	}
+
 	// Input specs
 	args = append(args, []string{
 		"-i", config.InputFilePath, // Input file
@@ -89,14 +106,30 @@ func TranscodeSegments(ctx context.Context, ffmpegBinary string, config Transcod
 			scale = fmt.Sprintf("scale=%d:-2", profile.Width)
 		}
 
-		args = append(args, []string{
-			"-vf", scale,
-			"-c:v", "libx264",
-			"-preset", "faster",
-			"-profile:v", "high",
-			"-level:v", "4.0",
-			"-b:v", fmt.Sprintf("%dk", profile.Bitrate),
-		}...)
+		if vaapiDevice != "" {
+			if profile.Width >= profile.Height {
+				scale = fmt.Sprintf("scale_vaapi=w=-2:h=%d:format=nv12", profile.Height)
+			} else {
+				scale = fmt.Sprintf("scale_vaapi=w=%d:h=-2:format=nv12", profile.Width)
+			}
+			if !hardwareDecode {
+				scale = "format=nv12,hwupload," + scale
+			}
+			args = append(args, []string{
+				"-vf", scale, "-c:v", "h264_vaapi", "-profile:v", "high",
+				"-level:v", "4.0", "-rc_mode", "VBR",
+				"-b:v", fmt.Sprintf("%dk", profile.Bitrate),
+				"-maxrate", fmt.Sprintf("%dk", profile.Bitrate*3/2),
+				"-bufsize", fmt.Sprintf("%dk", profile.Bitrate*2),
+				"-bf", "2", "-compression_level", "0",
+			}...)
+		} else {
+			args = append(args, []string{
+				"-vf", scale, "-c:v", "libx264", "-preset", "faster",
+				"-profile:v", "high", "-level:v", "4.0",
+				"-b:v", fmt.Sprintf("%dk", profile.Bitrate),
+			}...)
+		}
 	}
 
 	// Audio specs