fix(server): incorrect scaling of rotated videos when transcoding (#31473)

server/src/services/media.service.spec.ts

@@ -2182,6 +2182,45 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should scale horizontally when a vertical video is rotated to landscape', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({
+        ...asset,
+        ...probeStub.videoStreamRotatedHorizontal2160p,
+      });
+      mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          inputOptions: expect.any(Array),
+          outputOptions: expect.arrayContaining([expect.stringMatching(/scale(_.+)?=-2:720/)]),
+          twoPass: false,
+        }),
+      );
+    });
+
+    it.each([
+      { accel: TranscodeHardwareAcceleration.Nvenc, scaling: 'scale_cuda=-2:720' },
+      { accel: TranscodeHardwareAcceleration.Qsv, scaling: 'scale_qsv=-1:720' },
+      { accel: TranscodeHardwareAcceleration.Vaapi, scaling: 'scale_vaapi=-2:720' },
+      { accel: TranscodeHardwareAcceleration.Rkmpp, scaling: 'scale_rkrga=-2:720' },
+    ])('should scale rotated video by its stored dimensions when decoding with $accel', async ({ accel, scaling }) => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVertical2160p });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel, accelDecode: true, transcode: TranscodePolicy.Optimal },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-noautorotate']),
+          outputOptions: expect.arrayContaining([expect.stringContaining(scaling)]),
+        }),
+      );
+    });
+
     it('should always scale video if height is uneven', async () => {
       mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamOddHeight });
       mocks.systemMetadata.get.mockResolvedValue({

server/src/utils/media.ts

@@ -31,8 +31,9 @@ import {
 
 export const isVideoRotated = (videoStream: VideoStreamInfo): boolean => Math.abs(videoStream.rotation) === 90;
 
+// whether the video is portrait once its rotation is applied
 export const isVideoVertical = (videoStream: VideoStreamInfo): boolean =>
-  videoStream.height > videoStream.width || isVideoRotated(videoStream);
+  videoStream.height > videoStream.width !== isVideoRotated(videoStream);
 
 export const getOutputSize = (videoStream: VideoStreamInfo, targetRes: number) => {
   const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
@@ -364,7 +365,12 @@ export class BaseConfig implements VideoCodecSWConfig {
 
   getScaling(videoStream: VideoStreamInfo, mult = 2) {
     const targetResolution = this.getTargetResolution(videoStream);
-    return isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+    return this.isFrameVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+  }
+
+  // frames reach the filters already rotated, unless decoding with -noautorotate
+  isFrameVertical(videoStream: VideoStreamInfo) {
+    return isVideoVertical(videoStream);
   }
 
   isBitrateConstrained() {
@@ -741,6 +747,11 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
     return ['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda', '-noautorotate', ...this.getInputThreadOptions()];
   }
 
+  // -noautorotate keeps frames in their stored orientation
+  isFrameVertical(videoStream: VideoStreamInfo) {
+    return videoStream.height > videoStream.width;
+  }
+
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
     const tonemapOptions = this.getToneMapping(videoStream);
@@ -878,6 +889,11 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
     ];
   }
 
+  // -noautorotate keeps frames in their stored orientation
+  isFrameVertical(videoStream: VideoStreamInfo) {
+    return videoStream.height > videoStream.width;
+  }
+
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
     const tonemapOptions = this.getToneMapping(videoStream);
@@ -1000,6 +1016,11 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
     ];
   }
 
+  // -noautorotate keeps frames in their stored orientation
+  isFrameVertical(videoStream: VideoStreamInfo) {
+    return videoStream.height > videoStream.width;
+  }
+
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
     const tonemapOptions = this.getToneMapping(videoStream);
@@ -1087,6 +1108,11 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
     return ['-hwaccel', 'rkmpp', '-hwaccel_output_format', 'drm_prime', '-afbc', 'rga', '-noautorotate'];
   }
 
+  // -noautorotate keeps frames in their stored orientation
+  isFrameVertical(videoStream: VideoStreamInfo) {
+    return videoStream.height > videoStream.width;
+  }
+
   getFilterOptions(videoStream: VideoStreamInfo) {
     if (this.shouldToneMap(videoStream)) {
       const { primaries, transfer, matrix } = this.getColors();

server/test/fixtures/media.stub.ts

@@ -288,6 +288,31 @@ export const videoInfoStub = {
       },
     ],
   }),
+  videoStreamRotatedHorizontal2160p: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
+      {
+        index: 0,
+        height: 3840,
+        width: 2160,
+        codecName: 'h264',
+        frameCount: 100,
+        rotation: 90,
+        bitrate: 0,
+        colorPrimaries: ColorPrimaries.Bt709,
+        colorTransfer: ColorTransfer.Bt709,
+        colorMatrix: ColorMatrix.Bt709,
+        pixelFormat: 'yuv420p',
+        frameRate: 60,
+        timeBase: 600,
+        profile: H264Profile.High,
+        level: null,
+        dvBlSignalCompatibilityId: null,
+        dvLevel: null,
+        dvProfile: null,
+      },
+    ],
+  }),
   videoStreamOddHeight: Object.freeze<VideoInfo>({
     ...probeStubDefault,
     videoStreams: [