import { Injectable } from '@nestjs/common'; import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored'; import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg'; import _ from 'lodash'; import { Duration } from 'luxon'; import { spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import { Writable } from 'node:stream'; import sharp from 'sharp'; import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants'; import { Exif } from 'src/database'; import { AssetEditActionItem } from 'src/dtos/editing.dto'; import { AacProfile, Av1Profile, ColorMatrix, ColorPrimaries, Colorspace, ColorTransfer, DvProfile, DvSignalCompatibility, H264Profile, HevcProfile, LogLevel, RawExtractedFormat, } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { DecodeToBufferOptions, GenerateThumbhashOptions, GenerateThumbnailOptions, ImageDimensions, ProbeOptions, TranscodeCommand, VideoInfo, VideoPacketInfo, } from 'src/types'; import { handlePromiseError } from 'src/utils/misc'; import { createAffineMatrix } from 'src/utils/transform'; const probe = (input: string, options: string[]): Promise => new Promise((resolve, reject) => ffmpeg.ffprobe(input, options, (error, data) => (error ? reject(error) : resolve(data))), ); const pascalCase = (str: string) => _.upperFirst(_.camelCase(str.toLowerCase())); type ProgressEvent = { frames: number; currentFps: number; currentKbps: number; targetSize: number; timemark: string; percent?: number; }; export type ExtractResult = { buffer: Buffer; format: RawExtractedFormat; }; @Injectable() export class MediaRepository { constructor(private logger: LoggingRepository) { this.logger.setContext(MediaRepository.name); sharp.concurrency(0); sharp.cache({ files: 0 }); } /** * * @param input file path to the input image * @returns ExtractResult if succeeded, or null if failed */ async extract(input: string): Promise { for (const { tag, format } of [ { tag: 'JpgFromRaw2', format: RawExtractedFormat.Jpeg }, { tag: 'JpgFromRaw', format: RawExtractedFormat.Jpeg }, { tag: 'PreviewJXL', format: RawExtractedFormat.Jxl }, { tag: 'PreviewImage', format: RawExtractedFormat.Jpeg }, ]) { try { const buffer = await exiftool.extractBinaryTagToBuffer(tag, input); return { buffer, format }; } catch (error: any) { this.logger.debug(`Could not extract ${tag} buffer from image: ${error}`); } } return null; } async writeExif(tags: Partial, output: string): Promise { try { const tagsToWrite: WriteTags = { ExifImageWidth: tags.exifImageWidth, ExifImageHeight: tags.exifImageHeight, DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()), ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()), TimeZone: tags.timeZone, GPSLatitude: tags.latitude, GPSLongitude: tags.longitude, ProjectionType: tags.projectionType, City: tags.city, Country: tags.country, Make: tags.make, Model: tags.model, LensModel: tags.lensModel, Fnumber: tags.fNumber?.toFixed(1), FocalLength: tags.focalLength?.toFixed(1), ISO: tags.iso, ExposureTime: tags.exposureTime, ProfileDescription: tags.profileDescription, ColorSpace: tags.colorspace, Rating: tags.rating === null ? 0 : tags.rating, // specially convert Orientation to numeric Orientation# for exiftool 'Orientation#': tags.orientation ? Number(tags.orientation) : undefined, }; await exiftool.write(output, tagsToWrite, { ignoreMinorErrors: true, writeArgs: ['-overwrite_original'], }); return true; } catch (error: any) { this.logger.warn(`Could not write exif data to image: ${error.message}`); return false; } } async copyTagGroup(tagGroup: string, source: string, target: string): Promise { try { await exiftool.write( target, {}, { ignoreMinorErrors: true, writeArgs: ['-TagsFromFile', source, `-${tagGroup}:all>${tagGroup}:all`, '-overwrite_original'], }, ); return true; } catch (error: any) { this.logger.warn(`Could not copy tag data to image: ${error.message}`); return false; } } decodeImage(input: string | Buffer, options: DecodeToBufferOptions) { return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true }); } private applyEdits(pipeline: sharp.Sharp, edits: AssetEditActionItem[]): sharp.Sharp { const crop = edits.find((edit) => edit.action === 'crop'); if (crop) { pipeline = pipeline.extract({ left: Math.round(crop.parameters.x), top: Math.round(crop.parameters.y), width: Math.round(crop.parameters.width), height: Math.round(crop.parameters.height), }); } const affineEditOperations = edits.filter((edit) => edit.action !== 'crop'); if (affineEditOperations.length > 0) { const { a, b, c, d } = createAffineMatrix(affineEditOperations); pipeline = pipeline.affine([ [a, b], [c, d], ]); } return pipeline; } async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise { await this.getImageDecodingPipeline(input, options) .toFormat(options.format, { quality: options.quality, // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0', progressive: options.progressive, }) .toFile(output); } private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) { let pipeline = sharp(input, { // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes failOn: options.processInvalidImages ? 'none' : 'error', limitInputPixels: false, raw: options.raw, unlimited: true, }) .pipelineColorspace(options.colorspace === Colorspace.Srgb ? 'srgb' : 'rgb16') .withIccProfile(options.colorspace); if (!options.raw) { const { angle, flip, flop } = options.orientation ? ORIENTATION_TO_SHARP_ROTATION[options.orientation] : {}; pipeline = pipeline.rotate(angle); if (flip) { pipeline = pipeline.flip(); } if (flop) { pipeline = pipeline.flop(); } } if (options.edits && options.edits.length > 0) { pipeline = this.applyEdits(pipeline, options.edits); } if (options.size !== undefined) { pipeline = pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }); } return pipeline; } async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise { const { rgbaToThumbHash } = await import('thumbhash'); const { data, info } = await this.getImageDecodingPipeline(input, { colorspace: options.colorspace, processInvalidImages: options.processInvalidImages, raw: options.raw, edits: options.edits, }) .resize(100, 100, { fit: 'inside', withoutEnlargement: true }) .raw() .ensureAlpha() .toBuffer({ resolveWithObject: true }); return Buffer.from(rgbaToThumbHash(info.width, info.height, data)); } async probe(input: string, options?: ProbeOptions): Promise { const results = await probe(input, options?.countFrames ? ['-count_packets'] : []); // gets frame count quickly: https://stackoverflow.com/a/28376817 return { format: { formatName: results.format.format_name, formatLongName: results.format.format_long_name, duration: this.parseFloat(results.format.duration), bitrate: this.parseInt(results.format.bit_rate), }, videoStreams: results.streams .filter((stream) => stream.codec_type === 'video' && !stream.disposition?.attached_pic) .sort((a, b) => this.compareStreams(a, b)) .map((stream) => { const height = this.parseInt(stream.height); const dar = this.getDar(stream.display_aspect_ratio); return { index: stream.index, height, width: dar ? Math.round(height * dar) : this.parseInt(stream.width), codecName: stream.codec_name === 'h265' ? 'hevc' : (stream.codec_name ?? null), profile: this.parseVideoProfile(stream.codec_name, stream.profile as string | undefined) ?? null, level: this.parseOptionalInt(stream.level), frameCount: this.parseInt(options?.countFrames ? stream.nb_read_packets : stream.nb_frames), frameRate: this.parseFrameRate(stream.avg_frame_rate ?? stream.r_frame_rate), timeBase: this.parseRational(stream.time_base)?.den ?? null, rotation: this.parseInt(stream.rotation), bitrate: this.parseInt(stream.bit_rate), pixelFormat: stream.pix_fmt || 'yuv420p', colorPrimaries: this.parseEnum(ColorPrimaries, stream.color_primaries) ?? ColorPrimaries.Unknown, colorMatrix: this.parseEnum(ColorMatrix, stream.color_space) ?? ColorMatrix.Unknown, colorTransfer: this.parseEnum(ColorTransfer, stream.color_transfer) ?? ColorTransfer.Unknown, dvProfile: this.parseOptionalInt(stream.dv_profile) as DvProfile | null, dvLevel: this.parseOptionalInt(stream.dv_level), dvBlSignalCompatibilityId: this.parseOptionalInt( stream.dv_bl_signal_compatibility_id, ) as DvSignalCompatibility | null, }; }), audioStreams: results.streams .filter((stream) => stream.codec_type === 'audio') .sort((a, b) => this.compareStreams(a, b)) .map((stream) => ({ index: stream.index, codecName: stream.codec_name ?? null, profile: stream.codec_name === 'aac' ? this.parseEnum(AacProfile, stream.profile as string | undefined) : null, bitrate: this.parseInt(stream.bit_rate), })), }; } /** * Needed for accurate segments, especially when remuxing, seeking and/or VFR is involved. * Scanning packets for keyframes in JS is much faster than -skip_frame nokey since it avoids decoding the video. */ probePackets(input: string, streamIndex: number): Promise { const ffprobe = spawn( 'ffprobe', [ '-v', 'error', '-select_streams', String(streamIndex), '-show_entries', 'packet=pts,duration,flags', '-of', 'csv=p=0', input, ], { stdio: ['ignore', 'pipe', 'pipe'] }, ); let totalDuration = 0; const keyframePts: number[] = []; const keyframeAccDuration: number[] = []; const keyframeOwnDuration: number[] = []; const postDiscard: { pts: number; duration: number }[] = []; const parseLine = (line: string) => { if (!line) { return; } const [ptsStr, durationStr, flags] = line.split(',', 3); const pts = Number.parseInt(ptsStr); const duration = Number.parseInt(durationStr); if (Number.isNaN(pts) || Number.isNaN(duration) || !flags) { return; } // Discarded packets don't contribute to packet count, but still contribute to video duration totalDuration += duration; if (flags[1] !== 'D') { postDiscard.push({ pts, duration }); } if (flags[0] === 'K') { keyframePts.push(pts); keyframeAccDuration.push(totalDuration); // VFR content can have variable duration keyframes, // so we need to track their duration separately for accurate segment boundaries. // Non-keyframes are accounted for in totalDuration. keyframeOwnDuration.push(duration); } }; let stderr = ''; let remainder = ''; ffprobe.stderr.setEncoding('utf8'); ffprobe.stderr.on('data', (chunk: string) => (stderr += chunk)); ffprobe.stdout.setEncoding('utf8'); ffprobe.stdout.on('data', (chunk: string) => { const lines = chunk.split('\n'); lines[0] = remainder + lines[0]; remainder = lines.pop() as string; for (const line of lines) { parseLine(line); } }); return new Promise((resolve, reject) => { ffprobe.on('error', reject); ffprobe.on('close', (code) => { if (code !== 0) { return reject(new Error(`ffprobe exited with code ${code}: ${stderr.trim()}`)); } parseLine(remainder); if (postDiscard.length === 0) { return resolve(null); } resolve({ totalDuration, packetCount: postDiscard.length, outputFrames: this.cfrOutputFrames(postDiscard, postDiscard.length / totalDuration), keyframePts, keyframeAccDuration, keyframeOwnDuration, }); }); }); } transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise { if (!options.twoPass) { return new Promise((resolve, reject) => { this.configureFfmpegCall(input, output, options) .on('error', reject) .on('end', () => resolve()) .run(); }); } if (typeof output !== 'string') { throw new TypeError('Two-pass transcoding does not support writing to a stream'); } // two-pass allows for precise control of bitrate at the cost of running twice // recommended for vp9 for better quality and compression return new Promise((resolve, reject) => { // first pass output is not saved as only the .log file is needed this.configureFfmpegCall(input, '/dev/null', options) .addOptions('-pass', '1') .addOptions('-passlogfile', output) .addOptions('-f null') .on('error', reject) .on('end', () => { // second pass this.configureFfmpegCall(input, output, options) .addOptions('-pass', '2') .addOptions('-passlogfile', output) .on('error', reject) .on('end', () => handlePromiseError(fs.unlink(`${output}-0.log`), this.logger)) .on('end', () => handlePromiseError(fs.rm(`${output}-0.log.mbtree`, { force: true }), this.logger)) .on('end', () => resolve()) .run(); }) .run(); }); } async getImageMetadata(input: string | Buffer): Promise { const { width = 0, height = 0, hasAlpha = false } = await sharp(input, { unlimited: true }).metadata(); return { width, height, isTransparent: hasAlpha }; } private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeCommand) { const ffmpegCall = ffmpeg(input, { niceness: 10 }) .inputOptions(options.inputOptions) .outputOptions(options.outputOptions) .output(output) .on('start', (command: string) => this.logger.debug(command)) .on('error', (error, _, stderr) => this.logger.error(stderr || error)); const { frameCount, percentInterval } = options.progress; const frameInterval = Math.ceil(frameCount / (100 / percentInterval)); if (this.logger.isLevelEnabled(LogLevel.Debug) && frameCount && frameInterval) { let lastProgressFrame: number = 0; ffmpegCall.on('progress', (progress: ProgressEvent) => { if (progress.frames - lastProgressFrame < frameInterval) { return; } lastProgressFrame = progress.frames; const percent = ((progress.frames / frameCount) * 100).toFixed(2); const ms = progress.currentFps ? Math.floor((frameCount - progress.frames) / progress.currentFps) * 1000 : 0; const duration = ms ? Duration.fromMillis(ms).rescale().toHuman({ unitDisplay: 'narrow' }) : ''; const outputText = output instanceof Writable ? 'stream' : output.split('/').pop(); this.logger.debug( `Transcoding ${percent}% done${duration ? `, estimated ${duration} remaining` : ''} for output ${outputText}`, ); }); } return ffmpegCall; } private parseInt(value: string | number | undefined): number { return Number.parseInt(value as string) || 0; } private parseFloat(value: string | number | undefined): number { // eslint-disable-next-line unicorn/prefer-number-coercion return Number.parseFloat(value as string) || 0; } private parseOptionalInt(value: string | number | undefined): number | null { const parsed = Number.parseInt(value as string); return Number.isNaN(parsed) ? null : parsed; } private parseEnum>(enumObj: E, value?: string) { return value ? ((enumObj[pascalCase(value)] as Extract | undefined) ?? null) : null; } /** Parse a rational like "60000/1001" or "1/600" into `{ num, den }`. */ private parseRational(value: string | undefined): { num: number; den: number } | null { if (value) { const [num, den = 1] = value.split('/').map(Number); if (num && den) { return { num, den }; } } return null; } private parseFrameRate(value: string | undefined): number | null { const r = this.parseRational(value); return r ? r.num / r.den : null; } private getDar(dar: string | undefined): number { if (dar) { const [darW, darH] = dar.split(':').map(Number); if (darW && darH) { return darW / darH; } } return 0; } private parseVideoProfile(codec?: string, profile?: string) { switch (codec) { case 'h264': { return this.parseEnum(H264Profile, profile); } case 'h265': case 'hevc': { return this.parseEnum(HevcProfile, profile); } case 'av1': { return this.parseEnum(Av1Profile, profile); } default: { return null; } } } private compareStreams(a: FfprobeStream, b: FfprobeStream): number { const d = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0); if (d !== 0) { return d; } return this.parseInt(b.bit_rate) - this.parseInt(a.bit_rate); } /* Ported from https://code.ffmpeg.org/FFmpeg/FFmpeg/src/commit/5c44245878e235ae64fe87fb9877644856d33d1d/fftools/ffmpeg_filter.c * SPDX-License-Identifier: LGPL-2.1-or-later * Copyright (c) FFmpeg authors and contributors — https://ffmpeg.org/ * Modifications: TS port operating on probe-derived packet metadata rather than decoded AVFrames. */ private cfrOutputFrames(packets: { pts: number; duration: number }[], slotsPerTick: number) { packets.sort((a, b) => a.pts - b.pts); const firstPts = packets[0].pts; let outputFrames = 0; let nextPts = 0; const history = [0, 0, 0]; for (const pkt of packets) { const syncIpts = (pkt.pts - firstPts) * slotsPerTick; const duration = pkt.duration * slotsPerTick; let delta0 = syncIpts - nextPts; const delta = delta0 + duration; if (delta0 < 0 && delta > 0) { delta0 = 0; } let nb = 1; let nbPrev = 0; if (delta < -1.1) { nb = 0; } else if (delta > 1.1) { nb = Math.round(delta); if (delta0 > 1.1) { nbPrev = Math.round(delta0 - 0.6); } } outputFrames += nb; nextPts += nb; history[2] = history[1]; history[1] = history[0]; history[0] = nbPrev; } const median = history.sort((a, b) => a - b)[1]; return outputFrames + median; } }