Skip to content

fix(avfoundation): composite every OSD object into one overlay #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
demrom wants to merge 1 commit into
base: avfoundation-ao-tvos
Choose a base branch
from
Open

fix(avfoundation): composite every OSD object into one overlay #4

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 81 additions & 106 deletions Sources/BuildScripts/patch/libmpv/0004-avfoundation-video-output.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ Subject: [PATCH] Add avfoundation support
meson.build | 8 +
meson.options | 1 +
video/out/vo.c | 8 +
video/out/vo_avfoundation.m | 1281 +++++++++++++++++++++++++++++++++++
4 files changed, 1298 insertions(+)
video/out/vo_avfoundation.m | 1256 +++++++++++++++++++++++++++++++++++
4 files changed, 1273 insertions(+)
create mode 100644 video/out/vo_avfoundation.m

diff --git a/meson.build b/meson.build
Expand Down Expand Up @@ -70,10 +70,10 @@ index c9cde30478..64484b1b71 100644

diff --git a/video/out/vo_avfoundation.m b/video/out/vo_avfoundation.m
new file mode 100644
index 0000000000..8155d7a0e8
index 0000000000..f6b2cc7995
--- /dev/null
+++ b/video/out/vo_avfoundation.m
@@ -0,0 +1,1281 @@
@@ -0,0 +1,1256 @@
+/*
+ * This file is part of mpv.
+ *
Expand Down Expand Up @@ -159,9 +159,6 @@ index 0000000000..8155d7a0e8
+ // Frame counter for periodic maintenance
+ int frame_count;
+
+ // Track if OSD callback was invoked this frame
+ bool osd_callback_invoked;
+
+ // Display resolution for scaling subtitles (0 = no upscaling)
+ int display_height;
+};
Expand Down Expand Up @@ -379,22 +376,25 @@ index 0000000000..8155d7a0e8
+
+// Calculate tight bounding box around all subtitle parts
+// Returns rect in CoreGraphics coordinates (origin at bottom-left)
+static CGRect calculate_subtitle_bounds(struct sub_bitmaps *imgs, int frame_w, int frame_h)
+static CGRect calculate_subtitle_bounds(struct sub_bitmap_list *list, int frame_w, int frame_h)
+{
+ if (!imgs || imgs->num_parts == 0)
+ if (!list || list->num_items == 0)
+ return CGRectZero;
+
+ int minX = INT_MAX, minY = INT_MAX, maxX = 0, maxY = 0;
+
+ for (int i = 0; i < imgs->num_parts; i++) {
+ struct sub_bitmap *sb = &imgs->parts[i];
+ if (!sb->bitmap || sb->w <= 0 || sb->h <= 0)
+ continue;
+
+ minX = MPMIN(minX, sb->x);
+ minY = MPMIN(minY, sb->y);
+ maxX = MPMAX(maxX, sb->x + sb->dw);
+ maxY = MPMAX(maxY, sb->y + sb->dh);
+ for (int n = 0; n < list->num_items; n++) {
+ struct sub_bitmaps *imgs = list->items[n];
+ for (int i = 0; i < imgs->num_parts; i++) {
+ struct sub_bitmap *sb = &imgs->parts[i];
+ if (!sb->bitmap || sb->w <= 0 || sb->h <= 0)
+ continue;
+
+ minX = MPMIN(minX, sb->x);
+ minY = MPMIN(minY, sb->y);
+ maxX = MPMAX(maxX, sb->x + sb->dw);
+ maxY = MPMAX(maxY, sb->y + sb->dh);
+ }
+ }
+
+ if (minX >= maxX || minY >= maxY)
Expand All @@ -416,11 +416,11 @@ index 0000000000..8155d7a0e8
+// Render subtitle bitmap to CGImage
+// In composite mode: uses dirty rect optimization (smaller buffer, positioned in composite_frame)
+// In CALayer mode: renders full frame (CALayer displays as overlay)
+static CGImageRef render_subtitle_image(struct vo *vo, struct sub_bitmaps *imgs)
+static CGImageRef render_subtitle_image(struct vo *vo, struct sub_bitmap_list *list)
+{
+ struct priv *p = vo->priv;
+
+ if (!imgs || imgs->num_parts == 0)
+ if (!list || list->num_items == 0)
+ return NULL;
+
+ int frame_w = p->osd_w;
Expand All @@ -439,7 +439,7 @@ index 0000000000..8155d7a0e8
+ if (p->composite_osd) {
+ // Composite mode: use dirty rect optimization
+ // Only allocates memory for the subtitle bounding box, not full frame
+ CGRect bounds = calculate_subtitle_bounds(imgs, frame_w, frame_h);
+ CGRect bounds = calculate_subtitle_bounds(list, frame_w, frame_h);
+ if (CGRectIsEmpty(bounds)) {
+ CGColorSpaceRelease(colorSpace);
+ return NULL;
Expand Down Expand Up @@ -502,62 +502,66 @@ index 0000000000..8155d7a0e8
+ CGContextClearRect(context, CGRectMake(0, 0, subW, subH));
+
+ // Create colorspace once for all BGRA parts
+ CGColorSpaceRef partColorSpace = (imgs->format == SUBBITMAP_BGRA)
+ ? CGColorSpaceCreateDeviceRGB() : NULL;
+
+ for (int i = 0; i < imgs->num_parts; i++) {
+ struct sub_bitmap *sb = &imgs->parts[i];
+
+ if (!sb->bitmap || sb->w <= 0 || sb->h <= 0)
+ continue;
+
+ CGDataProviderRef provider = CGDataProviderCreateWithData(
+ NULL, sb->bitmap, sb->stride * sb->h, NULL);
+ CGColorSpaceRef partColorSpace = CGColorSpaceCreateDeviceRGB();
+
+ // osd_render returns separate objects for primary, secondary and other OSD.
+ // Draw all of them in list order into the same union-bounds surface.
+ for (int n = 0; n < list->num_items; n++) {
+ struct sub_bitmaps *imgs = list->items[n];
+ for (int i = 0; i < imgs->num_parts; i++) {
+ struct sub_bitmap *sb = &imgs->parts[i];
+
+ if (!provider)
+ continue;
+
+ // Calculate position relative to bounding box (not full frame)
+ // sb->x, sb->y are in mpv coords (origin top-left)
+ int relX = sb->x - offsetX;
+ int relY = sb->y - offsetY_mpv;
+
+ // Convert to CG coords (origin bottom-left of bounding box)
+ CGRect destRect = CGRectMake(relX, subH - relY - sb->dh, sb->dw, sb->dh);
+ if (!sb->bitmap || sb->w <= 0 || sb->h <= 0)
+ continue;
+
+ if (imgs->format == SUBBITMAP_BGRA) {
+ CGImageRef partImage = CGImageCreate(
+ sb->w, sb->h, 8, 32, sb->stride, partColorSpace,
+ kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little,
+ provider, NULL, false, kCGRenderingIntentDefault);
+ CGDataProviderRef provider = CGDataProviderCreateWithData(
+ NULL, sb->bitmap, sb->stride * sb->h, NULL);
+
+ if (!provider)
+ continue;
+
+ // Calculate position relative to bounding box (not full frame)
+ // sb->x, sb->y are in mpv coords (origin top-left)
+ int relX = sb->x - offsetX;
+ int relY = sb->y - offsetY_mpv;
+
+ // Convert to CG coords (origin bottom-left of bounding box)
+ CGRect destRect = CGRectMake(relX, subH - relY - sb->dh, sb->dw, sb->dh);
+
+ if (imgs->format == SUBBITMAP_BGRA) {
+ CGImageRef partImage = CGImageCreate(
+ sb->w, sb->h, 8, 32, sb->stride, partColorSpace,
+ kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little,
+ provider, NULL, false, kCGRenderingIntentDefault);
+
+ if (partImage) {
+ CGContextDrawImage(context, destRect, partImage);
+ CGImageRelease(partImage);
+ }
+ } else if (imgs->format == SUBBITMAP_LIBASS) {
+ CGFloat decode[] = {1, 0};
+ CGImageRef mask = CGImageMaskCreate(
+ sb->w, sb->h, 8, 8, sb->stride, provider, decode, false);
+ if (partImage) {
+ CGContextDrawImage(context, destRect, partImage);
+ CGImageRelease(partImage);
+ }
+ } else if (imgs->format == SUBBITMAP_LIBASS) {
+ CGFloat decode[] = {1, 0};
+ CGImageRef mask = CGImageMaskCreate(
+ sb->w, sb->h, 8, 8, sb->stride, provider, decode, false);
+
+ if (mask) {
+ CGContextSaveGState(context);
+ CGContextClipToMask(context, destRect, mask);
+ if (mask) {
+ CGContextSaveGState(context);
+ CGContextClipToMask(context, destRect, mask);
+
+ uint32_t c = sb->libass.color;
+ float r = ((c >> 24) & 0xFF) / 255.0f;
+ float g = ((c >> 16) & 0xFF) / 255.0f;
+ float b = ((c >> 8) & 0xFF) / 255.0f;
+ float a = 1.0f - ((c >> 0) & 0xFF) / 255.0f;
+ uint32_t c = sb->libass.color;
+ float r = ((c >> 24) & 0xFF) / 255.0f;
+ float g = ((c >> 16) & 0xFF) / 255.0f;
+ float b = ((c >> 8) & 0xFF) / 255.0f;
+ float a = 1.0f - ((c >> 0) & 0xFF) / 255.0f;
+
+ CGContextSetRGBFillColor(context, r, g, b, a);
+ CGContextFillRect(context, destRect);
+ CGContextRestoreGState(context);
+ CGImageRelease(mask);
+ CGContextSetRGBFillColor(context, r, g, b, a);
+ CGContextFillRect(context, destRect);
+ CGContextRestoreGState(context);
+ CGImageRelease(mask);
+ }
+ }
+ }
+
+ CGDataProviderRelease(provider);
+ CGDataProviderRelease(provider);
+ }
+ }
+
+ // (debug indicator removed)
Expand All @@ -574,17 +578,13 @@ index 0000000000..8155d7a0e8
+ return result;
+}
+
+// Callback for osd_draw - renders subtitle bitmaps
+static void draw_osd_cb(void *ctx, struct sub_bitmaps *imgs)
+// Publish one complete OSD image, never one image per subtitle object.
+static void update_osd_image(struct vo *vo, struct sub_bitmap_list *list)
+{
+ struct vo *vo = ctx;
+ struct priv *p = vo->priv;
+
+ // Mark that callback was invoked this frame
+ p->osd_callback_invoked = true;
+
+ // Clear subtitles when there are none
+ if (!imgs || imgs->num_parts == 0) {
+ if (!list || list->num_items == 0) {
+ if (p->has_subtitle) {
+ p->has_subtitle = false;
+ p->osd_change_id = 0; // Reset change_id so next subtitle renders
Expand All @@ -611,10 +611,10 @@ index 0000000000..8155d7a0e8
+
+ // Always redraw to reflect live changes (e.g. scaling) even when paused.
+ // Disable change_id-based caching per request to ensure immediate updates.
+ p->osd_change_id = imgs->change_id;
+ p->osd_change_id = list->change_id;
+ p->has_subtitle = true;
+
+ CGImageRef subtitleImage = render_subtitle_image(vo, imgs);
+ CGImageRef subtitleImage = render_subtitle_image(vo, list);
+ if (!subtitleImage)
+ return;
+
Expand Down Expand Up @@ -701,34 +701,9 @@ index 0000000000..8155d7a0e8
+ [SUBBITMAP_LIBASS] = true,
+ };
+
+ // Reset flag before calling osd_draw
+ p->osd_callback_invoked = false;
+
+ osd_draw(vo->osd, osd_res, pts, 0, formats, draw_osd_cb, vo);
+
+ // If callback wasn't invoked, there are no subtitles for this PTS
+ // Clear any cached subtitle state
+ if (!p->osd_callback_invoked && p->has_subtitle) {
+ p->has_subtitle = false;
+ p->osd_change_id = 0;
+ if (p->composite_osd) {
+ if (p->cachedSubtitleImage) {
+ CGImageRelease(p->cachedSubtitleImage);
+ p->cachedSubtitleImage = NULL;
+ }
+ if (p->cachedSubtitleCIImage) {
+ CFRelease(p->cachedSubtitleCIImage);
+ p->cachedSubtitleCIImage = NULL;
+ }
+ } else if (p->osdLayer) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [CATransaction begin];
+ [CATransaction setDisableActions:YES];
+ p->osdLayer.contents = nil;
+ [CATransaction commit];
+ });
+ }
+ }
+ struct sub_bitmap_list *list = osd_render(vo->osd, osd_res, pts, 0, formats);
+ update_osd_image(vo, list);
+ talloc_free(list);
+}
+
+// Composite subtitle onto video frame using Core Image (GPU) - HDR compatible
Expand Down