Skip to content

V
vlc
Commit 10f43347 authored by Jean-Paul Saman's avatar Jean-Paul Saman
Browse files
Options

VAAPI-X11: Refactor subpicture handling code. 

Refactor the subpicture handling code to be more consistent.
parent dea831a9
Hide whitespace changes
Inline Side-by-side
Showing with 215 additions and 218 deletions
modules/codec/avcodec/vaapi_x11.c
View file @ 10f43347
......@@ -126,6 +126,10 @@ typedef struct
vlc_array_t subpictures; /* subpicture regions */
} subpicture_cache_t;
static vasubpicture_cache_t *SubpictureCreate(vout_display_t *vd, const subpicture_t *subpicture,
const subpicture_region_t *region, const int flags);
static void SubpictureDestroy(vout_display_t *vd, vlc_va_conn_t *conn, subpicture_cache_t *cache);
static subpicture_cache_t *cache_new(void)
{
subpicture_cache_t *cache = calloc(1, sizeof(subpicture_cache_t));
......@@ -199,48 +203,6 @@ static void subpicture_cache_destroy(vasubpicture_cache_t *cache)
free(cache);
}
/* Clear subpicture cache for this surface and
* release allocated VA API resources. Call this
* function when holding cache->lock */
static void SubpictureCacheClean(vout_display_t *vd, vlc_va_conn_t *conn, subpicture_cache_t *cache)
{
vout_display_sys_t *sys = (vout_display_sys_t *)vd->sys;
#ifdef VAAPI_DEBUG
msg_Dbg(vd, "subpictureCacheClean cache %d", cache->i_id);
#endif
const int num = vlc_array_count(&cache->subpictures);
for (int i = num - 1; i >= 0; i--)
{
vasubpicture_cache_t *subpic;
subpic = (vasubpicture_cache_t *)vlc_array_item_at_index(&cache->subpictures, i);
if (!subpic)
continue;
vlc_array_remove(&cache->subpictures, i);
#ifdef VAAPI_DEBUG
msg_Dbg(vd, "- %d: sub_id %d, image_id %d",
i, subpic->i_id, subpic->image.image_id);
#endif
assert(subpic->i_id != VA_INVALID_ID);
assert(subpic->image.image_id != VA_INVALID_ID);
sys->conn->lock();
vaDestroySubpicture(conn->p_display, subpic->i_id);
vaDestroyImage(conn->p_display, subpic->image.image_id);
subpic->i_id = VA_INVALID_ID;
subpic->image.buf = VA_INVALID_ID;
subpic->image.image_id = VA_INVALID_ID;
sys->conn->unlock();
subpicture_cache_destroy(subpic);
subpic = NULL;
}
}
/* VA API support functions */
static int VASubtitleFourCC(vout_display_t *vd)
{
......@@ -558,7 +520,7 @@ void CloseVaapiX11(vlc_object_t *obj)
vlc_array_remove(&sys->cache, i);
SubpictureCacheClean(vd, sys->conn, cache);
SubpictureDestroy(vd, sys->conn, cache);
cache_destroy(cache);
cache = NULL;
}
......@@ -694,23 +656,156 @@ static int Control (vout_display_t *vd, int query, va_list ap)
}
}
static VASubpictureID SubpictureCreate(vout_display_t *vd, const subpicture_t *subpicture,
const subpicture_region_t *region, VAImage *image, const int flags)
static inline void vaapi_fixup_alpha(uint8_t *p_pixel, size_t i_size)
{
vout_display_sys_t *sys = vd->sys;
for (size_t p = 0; p < i_size; p += 4)
{
/* RGBA */
uint8_t *R = &p_pixel[p];
uint8_t *G = &p_pixel[p+1];
uint8_t *B = &p_pixel[p+2];
uint8_t *A = &p_pixel[p+3];
sys->conn->lock();
if (*A != 255) /* alpha transparancy */
{
*A = 0; /* 0 = transparent, 255 = opaque */
*R = 0;
*G = 0;
*B = 0;
}
}
}
static int CopyPictureToVAImage(vout_display_t *vd, picture_t *pic,
VAImage *image, VAImageFormat *fmt,
const bool fixup_alpha)
{
vout_display_sys_t *sys = vd->sys;
VAStatus status;
VASubpictureID sub_id = VA_INVALID_ID;
status = vaCreateImage(sys->conn->p_display, fmt,
pic->format.i_width, pic->format.i_height, image);
if (status != VA_STATUS_SUCCESS)
return VLC_EGENERIC;
/* Sanity checks */
if ((image->image_id == VA_INVALID_ID) ||
(image->format.fourcc != VA_FOURCC_RGBA) ||
(image->num_planes != (unsigned int) pic->i_planes) ||
(image->height != pic->format.i_height) ||
(image->width != pic->format.i_width) ||
(image->format.bits_per_pixel != pic->format.i_bits_per_pixel))
{
msg_Err(vd, "subpicture region: planes %d/%d, depth %d, width %d/%d, height %d/%d, bpp %d/%d",
image->num_planes, pic->i_planes,
image->format.depth,
image->width, pic->format.i_width,
image->height, pic->format.i_height,
image->format.bits_per_pixel, pic->format.i_bits_per_pixel);
goto out_va_free;
}
assert(image->image_id != VA_INVALID_ID);
assert(image->buf != VA_INVALID_ID);
void *p_base = NULL;
status = vaMapBuffer(sys->conn->p_display, image->buf, &p_base);
if (status != VA_STATUS_SUCCESS)
{
msg_Err(vd, "failed mapping image for copying (%d)", status);
msg_Err(vd, "subpicture region: planes %d/%d, depth %d, width %d/%d, height %d/%d, bpp %d/%d",
image->num_planes, pic->i_planes,
image->format.depth,
image->width, pic->format.i_width,
image->height, pic->format.i_height,
image->format.bits_per_pixel, pic->format.i_bits_per_pixel);
goto out_va_free;
}
assert(p_base); /* should not happen */
for (unsigned int i_plane = 0; i_plane < image->num_planes; i_plane++)
{
/* Does the pitch match? Then just copy it directly */
if (image->pitches[i_plane] == (unsigned int)pic->p[i_plane].i_pitch)
{
vlc_memcpy((uint8_t*)(((uint8_t*)p_base) + image->offsets[i_plane]),
pic->p[i_plane].p_pixels,
pic->p[i_plane].i_pitch * pic->p[i_plane].i_visible_lines);
}
else
{
/* proceed line by line */
uint8_t *p_in = pic->p[i_plane].p_pixels;
uint8_t *p_out = (uint8_t *)(((uint8_t*)p_base) + image->offsets[i_plane]);
int i_line = pic->format.i_y_offset * pic->p[i_plane].i_visible_pitch;
for (; i_line < pic->p[i_plane].i_visible_lines; i_line++)
{
/* NOTE: image->pitch[i_plane] includes image->offset[i_plane], thus
* visible_pitch = image->pitch[i_plane] - image->offset[i_plane] */
size_t i_size = __MIN(image->pitches[i_plane],
(unsigned int)pic->p[i_plane].i_pitch);
vlc_memcpy(&p_out[i_line * image->pitches[i_plane]], &p_in[i_line * pic->p[i_plane].i_pitch], i_size);
/* FIXME: Remove this code when VDPAU and XvBA backends for
* VAAPI properly support alpha or chroma keying in subpictures. */
if (fixup_alpha)
{
uint8_t *p_pixel = p_out + (i_line * image->pitches[i_plane]);
vaapi_fixup_alpha(p_pixel, i_size);
}
}
}
}
status = vaCreateSubpicture(sys->conn->p_display, image->image_id, &sub_id);
status = vaUnmapBuffer(sys->conn->p_display, image->buf);
if (status != VA_STATUS_SUCCESS)
{
sys->conn->unlock();
return VA_INVALID_ID;
msg_Err(vd, "failed unmapping image for copying (%d)", status);
goto out_va_free;
}
return VLC_SUCCESS;
out_va_free:
msg_Err(vd, "failed to copy region into vaImage id=%d", image->image_id);
vaDestroyImage(sys->conn->p_display, image->image_id);
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
return VLC_EGENERIC;
}
/* Create subpicture cache for this subpicture and
* allocate needed VA API resources. Call this
* function when holding cache->lock */
static vasubpicture_cache_t *SubpictureCreate(vout_display_t *vd, const subpicture_t *subpicture,
const subpicture_region_t *region, const int flags)
{
vout_display_sys_t *sys = vd->sys;
sys->conn->lock();
/* Copy pixels from region into VAImage */
vasubpicture_cache_t *vasub_cache = subpicture_cache_new();
if (!vasub_cache)
goto error;
VAImage *image = &vasub_cache->image;
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
int err = CopyPictureToVAImage(vd, region->p_picture, image, &sys->sub_fmt,
(sys->sflags == 0x0) ? true : false);
if (err != VLC_SUCCESS)
{
subpicture_cache_destroy(vasub_cache);
goto error;
}
VAStatus status;
status = vaCreateSubpicture(sys->conn->p_display, image->image_id, &vasub_cache->i_id);
if (status != VA_STATUS_SUCCESS)
goto cleanup;
bool b_alpha = (flags & VA_SUBPICTURE_GLOBAL_ALPHA);
bool b_chroma = (flags & VA_SUBPICTURE_CHROMA_KEYING);
......@@ -719,7 +814,7 @@ static VASubpictureID SubpictureCreate(vout_display_t *vd, const subpicture_t *s
assert(!b_chroma);
float global_alpha = (float)(subpicture->i_alpha * region->i_alpha / 255) / 255;
status = vaSetSubpictureGlobalAlpha(sys->conn->p_display, sub_id, global_alpha);
status = vaSetSubpictureGlobalAlpha(sys->conn->p_display, vasub_cache->i_id, global_alpha);
if (status != VA_STATUS_SUCCESS)
msg_Err(vd, "failed applying alpha value to subpicture");
}
......@@ -735,14 +830,73 @@ static VASubpictureID SubpictureCreate(vout_display_t *vd, const subpicture_t *s
(region->p_picture->format.i_gmask << 16) |
(region->p_picture->format.i_bmask << 8) | 0;
status = vaSetSubpictureChromakey(sys->conn->p_display, sub_id,
status = vaSetSubpictureChromakey(sys->conn->p_display, vasub_cache->i_id,
min, max, mask);
if (status != VA_STATUS_SUCCESS)
msg_Err(vd, "failed applying chroma valuesto subpicture");
}
vasub_cache->rect.i_x = region->i_x;
vasub_cache->rect.i_y = region->i_y;
vasub_cache->rect.i_width = region->fmt.i_visible_width;
vasub_cache->rect.i_height = region->fmt.i_visible_height;
sys->conn->unlock();
return sub_id;
return vasub_cache;
cleanup:
vaDestroyImage(sys->conn->p_display, image->image_id);
vasub_cache->i_id = VA_INVALID_ID;
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
subpicture_cache_destroy(vasub_cache);
vasub_cache = NULL;
error:
sys->conn->unlock();
return NULL;
}
/* Destroy subpicture cache for this subpicture and
* release allocated VA API resources. Call this
* function when holding cache->lock */
static void SubpictureDestroy(vout_display_t *vd, vlc_va_conn_t *conn, subpicture_cache_t *cache)
{
vout_display_sys_t *sys = (vout_display_sys_t *)vd->sys;
#ifdef VAAPI_DEBUG
msg_Dbg(vd, "SubpictureDestroy cache %d", cache->i_id);
#endif
const int num = vlc_array_count(&cache->subpictures);
for (int i = num - 1; i >= 0; i--)
{
vasubpicture_cache_t *subpic;
subpic = (vasubpicture_cache_t *)vlc_array_item_at_index(&cache->subpictures, i);
if (!subpic)
continue;
vlc_array_remove(&cache->subpictures, i);
#ifdef VAAPI_DEBUG
msg_Dbg(vd, "- %d: sub_id %d, image_id %d",
i, subpic->i_id, subpic->image.image_id);
#endif
assert(subpic->i_id != VA_INVALID_ID);
assert(subpic->image.image_id != VA_INVALID_ID);
sys->conn->lock();
vaDestroySubpicture(conn->p_display, subpic->i_id);
vaDestroyImage(conn->p_display, subpic->image.image_id);
subpic->i_id = VA_INVALID_ID;
subpic->image.buf = VA_INVALID_ID;
subpic->image.image_id = VA_INVALID_ID;
sys->conn->unlock();
subpicture_cache_destroy(subpic);
subpic = NULL;
}
}
static int SubpictureRegionsLink(vout_display_t *vd, const picture_t *picture)
......@@ -895,7 +1049,7 @@ static void SubpictureRegionsUnlink(vout_display_t *vd, picture_t *picture)
{
int index = vlc_array_index_of_item(&sys->cache, (void*)cache);
vlc_array_remove(&sys->cache, index);
SubpictureCacheClean(vd, sys->conn, cache);
SubpictureDestroy(vd, sys->conn, cache);
cache_destroy(cache);
cache = NULL;
}
......@@ -906,131 +1060,6 @@ error:
vlc_mutex_unlock(&sys->cache_lock);
}
static inline void vaapi_fixup_alpha(uint8_t *p_pixel, size_t i_size)
{
for (size_t p = 0; p < i_size; p += 4)
{
/* RGBA */
uint8_t *R = &p_pixel[p];
uint8_t *G = &p_pixel[p+1];
uint8_t *B = &p_pixel[p+2];
uint8_t *A = &p_pixel[p+3];
if (*A != 255) /* alpha transparancy */
{
*A = 0; /* 0 = transparent, 255 = opaque */
*R = 0;
*G = 0;
*B = 0;
}
}
}
static int CopyPictureToVAImage(vout_display_t *vd, picture_t *pic,
VAImage *image, VAImageFormat *fmt,
const bool fixup_alpha)
{
vout_display_sys_t *sys = vd->sys;
sys->conn->lock();
VAStatus status;
status = vaCreateImage(sys->conn->p_display, fmt,
pic->format.i_width, pic->format.i_height, image);
if (status != VA_STATUS_SUCCESS)
{
sys->conn->unlock();
return VLC_EGENERIC;
}
/* Sanity checks */
if ((image->image_id == VA_INVALID_ID) ||
(image->format.fourcc != VA_FOURCC_RGBA) ||
(image->num_planes != (unsigned int) pic->i_planes) ||
(image->height != pic->format.i_height) ||
(image->width != pic->format.i_width) ||
(image->format.bits_per_pixel != pic->format.i_bits_per_pixel))
{
msg_Err(vd, "subpicture region: planes %d/%d, depth %d, width %d/%d, height %d/%d, bpp %d/%d",
image->num_planes, pic->i_planes,
image->format.depth,
image->width, pic->format.i_width,
image->height, pic->format.i_height,
image->format.bits_per_pixel, pic->format.i_bits_per_pixel);
goto out_va_free;
}
assert(image->image_id != VA_INVALID_ID);
assert(image->buf != VA_INVALID_ID);
void *p_base = NULL;
status = vaMapBuffer(sys->conn->p_display, image->buf, &p_base);
if (status != VA_STATUS_SUCCESS)
{
msg_Err(vd, "failed mapping image for copying (%d)", status);
msg_Err(vd, "subpicture region: planes %d/%d, depth %d, width %d/%d, height %d/%d, bpp %d/%d",
image->num_planes, pic->i_planes,
image->format.depth,
image->width, pic->format.i_width,
image->height, pic->format.i_height,
image->format.bits_per_pixel, pic->format.i_bits_per_pixel);
goto out_va_free;
}
assert(p_base); /* should not happen */
for (unsigned int i_plane = 0; i_plane < image->num_planes; i_plane++)
{
/* Does the pitch match? Then just copy it directly */
if (image->pitches[i_plane] == (unsigned int)pic->p[i_plane].i_pitch)
{
vlc_memcpy((uint8_t*)(((uint8_t*)p_base) + image->offsets[i_plane]),
pic->p[i_plane].p_pixels,
pic->p[i_plane].i_pitch * pic->p[i_plane].i_visible_lines);
}
else
{
/* proceed line by line */
uint8_t *p_in = pic->p[i_plane].p_pixels;
uint8_t *p_out = (uint8_t *)(((uint8_t*)p_base) + image->offsets[i_plane]);
int i_line = pic->format.i_y_offset * pic->p[i_plane].i_visible_pitch;
for (; i_line < pic->p[i_plane].i_visible_lines; i_line++)
{
/* NOTE: image->pitch[i_plane] includes image->offset[i_plane], thus
* visible_pitch = image->pitch[i_plane] - image->offset[i_plane] */
size_t i_size = __MIN(image->pitches[i_plane],
(unsigned int)pic->p[i_plane].i_pitch);
vlc_memcpy(&p_out[i_line * image->pitches[i_plane]], &p_in[i_line * pic->p[i_plane].i_pitch], i_size);
/* FIXME: Remove this code when VDPAU and XvBA backends for
* VAAPI properly support alpha or chroma keying in subpictures. */
if (fixup_alpha)
{
uint8_t *p_pixel = p_out + (i_line * image->pitches[i_plane]);
vaapi_fixup_alpha(p_pixel, i_size);
}
}
}
}
status = vaUnmapBuffer(sys->conn->p_display, image->buf);
if (status != VA_STATUS_SUCCESS)
{
msg_Err(vd, "failed unmapping image for copying (%d)", status);
goto out_va_free;
}
sys->conn->unlock();
return VLC_SUCCESS;
out_va_free:
msg_Err(vd, "failed to copy region into vaImage id=%d", image->image_id);
vaDestroyImage(sys->conn->p_display, image->image_id);
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
sys->conn->unlock();
return VLC_EGENERIC;
}
static int RenderDirectSubpicture(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
{
vout_display_sys_t *sys = vd->sys;
......@@ -1058,43 +1087,11 @@ static int RenderDirectSubpicture(vout_display_t *vd, picture_t *picture, subpic
assert(region->p_picture->format.i_width > 0);
assert(region->p_picture->format.i_height > 0);
/* Copy pixels from region into VAImage */
vasubpicture_cache_t *vasub_cache = subpicture_cache_new();
if (!vasub_cache)
goto cleanup;
VAImage *image = &vasub_cache->image;
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
int err = CopyPictureToVAImage(vd, region->p_picture, image, &sys->sub_fmt,
(sys->sflags == 0x0) ? true : false);
if (err != VLC_SUCCESS)
{
free(vasub_cache);
goto cleanup;
}
/* Create Subpicture */
vasub_cache->i_id = SubpictureCreate(vd, subpicture, region,
image, sys->sflags);
if (vasub_cache->i_id == VA_INVALID_ID)
{
sys->conn->lock();
vaDestroyImage(sys->conn->p_display, image->image_id);
vasub_cache->i_id = VA_INVALID_ID;
image->image_id = VA_INVALID_ID;
image->buf = VA_INVALID_ID;
sys->conn->unlock();
subpicture_cache_destroy(vasub_cache);
vasub_cache = NULL;
vasubpicture_cache_t *vasub_cache;
vasub_cache = SubpictureCreate(vd, subpicture, region, sys->sflags);
if (vasub_cache == NULL)
goto cleanup;
}
vasub_cache->rect.i_x = region->i_x;
vasub_cache->rect.i_y = region->i_y;
vasub_cache->rect.i_width = region->fmt.i_visible_width;
vasub_cache->rect.i_height = region->fmt.i_visible_height;
#ifdef VAAPI_DEBUG
msg_Dbg(vd, "+ %d: subid %d", i_sub, vasub_cache->i_id);
......@@ -1119,7 +1116,7 @@ cleanup:
surface->i_cache = VA_INVALID_ID;
msg_Err(vd, "failed creating subpicture cache %d for surface %d",
cache->i_id, surface->i_id);
SubpictureCacheClean(vd, sys->conn, cache);
SubpictureDestroy(vd, sys->conn, cache);
cache_destroy(cache);
cache = NULL;
return VLC_EGENERIC;
......@@ -1274,7 +1271,7 @@ static void PictureRelease(picture_t *picture)
{
int index = vlc_array_index_of_item(&vd->sys->cache, (void*)cache);
vlc_array_remove(&vd->sys->cache, index);
SubpictureCacheClean(vd, vd->sys->conn, cache);
SubpictureDestroy(vd, vd->sys->conn, cache);
cache_destroy(cache);
cache = NULL;
}
......@@ -1394,7 +1391,7 @@ static void PictureUnlock(picture_t *picture)
{
int index = vlc_array_index_of_item(&sys->cache, (void*)cache);
vlc_array_remove(&sys->cache, index);
SubpictureCacheClean(vd, sys->conn, cache);
SubpictureDestroy(vd, sys->conn, cache);
cache_destroy(cache);
cache = NULL;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment