vmem: set the video format via callback

modules/video_output/vmem.c

@@ -86,14 +86,24 @@ struct picture_sys_t {
     void *id;
 };
 
+/* NOTE: the callback prototypes must match those of LibVLC */
 struct vout_display_sys_t {
     picture_pool_t *pool;
+    unsigned        count;
+
+    void *opaque;
     void *(*lock)(void *sys, void **plane);
     void (*unlock)(void *sys, void *id, void *const *plane);
     void (*display)(void *sys, void *id);
-    void *opaque;
+    void (*cleanup)(void *sys);
+
+    unsigned pitches[PICTURE_PLANE_MAX];
+    unsigned lines[PICTURE_PLANE_MAX];
 };
 
+typedef unsigned (*vlc_format_cb)(void **, char *, unsigned *, unsigned *,
+                                  unsigned *, unsigned *);
+
 static picture_pool_t *Pool  (vout_display_t *, unsigned);
 static void           Display(vout_display_t *, picture_t *);
 static int            Control(vout_display_t *, int, va_list);
@@ -110,25 +120,71 @@ static void           Unlock(picture_t *);
 static int Open(vlc_object_t *object)
 {
     vout_display_t *vd = (vout_display_t *)object;
+    vout_display_sys_t *sys = malloc(sizeof(*sys));
+    if (unlikely(!sys))
+        return VLC_ENOMEM;
 
-    /* */
-    char *chroma_format = var_InheritString(vd, "vmem-chroma");
-    const vlc_fourcc_t chroma = vlc_fourcc_GetCodecFromString(VIDEO_ES, chroma_format);
-    free(chroma_format);
-    if (!chroma) {
-        msg_Err(vd, "vmem-chroma should be 4 characters long");
+    /* Get the callbacks */
+    vlc_format_cb setup = var_InheritAddress(vd, "vmem-setup");
+
+    sys->lock = var_InheritAddress(vd, "vmem-lock");
+    if (sys->lock == NULL) {
+        msg_Err(vd, "missing lock callback");
+        free(sys);
         return VLC_EGENERIC;
     }
+    sys->unlock = var_InheritAddress(vd, "vmem-unlock");
+    sys->display = var_InheritAddress(vd, "vmem-display");
+    sys->cleanup = var_InheritAddress(vd, "vmem-cleanup");
+    sys->opaque = var_InheritAddress(vd, "vmem-data");
+    sys->pool = NULL;
 
-    /* */
+    /* Define the video format */
     video_format_t fmt = vd->fmt;
 
-    fmt.i_chroma = chroma;
-    fmt.i_width  = var_InheritInteger(vd, "vmem-width");
-    fmt.i_height = var_InheritInteger(vd, "vmem-height");
+    if (setup != NULL) {
+        char chroma[5];
+
+        memcpy(chroma, &fmt.i_chroma, 4);
+        chroma[4] = '\0';
+        memset(sys->pitches, 0, sizeof(sys->pitches));
+        memset(sys->lines, 0, sizeof(sys->lines));
+
+        sys->count = setup(&sys->opaque, chroma, &fmt.i_width, &fmt.i_height,
+                           sys->pitches, sys->lines);
+        if (sys->count == 0) {
+            msg_Err(vd, "video format setup failure (no pictures)");
+            free(sys);
+            return VLC_EGENERIC;
+        }
+        fmt.i_chroma = vlc_fourcc_GetCodecFromString(VIDEO_ES, chroma);
+
+    } else {
+        char *chroma = var_InheritString(vd, "vmem-chroma");
+        fmt.i_chroma = vlc_fourcc_GetCodecFromString(VIDEO_ES, chroma);
+        free(chroma);
+
+        fmt.i_width  = var_InheritInteger(vd, "vmem-width");
+        fmt.i_height = var_InheritInteger(vd, "vmem-height");
+        sys->pitches[0] = var_InheritInteger(vd, "vmem-pitch");
+        sys->lines[0] = fmt.i_height;
+        for (size_t i = 1; i < PICTURE_PLANE_MAX; i++)
+        {
+            sys->pitches[i] = sys->pitches[0];
+            sys->lines[i] = sys->lines[0];
+        }
+        sys->count = 1;
+        sys->cleanup = NULL;
+    }
+
+    if (!fmt.i_chroma) {
+        msg_Err(vd, "vmem-chroma should be 4 characters long");
+        free(sys);
+        return VLC_EGENERIC;
+    }
 
     /* Define the bitmasks */
-    switch (chroma)
+    switch (fmt.i_chroma)
     {
     case VLC_CODEC_RGB15:
         fmt.i_rmask = 0x001f;
@@ -153,64 +209,12 @@ static int Open(vlc_object_t *object)
         break;
     }
 
-    /* */
-    vout_display_sys_t *sys;
-    vd->sys = sys = calloc(1, sizeof(*sys));
-    if (unlikely(!sys))
-        return VLC_ENOMEM;
-
-    sys->lock = var_InheritAddress(vd, "vmem-lock");
-    if (sys->lock == NULL) {
-        msg_Err(vd, "Invalid lock callback");
-        free(sys);
-        return VLC_EGENERIC;
-    }
-    sys->unlock = var_InheritAddress(vd, "vmem-unlock");
-    sys->display = var_InheritAddress(vd, "vmem-display");
-    sys->opaque = var_InheritAddress(vd, "vmem-data");
-
-    /* */
-    const int pitch = var_InheritInteger(vd, "vmem-pitch");
-    picture_resource_t rsc;
-    rsc.p_sys = malloc(sizeof(*rsc.p_sys));
-    if(unlikely(!rsc.p_sys)) {
-        free(sys);
-        return VLC_ENOMEM;
-    }
-    rsc.p_sys->sys = sys;
-    rsc.p_sys->id = NULL;
-    for (int i = 0; i < PICTURE_PLANE_MAX; i++) {
-        /* vmem-lock is responsible for the allocation */
-        rsc.p[i].p_pixels = NULL;
-        rsc.p[i].i_lines  = fmt.i_height;
-        rsc.p[i].i_pitch  = pitch;
-    }
-    picture_t *picture = picture_NewFromResource(&fmt, &rsc);
-    if (!picture) {
-        free(rsc.p_sys);
-        free(sys);
-        return VLC_EGENERIC;
-    }
-
-    /* */
-    picture_pool_configuration_t pool;
-    memset(&pool, 0, sizeof(pool));
-    pool.picture_count = 1;
-    pool.picture       = &picture;
-    pool.lock          = Lock;
-    pool.unlock        = Unlock;
-    sys->pool = picture_pool_NewExtended(&pool);
-    if (!sys->pool) {
-        picture_Release(picture);
-        free(sys);
-        return VLC_EGENERIC;
-    }
-
     /* */
     vout_display_info_t info = vd->info;
     info.has_hide_mouse = true;
 
     /* */
+    vd->sys     = sys;
     vd->fmt     = fmt;
     vd->info    = info;
     vd->pool    = Pool;
@@ -230,6 +234,8 @@ static void Close(vlc_object_t *object)
     vout_display_t *vd = (vout_display_t *)object;
     vout_display_sys_t *sys = vd->sys;
 
+    if (sys->cleanup)
+        sys->cleanup(sys->opaque);
     picture_pool_Delete(sys->pool);
     free(sys);
 }
@@ -237,8 +243,57 @@ static void Close(vlc_object_t *object)
 /* */
 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
 {
-    VLC_UNUSED(count);
-    return vd->sys->pool;
+    vout_display_sys_t *sys = vd->sys;
+
+    if (sys->pool)
+        return sys->pool;
+
+    if (count > sys->count)
+        count = sys->count;
+
+    picture_t *pictures[count];
+
+    for (unsigned i = 0; i < count; i++) {
+        picture_resource_t rsc;
+
+        rsc.p_sys = malloc(sizeof(*rsc.p_sys));
+        if (unlikely(!rsc.p_sys)) {
+            count = i;
+            break;
+        }
+
+        rsc.p_sys->sys = sys;
+        rsc.p_sys->id = NULL;
+
+        for (unsigned i = 0; i < PICTURE_PLANE_MAX; i++) {
+            /* vmem-lock is responsible for the allocation */
+            rsc.p[i].p_pixels = NULL;
+            rsc.p[i].i_lines  = sys->lines[i];
+            rsc.p[i].i_pitch  = sys->pitches[i];
+        }
+
+        pictures[i] = picture_NewFromResource(&vd->fmt, &rsc);
+        if (!pictures[i]) {
+            free(rsc.p_sys);
+            count = i;
+            break;
+        }
+    }
+
+    /* */
+    picture_pool_configuration_t pool;
+    memset(&pool, 0, sizeof(pool));
+    pool.picture_count = count;
+    pool.picture       = pictures;
+    pool.lock          = Lock;
+    pool.unlock        = Unlock;
+    sys->pool = picture_pool_NewExtended(&pool);
+    if (!sys->pool) {
+        for (unsigned i = 0; i < count; i++)
+            picture_Release(pictures[i]);
+    }
+
+    return sys->pool;
 }
 
 static void Display(vout_display_t *vd, picture_t *picture)