mediacodec: add audio support in mc_api

modules/codec/omxil/mediacodec.c

@@ -339,9 +339,10 @@ static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
 static int StartMediaCodec(decoder_t *p_dec)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    int i_angle = 0, i_ret;
+    int i_ret;
     size_t h264_profile = 0;
     char *psz_name = NULL;
+    union mc_api_args args;
 
     if (p_dec->fmt_in.i_extra && !p_sys->p_csd)
     {
@@ -393,27 +394,30 @@ static int StartMediaCodec(decoder_t *p_dec)
 
         p_sys->i_csd_send = 0;
     }
+
     if (!p_sys->i_width || !p_sys->i_height)
     {
         msg_Err(p_dec, "invalid size, abort MediaCodec");
         return VLC_EGENERIC;
     }
+    args.video.i_width = p_sys->i_width;
+    args.video.i_height = p_sys->i_height;
 
-    if ( p_dec->fmt_in.video.orientation != ORIENT_NORMAL)
+    if (p_dec->fmt_in.video.orientation != ORIENT_NORMAL)
     {
         switch (p_dec->fmt_in.video.orientation)
         {
             case ORIENT_ROTATED_90:
-                i_angle = 90;
+                args.video.i_angle = 90;
                 break;
             case ORIENT_ROTATED_180:
-                i_angle = 180;
+                args.video.i_angle = 180;
                 break;
             case ORIENT_ROTATED_270:
-                i_angle = 270;
+                args.video.i_angle = 270;
                 break;
             default:
-                i_angle = 0;
+                args.video.i_angle = 0;
         }
     }
 
@@ -426,8 +430,8 @@ static int StartMediaCodec(decoder_t *p_dec)
 
     if (!p_sys->p_awh && var_InheritBool(p_dec, CFG_PREFIX "dr"))
         p_sys->p_awh = AWindowHandler_new(VLC_OBJECT(p_dec));
-    i_ret = p_sys->api->start(p_sys->api, p_sys->p_awh, psz_name, p_sys->mime,
-                              p_sys->i_width, p_sys->i_height, i_angle);
+    args.video.p_awh = p_sys->p_awh;
+    i_ret = p_sys->api->start(p_sys->api, psz_name, p_sys->mime, &args);
 
     if (i_ret == VLC_SUCCESS)
     {
@@ -510,6 +514,7 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
     if (!api)
         return VLC_ENOMEM;
     api->p_obj = p_this;
+    api->b_video = p_dec->fmt_in.i_cat == VIDEO_ES;
     if (pf_init(api) != VLC_SUCCESS)
     {
         free(api);
@@ -778,7 +783,7 @@ static int GetOutput(decoder_t *p_dec, picture_t *p_pic,
         return 1;
     } else {
         assert(out.type == MC_OUT_TYPE_CONF);
-        p_sys->pixel_format = out.u.conf.pixel_format;
+        p_sys->pixel_format = out.u.conf.video.pixel_format;
         ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format,
                                              &p_sys->architecture_specific_data);
 
@@ -792,33 +797,33 @@ static int GetOutput(decoder_t *p_dec, picture_t *p_pic,
         }
 
         msg_Err(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
-                p_sys->pixel_format, name, out.u.conf.width, out.u.conf.height,
-                out.u.conf.stride, out.u.conf.slice_height,
-                out.u.conf.crop_left, out.u.conf.crop_top,
-                out.u.conf.crop_right, out.u.conf.crop_bottom);
+                p_sys->pixel_format, name, out.u.conf.video.width, out.u.conf.video.height,
+                out.u.conf.video.stride, out.u.conf.video.slice_height,
+                out.u.conf.video.crop_left, out.u.conf.video.crop_top,
+                out.u.conf.video.crop_right, out.u.conf.video.crop_bottom);
 
-        p_dec->fmt_out.video.i_width = out.u.conf.crop_right + 1 - out.u.conf.crop_left;
-        p_dec->fmt_out.video.i_height = out.u.conf.crop_bottom + 1 - out.u.conf.crop_top;
+        p_dec->fmt_out.video.i_width = out.u.conf.video.crop_right + 1 - out.u.conf.video.crop_left;
+        p_dec->fmt_out.video.i_height = out.u.conf.video.crop_bottom + 1 - out.u.conf.video.crop_top;
         if (p_dec->fmt_out.video.i_width <= 1
             || p_dec->fmt_out.video.i_height <= 1) {
-            p_dec->fmt_out.video.i_width = out.u.conf.width;
-            p_dec->fmt_out.video.i_height = out.u.conf.height;
+            p_dec->fmt_out.video.i_width = out.u.conf.video.width;
+            p_dec->fmt_out.video.i_height = out.u.conf.video.height;
         }
         p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
         p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
 
-        p_sys->stride = out.u.conf.stride;
-        p_sys->slice_height = out.u.conf.slice_height;
+        p_sys->stride = out.u.conf.video.stride;
+        p_sys->slice_height = out.u.conf.video.slice_height;
         if (p_sys->stride <= 0)
-            p_sys->stride = out.u.conf.width;
+            p_sys->stride = out.u.conf.video.width;
         if (p_sys->slice_height <= 0)
-            p_sys->slice_height = out.u.conf.height;
+            p_sys->slice_height = out.u.conf.video.height;
 
-        ArchitectureSpecificCopyHooks(p_dec, out.u.conf.pixel_format,
-                                      out.u.conf.slice_height,
+        ArchitectureSpecificCopyHooks(p_dec, out.u.conf.video.pixel_format,
+                                      out.u.conf.video.slice_height,
                                       p_sys->stride, &p_sys->architecture_specific_data);
         if (p_sys->pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
-            p_sys->slice_height -= out.u.conf.crop_top/2;
+            p_sys->slice_height -= out.u.conf.video.crop_top/2;
         if (IgnoreOmxDecoderPadding(p_sys->psz_name)) {
             p_sys->slice_height = 0;
             p_sys->stride = p_dec->fmt_out.video.i_width;

modules/codec/omxil/mediacodec.h

@@ -50,20 +50,45 @@ struct mc_api_out
             const void *p_ptr;
             int i_size;
         } buf;
-        struct
+        union
         {
-            int width, height;
-            int stride;
-            int slice_height;
-            int pixel_format;
-            int crop_left;
-            int crop_top;
-            int crop_right;
-            int crop_bottom;
+            struct
+            {
+                int width, height;
+                int stride;
+                int slice_height;
+                int pixel_format;
+                int crop_left;
+                int crop_top;
+                int crop_right;
+                int crop_bottom;
+            } video;
+            struct
+            {
+                int channel_count;
+                int channel_mask;
+                int sample_rate;
+            } audio;
         } conf;
     } u;
 };
 
+union mc_api_args
+{
+    struct
+    {
+        AWindowHandler *p_awh;
+        int i_width;
+        int i_height;
+        int i_angle;
+    } video;
+    struct
+    {
+        int i_sample_rate;
+        int i_channel_count;
+    } audio;
+};
+
 struct mc_api
 {
     vlc_object_t *p_obj;
@@ -71,12 +96,13 @@ struct mc_api
     mc_api_sys *p_sys;
 
     bool b_started;
+    bool b_video;
     bool b_direct_rendering;
     bool b_support_interlaced;
 
     void (*clean)(mc_api *);
-    int (*start)(mc_api *, AWindowHandler *p_awh, const char *psz_name,
-                 const char *psz_mime, int i_width, int i_height, int i_angle);
+    int (*start)(mc_api *, const char *psz_name, const char *psz_mime,
+                 union mc_api_args *p_args);
     int (*stop)(mc_api *);
     int (*flush)(mc_api *);
     int (*put_in)(mc_api *, const void *p_buf, size_t i_size,

modules/codec/omxil/mediacodec_jni.c

@@ -62,7 +62,8 @@ struct jfields
     jmethodID get_output_buffers, get_output_buffer;
     jmethodID dequeue_input_buffer, dequeue_output_buffer, queue_input_buffer;
     jmethodID release_output_buffer;
-    jmethodID create_video_format, set_integer, set_bytebuffer, get_integer;
+    jmethodID create_video_format, create_audio_format;
+    jmethodID set_integer, set_bytebuffer, get_integer;
     jmethodID buffer_info_ctor;
     jfieldID size_field, offset_field, pts_field;
 };
@@ -130,6 +131,7 @@ static const struct member members[] = {
     { "releaseOutputBuffer", "(IZ)V", "android/media/MediaCodec", OFF(release_output_buffer), METHOD, true },
 
     { "createVideoFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_video_format), STATIC_METHOD, true },
+    { "createAudioFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_audio_format), STATIC_METHOD, true },
     { "setInteger", "(Ljava/lang/String;I)V", "android/media/MediaFormat", OFF(set_integer), METHOD, true },
     { "getInteger", "(Ljava/lang/String;)I", "android/media/MediaFormat", OFF(get_integer), METHOD, true },
     { "setByteBuffer", "(Ljava/lang/String;Ljava/nio/ByteBuffer;)V", "android/media/MediaFormat", OFF(set_bytebuffer), METHOD, true },
@@ -457,13 +459,13 @@ static int Stop(mc_api *api)
 /*****************************************************************************
  * Start
  *****************************************************************************/
-static int Start(mc_api *api, AWindowHandler *p_awh, const char *psz_name,
-                 const char *psz_mime, int i_width, int i_height, int i_angle)
+static int Start(mc_api *api, const char *psz_name, const char *psz_mime,
+                 union mc_api_args *p_args)
 {
     mc_api_sys *p_sys = api->p_sys;
     JNIEnv* env = NULL;
     int i_ret = VLC_EGENERIC;
-    bool b_direct_rendering;
+    bool b_direct_rendering = false;
     jstring jmime = NULL;
     jstring jcodec_name = NULL;
     jobject jcodec = NULL;
@@ -494,31 +496,45 @@ static int Start(mc_api *api, AWindowHandler *p_awh, const char *psz_name,
     }
     p_sys->codec = (*env)->NewGlobalRef(env, jcodec);
 
-    jformat = (*env)->CallStaticObjectMethod(env, jfields.media_format_class,
-                                             jfields.create_video_format, jmime,
-                                             i_width, i_height);
-
-    if (p_awh)
-        jsurface = AWindowHandler_getSurface(p_awh, AWindow_Video);
-    b_direct_rendering = !!jsurface;
-
-    /* There is no way to rotate the video using direct rendering (and using a
-     * SurfaceView) before  API 21 (Lollipop). Therefore, we deactivate direct
-     * rendering if video doesn't have a normal rotation and if
-     * get_input_buffer method is not present (This method exists since API
-     * 21). */
-    if (b_direct_rendering && i_angle != 0 && !jfields.get_input_buffer)
-        b_direct_rendering = false;
-
-    if (b_direct_rendering)
+    if (api->b_video)
     {
-        if (i_angle != 0)
-        {
+        jformat = (*env)->CallStaticObjectMethod(env,
+                                                 jfields.media_format_class,
+                                                 jfields.create_video_format,
+                                                 jmime,
+                                                 p_args->video.i_width,
+                                                 p_args->video.i_height);
+        if (p_args->video.p_awh)
+            jsurface = AWindowHandler_getSurface(p_args->video.p_awh,
+                                                 AWindow_Video);
+        b_direct_rendering = !!jsurface;
+
+        /* There is no way to rotate the video using direct rendering (and
+         * using a SurfaceView) before  API 21 (Lollipop). Therefore, we
+         * deactivate direct rendering if video doesn't have a normal rotation
+         * and if get_input_buffer method is not present (This method exists
+         * since API 21). */
+        if (b_direct_rendering && p_args->video.i_angle != 0
+         && !jfields.get_input_buffer)
+            b_direct_rendering = false;
+
+        if (b_direct_rendering && p_args->video.i_angle != 0)
             jrotation_string = (*env)->NewStringUTF(env, "rotation-degrees");
             (*env)->CallVoidMethod(env, jformat, jfields.set_integer,
-                                   jrotation_string, i_angle);
-        }
+                                   jrotation_string, p_args->video.i_angle);
+    }
+    else
+    {
+        jformat = (*env)->CallStaticObjectMethod(env,
+                                                 jfields.media_format_class,
+                                                 jfields.create_audio_format,
+                                                 jmime,
+                                                 p_args->audio.i_sample_rate,
+                                                 p_args->audio.i_channel_count);
+    }
 
+    if (b_direct_rendering)
+    {
         // Configure MediaCodec with the Android surface.
         (*env)->CallVoidMethod(env, p_sys->codec, jfields.configure,
                                jformat, jsurface, NULL, 0);
@@ -769,15 +785,24 @@ static int GetOutput(mc_api *api, mc_api_out *p_out, mtime_t i_timeout)
         (*env)->ReleaseStringUTFChars(env, format_string, format_ptr);
 
         p_out->type = MC_OUT_TYPE_CONF;
-        p_out->u.conf.width         = GET_INTEGER(format, "width");
-        p_out->u.conf.height        = GET_INTEGER(format, "height");
-        p_out->u.conf.stride        = GET_INTEGER(format, "stride");
-        p_out->u.conf.slice_height  = GET_INTEGER(format, "slice-height");
-        p_out->u.conf.pixel_format  = GET_INTEGER(format, "color-format");
-        p_out->u.conf.crop_left     = GET_INTEGER(format, "crop-left");
-        p_out->u.conf.crop_top      = GET_INTEGER(format, "crop-top");
-        p_out->u.conf.crop_right    = GET_INTEGER(format, "crop-right");
-        p_out->u.conf.crop_bottom   = GET_INTEGER(format, "crop-bottom");
+        if (api->b_video)
+        {
+            p_out->u.conf.video.width         = GET_INTEGER(format, "width");
+            p_out->u.conf.video.height        = GET_INTEGER(format, "height");
+            p_out->u.conf.video.stride        = GET_INTEGER(format, "stride");
+            p_out->u.conf.video.slice_height  = GET_INTEGER(format, "slice-height");
+            p_out->u.conf.video.pixel_format  = GET_INTEGER(format, "color-format");
+            p_out->u.conf.video.crop_left     = GET_INTEGER(format, "crop-left");
+            p_out->u.conf.video.crop_top      = GET_INTEGER(format, "crop-top");
+            p_out->u.conf.video.crop_right    = GET_INTEGER(format, "crop-right");
+            p_out->u.conf.video.crop_bottom   = GET_INTEGER(format, "crop-bottom");
+        }
+        else
+        {
+            p_out->u.conf.audio.channel_count = GET_INTEGER(format, "channel-count");
+            p_out->u.conf.audio.channel_mask = GET_INTEGER(format, "channel-mask");
+            p_out->u.conf.audio.sample_rate = GET_INTEGER(format, "sample-rate");
+        }
 
         (*env)->DeleteLocalRef(env, format);
         return 1;

modules/codec/omxil/mediacodec_ndk.c

@@ -296,8 +296,8 @@ static int Stop(mc_api *api)
 /*****************************************************************************
  * Start
  *****************************************************************************/
-static int Start(mc_api *api, AWindowHandler *p_awh, const char *psz_name,
-                 const char *psz_mime, int i_width, int i_height, int i_angle)
+static int Start(mc_api *api, const char *psz_name, const char *psz_mime,
+                 union mc_api_args *p_args)
 {
     mc_api_sys *p_sys = api->p_sys;
     int i_ret = VLC_EGENERIC;
@@ -317,14 +317,22 @@ static int Start(mc_api *api, AWindowHandler *p_awh, const char *psz_name,
         goto error;
     }
 
-    syms.AMediaFormat.setString(p_sys->p_format, "mime", psz_mime);
-    syms.AMediaFormat.setInt32(p_sys->p_format, "width", i_width);
-    syms.AMediaFormat.setInt32(p_sys->p_format, "height", i_height);
-    syms.AMediaFormat.setInt32(p_sys->p_format, "rotation-degrees", i_angle);
     syms.AMediaFormat.setInt32(p_sys->p_format, "encoder", 0);
-
-    if (p_awh)
-        p_anw = AWindowHandler_getANativeWindow(p_awh, AWindow_Video);
+    syms.AMediaFormat.setString(p_sys->p_format, "mime", psz_mime);
+    if (api->b_video)
+    {
+        syms.AMediaFormat.setInt32(p_sys->p_format, "width", p_args->video.i_width);
+        syms.AMediaFormat.setInt32(p_sys->p_format, "height", p_args->video.i_height);
+        syms.AMediaFormat.setInt32(p_sys->p_format, "rotation-degrees", p_args->video.i_angle);
+        if (p_args->video.p_awh)
+            p_anw = AWindowHandler_getANativeWindow(p_args->video.p_awh,
+                                                    AWindow_Video);
+    }
+    else
+    {
+        syms.AMediaFormat.setInt32(p_sys->p_format, "sample-rate", p_args->audio.i_sample_rate);
+        syms.AMediaFormat.setInt32(p_sys->p_format, "channel-count", p_args->audio.i_channel_count);
+    }
 
     if (syms.AMediaCodec.configure(p_sys->p_codec, p_sys->p_format,
                                    p_anw, NULL, 0) != AMEDIA_OK)
@@ -456,15 +464,24 @@ static int GetOutput(mc_api *api, mc_api_out *p_out, mtime_t i_timeout)
         AMediaFormat *format = syms.AMediaCodec.getOutputFormat(p_sys->p_codec);
 
         p_out->type = MC_OUT_TYPE_CONF;
-        p_out->u.conf.width = GetFormatInteger(format, "width");
-        p_out->u.conf.height = GetFormatInteger(format, "height");
-        p_out->u.conf.stride = GetFormatInteger(format, "stride");
-        p_out->u.conf.slice_height = GetFormatInteger(format, "slice-height");
-        p_out->u.conf.pixel_format = GetFormatInteger(format, "color-format");
-        p_out->u.conf.crop_left = GetFormatInteger(format, "crop-left");
-        p_out->u.conf.crop_top = GetFormatInteger(format, "crop-top");
-        p_out->u.conf.crop_right = GetFormatInteger(format, "crop-right");
-        p_out->u.conf.crop_bottom = GetFormatInteger(format, "crop-bottom");
+        if (api->b_video)
+        {
+            p_out->u.conf.video.width         = GetFormatInteger(format, "width");
+            p_out->u.conf.video.height        = GetFormatInteger(format, "height");
+            p_out->u.conf.video.stride        = GetFormatInteger(format, "stride");
+            p_out->u.conf.video.slice_height  = GetFormatInteger(format, "slice-height");
+            p_out->u.conf.video.pixel_format  = GetFormatInteger(format, "color-format");
+            p_out->u.conf.video.crop_left     = GetFormatInteger(format, "crop-left");
+            p_out->u.conf.video.crop_top      = GetFormatInteger(format, "crop-top");
+            p_out->u.conf.video.crop_right    = GetFormatInteger(format, "crop-right");
+            p_out->u.conf.video.crop_bottom   = GetFormatInteger(format, "crop-bottom");
+        }
+        else
+        {
+            p_out->u.conf.audio.channel_count = GetFormatInteger(format, "channel-count");
+            p_out->u.conf.audio.channel_mask  = GetFormatInteger(format, "channel-mask");
+            p_out->u.conf.audio.sample_rate   = GetFormatInteger(format, "sample-rate");
+        }
         return 1;
     }
     else if (i_index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED