audio_output/amem.c: calculate shared memory region correctly.

Correctly set output format ES values that are needed for setting up
the shared memory region correctly.

modules/audio_output/amem.c

@@ -229,7 +229,7 @@ static amem_buffer_t *amem_buffer_get(aout_instance_t *aout, unsigned int index)
     aout_sys_t *sys = (aout_sys_t *)aout->output.p_sys;
     amem_buffer_t *buffer = NULL;
 
-    size_t buflen = sizeof(amem_buffer_t) + aout->output.i_nb_samples * aout->output.output.i_bitspersample;
+    size_t buflen = sizeof(amem_buffer_t) + (aout->output.i_nb_samples * aout->output.output.i_bitspersample);
     buffer = (amem_buffer_t *)((uint8_t*)sys->header + sizeof(amem_shared_t) + index * buflen);
     buffer->samples = ((uint8_t *)buffer + sizeof(amem_buffer_t));
     return buffer;
@@ -527,11 +527,15 @@ static int Open(vlc_object_t *p_this)
     free(psz_format);
 
     aout->output.output.i_format = format_int[i];
+    aout->output.output.i_bitspersample = aout_BitsPerSample(aout->output.output.i_format);
     if (AOUT_FMT_NON_LINEAR(&aout->output.output))
     {
         aout->output.i_nb_samples = A52_FRAME_NB;
         aout->output.output.i_bytes_per_frame = AOUT_SPDIF_SIZE;
         aout->output.output.i_frame_length = A52_FRAME_NB;
+        if ((aout->output.output.i_bitspersample == 0) &&
+            (aout->output.output.i_format == VLC_CODEC_SPDIFL))
+            aout->output.output.i_bitspersample = 32;
         aout_VolumeNoneInit(aout);
     }
     else
@@ -627,8 +631,10 @@ static int SetupSharedMem(aout_instance_t *aout, unsigned int i_buffers)
     aout_sys_t *sys = (aout_sys_t *)aout->output.p_sys;
 
     /* Determine shared memory size */
-    ssize_t len = sizeof(amem_shared_t) + i_buffers * (sizeof(amem_buffer_t)
-                       + aout->output.i_nb_samples * aout->output.output.i_bitspersample);
+    ssize_t buflen = sizeof(amem_buffer_t) +
+                      (aout->output.i_nb_samples * aout->output.output.i_bitspersample);
+    ssize_t len = sizeof(amem_shared_t) + (i_buffers * buflen);
+
     if (*sys->psz_name != '/')
     {
         msg_Err(aout, "invalid shared segment name (does not begin with /)");