VAAPI spu: Various fixes

- use fixup_alpha only for libva < 0.33.0
- do not call abort() in RenderCachedSubpictures() on error, but continue to cleanup
  (except when VAAPI_DEBUG define has been set)

modules/codec/avcodec/vasub.c

@@ -307,6 +307,7 @@ error:
     return VLC_EGENERIC;
 }
 
+#if !VA_CHECK_VERSION(0,33,0)
 static inline void vaapi_fixup_alpha(uint8_t *p_pixel, size_t i_size)
 {
     for (size_t p = 0; p < i_size; p += 4)
@@ -326,6 +327,7 @@ static inline void vaapi_fixup_alpha(uint8_t *p_pixel, size_t i_size)
         }
     }
 }
+#endif
 
 static int CopyRGBAToVAImage(vlc_object_t *obj, VADisplay p_display,
                              picture_t *pic, VAImage *image, VAImageFormat *fmt,
@@ -333,6 +335,9 @@ static int CopyRGBAToVAImage(vlc_object_t *obj, VADisplay p_display,
 {
     assert(image);
     assert(pic->format.i_chroma == VLC_CODEC_RGBA);
+#if VA_CHECK_VERSION(0,33,0)
+    VLC_UNUSED(fixup_alpha);
+#endif
 
     /* Only supports RGBA images */
     if (!image || !fmt ||
@@ -405,6 +410,7 @@ static int CopyRGBAToVAImage(vlc_object_t *obj, VADisplay p_display,
             bytes_copied += i_size;
             assert(bytes_copied <= image->data_size);
 
+#if !VA_CHECK_VERSION(0,33,0)
             /* FIXME: Remove this code when VDPAU and XvBA backends for
              * VAAPI properly support alpha or chroma keying in subpictures. */
             if (fixup_alpha)
@@ -412,6 +418,7 @@ static int CopyRGBAToVAImage(vlc_object_t *obj, VADisplay p_display,
                 uint8_t *p_pixel = p_out + (i_line * image->pitches[0]);
                 vaapi_fixup_alpha(p_pixel, i_size);
             }
+#endif
         }
     }
 
@@ -757,7 +764,9 @@ subpicture_cache_t *RenderDirectSubpicture(vlc_object_t *obj, vlc_va_conn_t *vac
     cache->i_start = subpicture->i_start;
     cache->i_stop  = subpicture->i_stop;
 
+#ifdef VAAPI_DEBUG
     int i_sub = 0;
+#endif
     subpicture_region_t *region;
     for (region = subpicture->p_region; region != NULL; region = region->p_next)
     {
@@ -776,12 +785,11 @@ subpicture_cache_t *RenderDirectSubpicture(vlc_object_t *obj, vlc_va_conn_t *vac
 
 #ifdef VAAPI_DEBUG
         msg_Dbg(obj, "+ %d: subid %d", i_sub, vasub_cache->i_id);
+        i_sub++;
 #endif
 
         vlc_array_append(&cache->subpictures, (void *)vasub_cache);
         vasub_cache = NULL;
-
-        i_sub++;
     }
 
     /* remember what was created, so it can be released in DisplayPicture() */
@@ -832,7 +840,11 @@ int RenderCachedSubpictures(vlc_object_t *obj, vlc_va_conn_t *vaconn, vlc_array_
 
         /* Ooh: big booboo */
         if (i_sub >= count)
+#ifdef VAAPI_DEBUG
             abort();
+#else
+            goto cleanup;
+#endif
 
         vasubpicture_cache_t *vasubpic;
         vasubpic = (vasubpicture_cache_t *) vlc_array_item_at_index(&cache->subpictures, i_sub);