lavf: Support rotation side data

Support stream side data only

Close #11473
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/demux/avformat/demux.c

@@ -47,6 +47,14 @@
 
 #include <libavformat/avformat.h>
 
+#if ( (LIBAVUTIL_VERSION_MICRO <  100 && LIBAVUTIL_VERSION_INT >= AV_VERSION_INT( 53, 15, 0) ) || \
+      (LIBAVUTIL_VERSION_MICRO >= 100 && LIBAVUTIL_VERSION_INT >= AV_VERSION_INT( 52, 85, 100 ) )  )
+# if LIBAVFORMAT_VERSION_CHECK( 55, 18, 0, 40, 100)
+#  include <libavutil/display.h>
+#  define HAVE_AV_STREAM_GET_SIDE_DATA
+# endif
+#endif
+
 //#define AVFORMAT_DEBUG 1
 
 # define HAVE_AVUTIL_CODEC_ATTACHMENT 1
@@ -102,6 +110,49 @@ static vlc_fourcc_t CodecTagToFourcc( uint32_t codec_tag )
 /*****************************************************************************
  * Open
  *****************************************************************************/
+
+static void get_rotation(es_format_t *fmt, AVStream *s)
+{
+    char const *kRotateKey = "rotate";
+    AVDictionaryEntry *rotation = av_dict_get(s->metadata, kRotateKey, NULL, 0);
+    long angle = 0;
+
+    if( rotation )
+    {
+        angle = strtol(rotation->value, NULL, 10);
+
+        if (angle > 45 && angle < 135)
+            fmt->video.orientation = ORIENT_ROTATED_90;
+
+        else if (angle > 135 && angle < 225)
+            fmt->video.orientation = ORIENT_ROTATED_180;
+
+        else if (angle > 225 && angle < 315)
+            fmt->video.orientation = ORIENT_ROTATED_270;
+
+        else
+            fmt->video.orientation = ORIENT_NORMAL;
+    }
+#ifdef HAVE_AV_STREAM_GET_SIDE_DATA
+    int32_t *matrix = (int32_t *)av_stream_get_side_data(s, AV_PKT_DATA_DISPLAYMATRIX, NULL);
+    if( matrix ) {
+        angle = lround(av_display_rotation_get(matrix));
+
+        if (angle > 45 && angle < 135)
+            fmt->video.orientation = ORIENT_ROTATED_270;
+
+        else if (angle > 135 || angle < -135)
+            fmt->video.orientation = ORIENT_ROTATED_180;
+
+        else if (angle < -45 && angle > -135)
+            fmt->video.orientation = ORIENT_ROTATED_90;
+
+        else
+            fmt->video.orientation = ORIENT_NORMAL;
+    }
+#endif
+}
+
 int OpenDemux( vlc_object_t *p_this )
 {
     demux_t       *p_demux = (demux_t*)p_this;
@@ -329,26 +380,7 @@ int OpenDemux( vlc_object_t *p_this )
             fmt.video.i_width = cc->width;
             fmt.video.i_height = cc->height;
 
-            char const *kRotateKey = "rotate";
-            AVDictionaryEntry *rotation = av_dict_get(s->metadata, kRotateKey, NULL, 0);
-
-            if( rotation )
-            {
-
-                long angle = strtol(rotation->value, NULL, 10);
-
-                if (angle > 45 && angle < 135)
-                    fmt.video.orientation = ORIENT_ROTATED_90;
-
-                else if (angle > 135 && angle < 225)
-                    fmt.video.orientation = ORIENT_ROTATED_180;
-
-                else if (angle > 225 && angle < 315)
-                    fmt.video.orientation = ORIENT_ROTATED_270;
-
-                else
-                    fmt.video.orientation = ORIENT_NORMAL;
-            }
+            get_rotation(&fmt, s);
 
 #if LIBAVCODEC_VERSION_MAJOR < 54
             if( cc->palctrl )