video_filter: hqdn3d: handle alloc failures

modules/video_filter/hqdn3d.c

@@ -245,6 +245,13 @@ static picture_t *Filter(filter_t *filter, picture_t *src)
             cfg->Coefs[2],
             cfg->Coefs[3]);
 
+    if(unlikely(!cfg->Frame[0] || !cfg->Frame[1] || !cfg->Frame[2]))
+    {
+        picture_Release( src );
+        picture_Release( dst );
+        return NULL;
+    }
+
     return CopyInfoAndRelease(dst, src);
 }
 

modules/video_filter/hqdn3d.h

@@ -120,6 +120,8 @@ static void deNoise(unsigned char *Frame,        // mpi->planes[x]
 
     if(!FrameAnt){
         (*FrameAntPtr)=FrameAnt=malloc(W*H*sizeof(unsigned short));
+        if(!FrameAnt)
+            return;
         for (long Y = 0; Y < H; Y++){
             unsigned short* dst=&FrameAnt[Y*W];
             unsigned char* src=Frame+Y*sStride;