diff --git a/libavcodec/vaapi_mpeg4.c b/libavcodec/vaapi_mpeg4.c
index 3a695cc..50d0850 100644
--- a/libavcodec/vaapi_mpeg4.c
+++ b/libavcodec/vaapi_mpeg4.c
@@ -91,9 +91,11 @@ static int vaapi_mpeg4_start_frame(AVCodecContext *avctx, av_unused const uint8_
     pic_param->TRB                                      = s->pb_time;
     pic_param->TRD                                      = s->pp_time;
 
-    if (s->pict_type == FF_B_TYPE)
-        pic_param->backward_reference_picture = ff_vaapi_get_surface_id(&s->next_picture);
-    if (s->pict_type != FF_I_TYPE)
+    if (s->pict_type == FF_B_TYPE) {
+        pic_param->forward_reference_picture  = ff_vaapi_get_surface_id(&s->next_picture);
+        pic_param->backward_reference_picture = ff_vaapi_get_surface_id(&s->last_picture);
+    }
+    else if (s->pict_type != FF_I_TYPE)
         pic_param->forward_reference_picture  = ff_vaapi_get_surface_id(&s->last_picture);
 
     /* Fill in VAIQMatrixBufferMPEG4 */