I've had exactly the same problem and wrote a patch to fix this. It seems to work, but I think more tests are needed.
I'm attaching the patch (against svn of trunk/hvirtual rev 840).
Regards,
--
Alexis Ballier.
--- cinelerra-cvs-20060827/quicktime/qth264.c 2006-08-27 00:32:07.130566427 +0200
+++ working/quicktime/qth264.c 2006-08-27 03:00:58.486699204 +0200
@@ -136,7 +136,11 @@
x264_param_t default_params;
x264_param_default(&default_params);
// Reset quantizer if fixed bitrate
+#if X264_BUILD < 48
if(codec->param.rc.b_cbr)
+#else
+ if(codec->param.rc.i_rc_method == X264_RC_ABR )
+#endif
{
codec->param.rc.i_qp_constant = default_params.rc.i_qp_constant;
codec->param.rc.i_qp_min = default_params.rc.i_qp_min;
@@ -469,7 +473,11 @@
}
else
if(!strcasecmp(key, "h264_fix_bitrate"))
+#if X264_BUILD < 48
codec->param.rc.b_cbr = (*(int*)value) / 1000;
+#else
+ codec->param.rc.i_bitrate = (*(int*)value) / 1000;
+#endif
}
}
--- cinelerra-cvs-20060827/quicktime/ffmpeg/libavcodec/x264.c 2006-08-27 00:32:00.564013252 +0200
+++ working/quicktime/ffmpeg/libavcodec/x264.c 2006-08-27 02:47:49.224599225 +0200
@@ -135,8 +135,13 @@
x4->params.i_keyint_max = avctx->gop_size;
x4->params.rc.i_bitrate = avctx->bit_rate / 1000;
x4->params.rc.i_vbv_buffer_size = avctx->rc_buffer_size / 1000;
- if(avctx->rc_buffer_size)
+ if(avctx->rc_buffer_size){
+#if X264_BUILD < 48
x4->params.rc.b_cbr = 1;
+#else
+ x4->params.rc.i_rc_method = X264_RC_ABR;
+#endif
+ }
x4->params.i_bframe = avctx->max_b_frames;
x4->params.b_cabac = avctx->coder_type == FF_CODER_TYPE_AC;
