Try to use the ffmpeg api more correctly:

call avcodec_receive_packet() in a loop, drain and flush the internal buffers.
Introduced private method to improve code readability (hopefully).

src/add-ons/media/plugins/ffmpeg/AVCodecEncoder.cpp

@@ -686,6 +686,7 @@ AVCodecEncoder::_EncodeVideo(const void* buffer, int64 frameCount,
 
 	status_t ret = B_OK;
 
+	AVPacket* pkt = av_packet_alloc();
 	while (frameCount > 0) {
 		size_t bpr = fInputFormat.u.raw_video.display.bytes_per_row;
 		size_t bufferSize = fInputFormat.u.raw_video.display.line_count * bpr;
@@ -699,20 +700,44 @@ AVCodecEncoder::_EncodeVideo(const void* buffer, int64 frameCount,
 			fInputFormat.u.raw_video.display.line_count, fDstFrame.data,
 			fDstFrame.linesize);
 		
+		if (_EncodeVideoFrame(fFrame, pkt, info) == B_OK) {
+			// Skip to the next frame (but usually, there is only one to encode
+			// for video).
+			frameCount--;
+			fFramesWritten++;
+			buffer = (const void*)((const uint8*)buffer + bufferSize);
+		}
+	}
+
+	// Pass a NULL AVFrame and enter "draining" mode, then flush buffers
+	// before restarting encoding, again.
+	// TODO: It's probably not very efficient to do it like this. We should
+	// do this only when there is no more data (when closing the "stream")
+	_EncodeVideoFrame(NULL, pkt, info);
+	avcodec_flush_buffers(fCodecContext);
+	av_packet_free(&pkt);
+	return ret;
+}
+
+
+status_t
+AVCodecEncoder::_EncodeVideoFrame(AVFrame* frame, AVPacket* pkt, media_encode_info* info)
+{
 	// Encode one video chunk/frame.
-		int result = avcodec_send_frame(fCodecContext, fFrame);
+	int result = avcodec_send_frame(fCodecContext, frame);
-		if (result != 0) {
+	if (result < 0) {
 		TRACE("  avcodec_send_frame() failed: %d\n", result);
 		return B_ERROR;
 	}
 
-		// avcodec.h says we need to set it.
+	// Increase the frame pts as in the ffmpeg sample code
-		fFrame->pts++;
+	if (frame != NULL)
+		frame->pts++;
 	
-		AVPacket* pkt = av_packet_alloc();
+	while (result == 0) {
-		// TODO: Since av_codec_receive_packet() could return more than one packet for one frame,
+		result = avcodec_receive_packet(fCodecContext, pkt);
-		// we should run this in a loop, like the ffmpeg example do. 
+		if (result == 0) {
-		if (avcodec_receive_packet(fCodecContext, pkt) == 0) {
+			TRACE("  avcodec_receive_packet: received one packet\n");
 			// Maybe we need to use this PTS to calculate start_time:
 			if (pkt->pts != AV_NOPTS_VALUE) {
 				TRACE("  codec frame PTS: %lld (codec time_base: %d/%d)\n",
@@ -732,21 +757,18 @@ AVCodecEncoder::_EncodeVideo(const void* buffer, int64 frameCount,
 				info->flags |= B_MEDIA_KEY_FRAME;
 	
 			// Write the chunk
-			ret = WriteChunk(pkt->data, pkt->size, info);
+			result = WriteChunk(pkt->data, pkt->size, info);
-			if (ret != B_OK) {
+			if (result != B_OK) {
-				TRACE("  error writing chunk: %s\n", strerror(ret));
+				TRACE("  error writing chunk: %s\n", strerror(result));
 				break;
 			}
+		}	
 		av_packet_unref(pkt);
 	}
+	if (result == AVERROR(EAGAIN))
+		return B_OK;
 
-		// Skip to the next frame (but usually, there is only one to encode
+	TRACE("   _EncodeVideoFrame(): returning...\n");
-		// for video).
+	return result;
-		frameCount--;
-		fFramesWritten++;
-		buffer = (const void*)((const uint8*)buffer + bufferSize);
-		av_packet_free(&pkt);
-	}
-	return ret;
 }
 

src/add-ons/media/plugins/ffmpeg/AVCodecEncoder.h

@@ -62,6 +62,9 @@ private:
 			status_t			_EncodeVideo(const void* buffer,
 									int64 frameCount,
 									media_encode_info* info);
+			status_t			_EncodeVideoFrame(AVFrame* frame,
+									AVPacket* pkt,
+									media_encode_info* info);
 
 private:
 			media_format		fInputFormat;