I am getting MMAL_EIO (Error 7) when trying to construct a H264 decoder.

[cmisip]

I have avcodec opening an rtsp stream. It reads packets from the stream and then I copy them over to the buffer that I send to the input of the decoder. The extradata is copied over from avcodec as well. The packet returned by avcodec contains NAL units. The first NAL unit contains the SPS, PPS, and IDR separated by 0x0000001. (This repeats periodically ). The subsequent NAL units contained in the avcodec packet contain the start code and the payload. So far as I can tell, the format is correct. The buffer is sent to the decoder, but the decoder does not return any output. It does not like the buffer for some reason. Any Ideas?

Code: Select all

#include "bcm_host.h"
#include "interface/mmal/mmal.h"
#include "interface/mmal/util/mmal_default_components.h"
#include "interface/mmal/util/mmal_util_params.h"
#include "interface/vcos/vcos.h"

#include <stdio.h>
#include <fcntl.h>

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/motion_vector.h>
#include <libavutil/imgutils.h>
#include <libavformat/avformat.h>
#include "libswscale/swscale.h"
#include <libavutil/frame.h>
#include "libavformat/avformat.h"
#include <libswscale/swscale.h>
}

//FUNCTIONS
#define CHECK_STATUS(status, msg) if (status != MMAL_SUCCESS) { fprintf(stderr, msg"\n"); }

struct CONTEXT_T {
   //VCOS_SEMAPHORE_T semaphore;
   MMAL_QUEUE_T *queue=NULL;
   MMAL_STATUS_T status;
} context;

MMAL_STATUS_T status = MMAL_EINVAL;
MMAL_COMPONENT_T *decoder = NULL;
MMAL_POOL_T *pool_in = NULL, *pool_out = NULL;
MMAL_PORT_T *input_port=NULL;
MMAL_PORT_T *output_port=NULL;


AVDictionary *opts = NULL;
int ret;
AVStream *st;
const char *src_filename = NULL;
AVFormatContext *fmt_ctx = NULL;
AVCodecContext *video_dec_ctx = NULL;
AVStream *video_stream = NULL;
AVCodec *dec = NULL;
AVCodecContext *dec_ctx = NULL;
int video_stream_idx = -1;
AVFrame *frame = NULL;
AVPacket pkt;



/** Callback from the control port.
 * Component is sending us an event. */
void control_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
{
   struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"CONTROL callback called\n");

   switch (buffer->cmd)
   {
   case MMAL_EVENT_EOS:
      /* Only sink component generate EOS events */
      fprintf(stderr,"EOS\n");
      break;
   case MMAL_EVENT_ERROR:
      /* Something went wrong. Signal this to the application */
      ctx->status = *(MMAL_STATUS_T *)buffer->data;
      fprintf(stderr,"ERROR: %d\n", ctx->status);
      break;
   default:
      break;
   }

   /* Done with the event, recycle it */
   mmal_buffer_header_release(buffer);

   /* Kick the processing thread */
   //vcos_semaphore_post(&ctx->semaphore);
}



void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
{
   //struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"INPUT callback called\n");
   /* The encoder is done with the data, just recycle the buffer header into its pool */
   mmal_buffer_header_release(buffer);

}

/** Callback from the output port.
 * Buffer has been produced by the port and is available for processing. */
void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer )
{
   struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"OUTPUT callback called\n");

   /* Queue the decoded video frame */
   mmal_queue_put(ctx->queue, buffer);

}

uint8_t ffmpeg_camera_initialize(){
	
	avformat_network_init();

    av_register_all();

    ret = avformat_open_input(&fmt_ctx, src_filename, NULL, NULL) ;
    if (ret <0 ) {
        fprintf(stderr, "Could not open source %s\n", src_filename);
        return ret;
    }


    ret = avformat_find_stream_info(fmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Could not find stream information\n");
        return ret;
    }

	
	ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);  //1 this version creates dec
    
    if (ret < 0) {
        fprintf(stderr, "Could not find %s stream in input file '%s'\n",
                av_get_media_type_string(AVMEDIA_TYPE_VIDEO), src_filename);
        return ret;
    } else {
        int stream_idx = ret;
        st = fmt_ctx->streams[stream_idx];
        
        dec_ctx = avcodec_alloc_context3(NULL);
        if (!dec_ctx) {
            fprintf(stderr, "Failed to allocate codec\n");
            return AVERROR(EINVAL);
        }

        ret = avcodec_parameters_to_context(dec_ctx, st->codecpar);
        if (ret < 0) {
            fprintf(stderr, "Failed to copy codec parameters to codec context\n");
            return ret;
        }
        
        video_stream_idx = stream_idx;
        video_stream = fmt_ctx->streams[video_stream_idx];
        video_dec_ctx = dec_ctx;
        
	    av_dump_format(fmt_ctx, 0, src_filename, 0);

        if (!video_stream) {
           fprintf(stderr, "Could not find video stream in the input, aborting\n");
           ret = 1;
        }
        
} 
        return ret;
};






uint8_t run(AVPacket *packet)
{   
	MMAL_BUFFER_HEADER_T *buffer;
	if (input_port) { 
	if ((buffer = mmal_queue_get(pool_in->queue)) != NULL)
      {  
         
         mmal_buffer_header_mem_lock(buffer);
         
         memcpy(buffer->data,packet->data,packet->size);
         buffer->length=packet->size;
         
         uint8_t nal_type=buffer->data[4] & 0x1f;
         fprintf(stderr,"NAL TYPE --------------------> %d\n",nal_type);
         //PRINT THE BUFFER CONTENTS
         for (uint32_t i = 0; i < buffer->length; ++i) {
                    fprintf(stderr, "\\%02x", (unsigned char)buffer->data[i]);
         }           
         fprintf(stderr,"\n");
         
         //buffer->pts=packet->pts;
         //buffer->dts=packet->pts;
         buffer->pts = buffer->dts = MMAL_TIME_UNKNOWN;
         mmal_buffer_header_mem_unlock(buffer);
            
         
         fprintf(stderr, "%s sending packet >>>>> %i bytes\n", decoder->input[0]->name, (int)buffer->length);
         status = mmal_port_send_buffer(decoder->input[0], buffer);
         CHECK_STATUS(status, "failed to send buffer to input port");
      }

      }
      
      if (output_port) {
      while ((buffer = mmal_queue_get(context.queue)) != NULL)
      {
         mmal_buffer_header_mem_lock(buffer);
         
         fprintf(stderr, "%s receiving %d bytes <<<<< frame\n", decoder->output[0]->name, buffer->length);
         
         mmal_buffer_header_mem_unlock(buffer); 
         mmal_buffer_header_release(buffer);
      }

     
      while ((buffer = mmal_queue_get(pool_out->queue)) != NULL)
      {
         status = mmal_port_send_buffer(decoder->output[0], buffer);
         CHECK_STATUS(status, "failed to send buffer to output port");
      }
      
  }
      
      
     return status;    
}	

int main(int argc, char **argv) {
   if (argc != 2) {
        fprintf(stderr, "Usage: %s rtsp://<user>:<pass>@url\n", argv[0]);
        return 1;
   }
    
   src_filename=argv[1];
     
   //Initialize ffmpeg
   ffmpeg_camera_initialize();
    
   uint16_t iwidth=video_dec_ctx->width;
   uint16_t iheight=video_dec_ctx->height; 
   
   status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &decoder);
   CHECK_STATUS(status, "failed to create decoder");
   
   // CONTROL PORT SETTINGS
   decoder->control->userdata = (MMAL_PORT_USERDATA_T *)&context;
   status = mmal_port_enable(decoder->control, control_callback);
   CHECK_STATUS(status, "failed to enable control port");
   
   
   //INPUT PORT SETTINGS
   input_port=decoder->input[0];
   
   MMAL_ES_FORMAT_T *format_in = decoder->input[0]->format;
   format_in->type = MMAL_ES_TYPE_VIDEO;
   format_in->encoding = MMAL_ENCODING_H264;
   
   format_in->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_in->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_in->es->video.frame_rate.num = 30;
   format_in->es->video.frame_rate.den = 1;
   format_in->es->video.par.num = 1;
   format_in->es->video.par.den = 1;
   format_in->es->video.crop.width = iwidth;
   format_in->es->video.crop.height = iheight;
   
   //ALLOCATE Extradata, copying from avcodec context
   status = mmal_format_extradata_alloc(format_in, video_dec_ctx->extradata_size);
   fprintf(stderr,"FORMAT pack size %d\n", video_dec_ctx->extradata_size);
   format_in->extradata_size = video_dec_ctx->extradata_size;
   if (format_in->extradata_size)
      memcpy(format_in->extradata, video_dec_ctx->extradata, video_dec_ctx->extradata_size);
   
   //PRINT the contents of extradata
   fprintf(stderr,"Extradata\n");
   for (size_t i = 0; i != format_in->extradata_size+1; ++i) {
                    fprintf(stderr, "\\%02x", (unsigned char)format_in->extradata[i]);
   }
   fprintf(stderr,"\n");
   
  
   /* Display the input port format */
   fprintf(stderr,"---------------------------------------------------\n");
   fprintf(stderr, "INPUT %s\n", decoder->input[0]->name);
   fprintf(stderr, " type: %i, fourcc: %4.4s\n", format_in->type, (char *)&format_in->encoding);
   fprintf(stderr, " bitrate: %i, framed: %i\n", format_in->bitrate,
           !!(format_in->flags & MMAL_ES_FORMAT_FLAG_FRAMED));
   fprintf(stderr, " extra data: %i, %p\n", format_in->extradata_size, format_in->extradata);
   fprintf(stderr, " width: %i, height: %i, (%i,%i,%i,%i)\n",
           format_in->es->video.width, format_in->es->video.height,
           format_in->es->video.crop.x, format_in->es->video.crop.y,
           format_in->es->video.crop.width, format_in->es->video.crop.height);
       
       
   MMAL_PARAMETER_BOOLEAN_T error_concealment;
   error_concealment.hdr.id = MMAL_PARAMETER_VIDEO_DECODE_ERROR_CONCEALMENT;
            error_concealment.hdr.size = sizeof(MMAL_PARAMETER_BOOLEAN_T);
            error_concealment.enable = MMAL_FALSE;
            status = mmal_port_parameter_set(decoder->input[0], &error_concealment.hdr);
            CHECK_STATUS(status, "failed to set error concealment");    
              
       
   
   status = mmal_port_format_commit(decoder->input[0]);
   CHECK_STATUS(status, "failed to commit input format");  

   status = mmal_port_enable(decoder->input[0], input_callback);
   CHECK_STATUS(status, "failed to enable input port");	
   
   decoder->input[0]->buffer_num = decoder->input[0]->buffer_num_recommended;
   decoder->input[0]->buffer_size = decoder->input[0]->buffer_size_recommended;
   
   pool_in = mmal_pool_create(decoder->input[0]->buffer_num,
                              decoder->input[0]->buffer_size);
   

   // Store a reference to our context in each port (will be used during callbacks) */
   decoder->input[0]->userdata = (MMAL_PORT_USERDATA_T *)&context;
   
   
   
   //OUTPUT PORT SETTINGS
   
   MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
   
   output_port=decoder->output[0];
    
   format_out->type = MMAL_ES_TYPE_VIDEO;
   format_out->encoding = MMAL_ENCODING_I420;
   
   format_out->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_out->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_out->es->video.frame_rate.num = 30;
   format_out->es->video.frame_rate.den = 1;
   format_out->es->video.par.num = 0; 
   format_out->es->video.par.den = 1;
   format_out->es->video.crop.width = iwidth;
   format_out->es->video.crop.height = iheight;
   
    /* Display the output port format */
   fprintf(stderr,"---------------------------------------------------\n");
   fprintf(stderr, "OUTPUT %s\n", decoder->output[0]->name);
   fprintf(stderr, " type: %i, fourcc: %4.4s\n", format_out->type, (char *)&format_out->encoding);
   fprintf(stderr, " bitrate: %i, framed: %i\n", format_out->bitrate,
           !!(format_out->flags & MMAL_ES_FORMAT_FLAG_FRAMED));
   fprintf(stderr, " extra data: %i, %p\n", format_out->extradata_size, format_out->extradata);
   fprintf(stderr, " width: %i, height: %i, (%i,%i,%i,%i)\n",
           format_out->es->video.width, format_out->es->video.height,
           format_out->es->video.crop.x, format_out->es->video.crop.y,
           format_out->es->video.crop.width, format_out->es->video.crop.height);
          
   
   
   status = mmal_port_format_commit(decoder->output[0]);
   CHECK_STATUS(status, "failed to commit output format");   
   
   status = mmal_port_enable(decoder->output[0], output_callback);
   CHECK_STATUS(status, "failed to enable output port"); 
   
   
   context.queue = mmal_queue_create();
	
   decoder->output[0]->buffer_num = decoder->output[0]->buffer_num_recommended;
   decoder->output[0]->buffer_size = decoder->output[0]->buffer_size_recommended; 
   
   pool_out = mmal_pool_create(decoder->output[0]->buffer_num,
                               decoder->output[0]->buffer_size);
                               
   decoder->output[0]->userdata = (MMAL_PORT_USERDATA_T *)&context; 
   
 
   fprintf(stderr,"Press Enter to Continue\n");
   fprintf(stderr,"Press Enter again to stop\n");
   getchar();
    
   /* Start decoding */
   fprintf(stderr, "start decoding\n");
    
   int framecount=0;
   char c;
   int n, tem;
    
    
   tem = fcntl(0, F_GETFL, 0);
   fcntl (0, F_SETFL, (tem | O_NDELAY));
    
   av_init_packet(&pkt);
    
     
   //Loop   
   while (av_read_frame(fmt_ctx, &pkt) >= 0) {
        if (pkt.stream_index == video_stream_idx) {
            
            run(&pkt);
            av_packet_unref(&pkt);
        }
        framecount++;
        fprintf(stderr, "Frame number %d\n",framecount);
        
        
        n = read(0, &c, 1);
        if (n > 0) break;
      
   }
    
    
   fcntl(0, F_SETFL, tem);
   /* End decoding */
   fprintf(stderr, "end decoding\n");
    
   avformat_network_deinit();
   
   
   if (video_dec_ctx)
      avcodec_free_context(&video_dec_ctx);
   if (fmt_ctx)
      avformat_close_input(&fmt_ctx);
   
      
   mmal_port_disable(decoder->input[0]);
   mmal_port_disable(decoder->output[0]);
   mmal_port_disable(decoder->control); 
      
   mmal_port_flush(decoder->input[0]);
   mmal_port_flush(decoder->output[0]);
   mmal_port_flush(decoder->control); 
      
      
   if (decoder)
      mmal_component_destroy(decoder);
   if (pool_in)
      mmal_pool_destroy(pool_in);
   if (pool_out)
      mmal_pool_destroy(pool_out);
   if (context.queue)
      mmal_queue_destroy(context.queue);
    
     
}

Here is a sample output:

Code: Select all

Input #0, rtsp, from 'rtsp://01432721:admin:password@192.168.0.26/onvif2':
  Metadata:
    title           : H.264 Video, RtspServer_0.0.0.2
  Duration: N/A, start: 0.100000, bitrate: N/A
    Stream #0:0: Video: h264 (Baseline), yuv420p(progressive), 320x192, 10 fps, 10 tbr, 90k tbn, 180k tbc
FORMAT pack size 22
Extradata
\00\00\00\01\67\42\00\1f\95\a8\14\01\6e\40\00\00\00\01\68\ce\3c\80\00
---------------------------------------------------
INPUT vc.ril.video_decode:in:0
 type: 3, fourcc: H264
 bitrate: 0, framed: 0
 extra data: 22, 0x1824fd8
 width: 320, height: 192, (0,0,320,192)
---------------------------------------------------
OUTPUT vc.ril.video_decode:out:0
 type: 3, fourcc: I420
 bitrate: 0, framed: 0
 extra data: 0, (nil)
 width: 320, height: 192, (0,0,320,192)
Press Enter to Continue
Press Enter again to stop
start decoding
NAL TYPE --------------------> 7
\00\00\00\01\67\42\00\1e\f4\0a\0c\c8\00\00\00\01\68\ce\3c\80\00\00\00\01\65\88\84\05\fc\71\c6\3f\62\85\00\02\70\54\b3\d5\01\f0\3d\46\6c\0b\aa\6f\23\7e\57\64\b9\0f\00\0b\09\fd\b5\c8\3a\80\ca\83\88\26\d6\53\36\1f\e0\31\5a\60\13\e4\3d\8b\ca\53\6f\3f\7f\72\37\90\ed\85\c4\5a\43\b1\5e\f3\93\33\62\25\63\b3\97\ff\ff\40\93\9c\5b\a1\0c\01\d0\05\9f\4a\16\e7\41\77\fc\1e\d4\b3\fa\49\ae\00\27\2f\e2\70\b3\09\cf\bf\c9\cb\e4\3d\a9\68\04\52\19\25\a9\03\59\b3\2c\81\0b\6f\70\86\62\8c\87\74\0f\3b\d7\bb\5f\65\38\7c\e4\c7\f9\1b\b1\d8\a5\18\78\c9\9e\31\8c\4e\43\22\8d\ec\11\cb\33\e8\14\8f\c0\4a\c5\2d\67\ee\d6\a0\e2\1b\e4\c2\5c\3f\d6\e7\f8\cc\12\a7\99\c0\0a\07\bf\cf\e4\1d\40\71\61\a2\3f\ea\2b\31\bb\f0\16\7a\24\c2\f7\0f\6f\67\05\bc\22\20\2d\21\ff\ff\b3\f7\cc\b4\3c\85\28\d4\9b\7b\fc\59\b2\1b\47\2b\f3\73\fe\f8\9f\77\f5\02\c4\63\86\10\90\8d\fa\ae\2d\08\85\30\1b\be\66\1c\67\20\0f\9a\ad\83\2c\b9\91\80\8a\44\32\45\0e\2a\30\fd\2d\97\cc\ce\cc\87\74\0b\3b\de\51\bb\5b\97\69\30\32\03\ed\bf\d4\f3\10\58\89\46\..............//This is quite long due to IDR.
vc.ril.video_decode:in:0(H264) sending packet >>>>> 3604 bytes
CONTROL callback called
ERROR: 7
INPUT callback called
Frame number 1
NAL TYPE --------------------> 1
\00\00\00\01\41\9a\24\0d\cf\c6\0a\81\db\2c\e0\0a\f2\4c\37\21\69\a2\e9\86\eb\3c\d9\e3\fe\7f\be\02\d0\d3\ae\6c\66\3f\41\7f\ff\45\73\34\47\0a\76\76\98\bc\3f\8c\98\e6\1e\49\cd\00\e3\c5\ff\50\00\61\1e\91\82\5f\8e\64\6c\92\f9\f9\e8\fe\0b\c4\42\ae\05\dc\74\0b\b0
vc.ril.video_decode:in:0(H264) sending packet >>>>> 85 bytes
Frame number 2
NAL TYPE --------------------> 1
\00\00\00\01\41\9a\48\09\d7\28\a1\e4\df\9b\89\b0\70\ca\59\de\37\01\e0\31\26\71\d1\18\5e
vc.ril.video_decode:in:0(H264) sending packet >>>>> 29 bytes
mmal: mmal_port_send_buffer: vc.ril.video_decode:in:0(H264): send failed: ENOMEM
failed to send buffer to input port
Frame number 3
NAL TYPE --------------------> 1
\00\00\00\01\41\9a\6c\17\3f\16\28\7d\0d\48\1c\0f\99\e9\93\a0\7a\5e\27\96\05\3f\87\e0\b8\78\08\7e\ca\5c\e1\6e\55\29\8d\27\79\ab\0a\51\60\8f\95\3b\e3\20
vc.ril.video_decode:in:0(H264) sending packet >>>>> 50 bytes
mmal: mmal_port_send_buffer: vc.ril.video_decode:in:0(H264): send failed: ENOMEM
failed to send buffer to input port
Frame number 4
NAL TYPE --------------------> 1
\00\00\00\01\41\9a\90\3d\70\80\a3\5c\91\7c\8b\b3\e6\1f\21\bd\77\5d\7c\dd\43\b3\fd\f8\8a\ad\57\22\fe\33\29\18\4a\c0\dc\15\09\48\ad\0c\73\e1\86\2d\43\b4\30\1d\44\40\a1\75\c6\61\bc\b9\9f\19\09\58\aa\ec\e2\c0\fb\6f\1d\f3\4d\2a\cf\e8\6d\7d\4c\5e\55\3b
CONTROL callback called
ERROR: 7

Every now and then I also get ENOMEM.

Thanks,
Chris

[cmisip]

So I got this working. I am getting output buffers in the correct size. The stumbling blocks were:

1. The sps and pps data must be sent in band as the first packet to the decoder. I had assumed that setting the input format's extradata fields were enough but that was not the case.

2. The input and output ports have to be enabled AFTER the input and output pools are created. Otherwise you get an ENOMEM.


A question for the devs.
1. Do I need to worry about splitting packets for the encoder and marking the first packet as MMAL_BUFFER_HEADER_FLAG_FRAME_START and the last packet as MMAL_BUFFER_HEADER_FLAG_FRAME_END? I have not encountered this situation. I expect it might happen when the resolution of the video frame is high enough that the packet size is greater than input buffer size.

Here is the working code for anyone going down this road

Thanks,
Chris

Code: Select all

#include "bcm_host.h"
#include "interface/mmal/mmal.h"
#include "interface/mmal/util/mmal_default_components.h"
#include "interface/mmal/util/mmal_util_params.h"
#include "interface/vcos/vcos.h"
#include "interface/mmal/vc/mmal_vc_api.h"

#include <stdio.h>
#include <fcntl.h>

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/motion_vector.h>
#include <libavutil/imgutils.h>
#include <libavformat/avformat.h>
#include "libswscale/swscale.h"
#include <libavutil/frame.h>
#include "libavformat/avformat.h"
#include <libswscale/swscale.h>
}

//FUNCTIONS
#define CHECK_STATUS(status, msg) if (status != MMAL_SUCCESS) { fprintf(stderr, msg"\n"); }

struct CONTEXT_T {
   //VCOS_SEMAPHORE_T semaphore;
   MMAL_QUEUE_T *queue=NULL;
   MMAL_STATUS_T status;
} context;

MMAL_STATUS_T status = MMAL_EINVAL;
MMAL_COMPONENT_T *decoder = NULL;
MMAL_POOL_T *pool_in = NULL, *pool_out = NULL;
MMAL_PORT_T *input_port=NULL;
MMAL_PORT_T *output_port=NULL;


AVDictionary *opts = NULL;
int ret;
AVStream *st;
const char *src_filename = NULL;
AVFormatContext *fmt_ctx = NULL;
AVCodecContext *video_dec_ctx = NULL;
AVStream *video_stream = NULL;
AVCodec *dec = NULL;
AVCodecContext *dec_ctx = NULL;
int video_stream_idx = -1;
AVFrame *frame = NULL;
AVPacket pkt;



/** Callback from the control port.
 * Component is sending us an event. */
void control_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
{
   struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"CONTROL callback called\n");

   switch (buffer->cmd)
   {
   case MMAL_EVENT_EOS:
      /* Only sink component generate EOS events */
      fprintf(stderr,"EOS\n");
      break;
   case MMAL_EVENT_ERROR:
      /* Something went wrong. Signal this to the application */
      ctx->status = *(MMAL_STATUS_T *)buffer->data;
      fprintf(stderr,"ERROR: %d\n", ctx->status);
      break;
   default:
      break;
   }

   /* Done with the event, recycle it */
   mmal_buffer_header_release(buffer);

   /* Kick the processing thread */
   //vcos_semaphore_post(&ctx->semaphore);
}



void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
{
   //struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"INPUT callback called\n");
   /* The encoder is done with the data, just recycle the buffer header into its pool */
   mmal_buffer_header_release(buffer);

}

/** Callback from the output port.
 * Buffer has been produced by the port and is available for processing. */
void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer )
{
   struct CONTEXT_T *ctx = (struct CONTEXT_T *)port->userdata;
   fprintf(stderr,"OUTPUT callback called\n");

   /* Queue the decoded video frame */
   mmal_queue_put(ctx->queue, buffer);

}

uint8_t ffmpeg_camera_initialize(){
	
	avformat_network_init();

    av_register_all();

    ret = avformat_open_input(&fmt_ctx, src_filename, NULL, NULL) ;
    if (ret <0 ) {
        fprintf(stderr, "Could not open source %s\n", src_filename);
        return ret;
    }


    ret = avformat_find_stream_info(fmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Could not find stream information\n");
        return ret;
    }

	
	ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);  //1 this version creates dec
    
    if (ret < 0) {
        fprintf(stderr, "Could not find %s stream in input file '%s'\n",
                av_get_media_type_string(AVMEDIA_TYPE_VIDEO), src_filename);
        return ret;
    } else {
        int stream_idx = ret;
        st = fmt_ctx->streams[stream_idx];
        
        dec_ctx = avcodec_alloc_context3(NULL);
        if (!dec_ctx) {
            fprintf(stderr, "Failed to allocate codec\n");
            return AVERROR(EINVAL);
        }

        ret = avcodec_parameters_to_context(dec_ctx, st->codecpar);
        if (ret < 0) {
            fprintf(stderr, "Failed to copy codec parameters to codec context\n");
            return ret;
        }
        
        video_stream_idx = stream_idx;
        video_stream = fmt_ctx->streams[video_stream_idx];
        video_dec_ctx = dec_ctx;
        
	    av_dump_format(fmt_ctx, 0, src_filename, 0);

        if (!video_stream) {
           fprintf(stderr, "Could not find video stream in the input, aborting\n");
           ret = 1;
        }
        
} 
        return ret;
};






uint8_t run(AVPacket *packet)
{   
	MMAL_BUFFER_HEADER_T *buffer;
	if (input_port) { 
	if ((buffer = mmal_queue_get(pool_in->queue)) != NULL)
      {  
         
         mmal_buffer_header_mem_lock(buffer);
         
         memcpy(buffer->data,packet->data,packet->size);
         buffer->length=packet->size;
         
         uint8_t nal_type=buffer->data[4] & 0x1f;
         fprintf(stderr,"NAL TYPE --------------------> %d\n",nal_type);
         //PRINT THE BUFFER CONTENTS
         for (uint32_t i = 0; ((i < buffer->length) && (i < 20)); ++i) {
                    fprintf(stderr, "\\%02x", (unsigned char)buffer->data[i]);
         }           
         fprintf(stderr,"\n");
         
         buffer->pts = buffer->dts = MMAL_TIME_UNKNOWN;
         buffer->flags=packet->flags;
         mmal_buffer_header_mem_unlock(buffer);
            
         
         fprintf(stderr, "%s sending packet >>>>> %i bytes\n", decoder->input[0]->name, (int)buffer->length);
         status = mmal_port_send_buffer(decoder->input[0], buffer);
         CHECK_STATUS(status, "failed to send buffer to input port");
      }

      }
      
      if (output_port) {
      while ((buffer = mmal_queue_get(context.queue)) != NULL)
      {
         mmal_buffer_header_mem_lock(buffer);
         
         fprintf(stderr, "%s receiving %d bytes <<<<< frame\n", decoder->output[0]->name, buffer->length);
         
         mmal_buffer_header_mem_unlock(buffer); 
         mmal_buffer_header_release(buffer);
      }

     
      while ((buffer = mmal_queue_get(pool_out->queue)) != NULL)
      {
         status = mmal_port_send_buffer(decoder->output[0], buffer);
         CHECK_STATUS(status, "failed to send buffer to output port");
      }
      
  }
      
      
     return status;    
}	

int main(int argc, char **argv) {
   if (argc != 2) {
        fprintf(stderr, "Usage: %s rtsp://<user>:<pass>@url\n", argv[0]);
        return 1;
   }
    
   src_filename=argv[1];
   
   bcm_host_init();
     
   //Initialize ffmpeg
   ffmpeg_camera_initialize();
    
   uint16_t iwidth=video_dec_ctx->width;
   uint16_t iheight=video_dec_ctx->height; 
   
   status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &decoder);
   CHECK_STATUS(status, "failed to create decoder");
   
   // CONTROL PORT SETTINGS
   decoder->control->userdata = (MMAL_PORT_USERDATA_T *)&context;
   status = mmal_port_enable(decoder->control, control_callback);
   CHECK_STATUS(status, "failed to enable control port");
   
   
   //INPUT PORT SETTINGS
   input_port=decoder->input[0];
   
   MMAL_ES_FORMAT_T *format_in = decoder->input[0]->format;
   format_in->type = MMAL_ES_TYPE_VIDEO;
   format_in->encoding = MMAL_ENCODING_H264;
   
   format_in->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_in->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_in->es->video.frame_rate.num = 30;
   format_in->es->video.frame_rate.den = 1;
   format_in->es->video.par.num = 1;
   format_in->es->video.par.den = 1;
   format_in->es->video.crop.width = iwidth;
   format_in->es->video.crop.height = iheight;
   
   //ALLOCATE Extradata, copying from avcodec context
   status = mmal_format_extradata_alloc(format_in, video_dec_ctx->extradata_size);
   fprintf(stderr,"FORMAT pack size %d\n", video_dec_ctx->extradata_size);
   format_in->extradata_size = video_dec_ctx->extradata_size;
   if (format_in->extradata_size)
      memcpy(format_in->extradata, video_dec_ctx->extradata, video_dec_ctx->extradata_size);
   
   //PRINT the contents of extradata
   fprintf(stderr,"Extradata\n");
   for (size_t i = 0; i != format_in->extradata_size+1; ++i) {
                    fprintf(stderr, "\\%02x", (unsigned char)format_in->extradata[i]);
   }
   fprintf(stderr,"\n");
   
  
   /* Display the input port format */
   fprintf(stderr,"---------------------------------------------------\n");
   fprintf(stderr, "INPUT %s\n", decoder->input[0]->name);
   fprintf(stderr, " type: %i, fourcc: %4.4s\n", format_in->type, (char *)&format_in->encoding);
   fprintf(stderr, " bitrate: %i, framed: %i\n", format_in->bitrate,
           !!(format_in->flags & MMAL_ES_FORMAT_FLAG_FRAMED));
   fprintf(stderr, " extra data: %i, %p\n", format_in->extradata_size, format_in->extradata);
   fprintf(stderr, " width: %i, height: %i, (%i,%i,%i,%i)\n",
           format_in->es->video.width, format_in->es->video.height,
           format_in->es->video.crop.x, format_in->es->video.crop.y,
           format_in->es->video.crop.width, format_in->es->video.crop.height);
       
       
   MMAL_PARAMETER_BOOLEAN_T error_concealment;
   error_concealment.hdr.id = MMAL_PARAMETER_VIDEO_DECODE_ERROR_CONCEALMENT;
            error_concealment.hdr.size = sizeof(MMAL_PARAMETER_BOOLEAN_T);
            error_concealment.enable = MMAL_FALSE;
            status = mmal_port_parameter_set(decoder->input[0], &error_concealment.hdr);
            CHECK_STATUS(status, "failed to set error concealment");    
              
       
   
   
   
   decoder->input[0]->buffer_num = decoder->input[0]->buffer_num_recommended;
   decoder->input[0]->buffer_size = decoder->input[0]->buffer_size_recommended;
   fprintf(stderr,"DECODER INPUT BUFFER SIZE %d NUM %d\n",decoder->input[0]->buffer_size,decoder->input[0]->buffer_num_recommended);
   
   status = mmal_port_format_commit(decoder->input[0]);
   CHECK_STATUS(status, "failed to commit input format");  

   status = mmal_port_enable(decoder->input[0], input_callback);
   CHECK_STATUS(status, "failed to enable input port");	
   
   pool_in = mmal_pool_create(decoder->input[0]->buffer_num,
                              decoder->input[0]->buffer_size);
   

   // Store a reference to our context in each port (will be used during callbacks) */
   decoder->input[0]->userdata = (MMAL_PORT_USERDATA_T *)&context;
   
   
   
   //OUTPUT PORT SETTINGS
   
   MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
   
   output_port=decoder->output[0];
    
   format_out->type = MMAL_ES_TYPE_VIDEO;
   format_out->encoding = MMAL_ENCODING_I420;
   
   format_out->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_out->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_out->es->video.frame_rate.num = 30;
   format_out->es->video.frame_rate.den = 1;
   format_out->es->video.par.num = 0; 
   format_out->es->video.par.den = 1;
   format_out->es->video.crop.width = iwidth;
   format_out->es->video.crop.height = iheight;
   
    /* Display the output port format */
   fprintf(stderr,"---------------------------------------------------\n");
   fprintf(stderr, "OUTPUT %s\n", decoder->output[0]->name);
   fprintf(stderr, " type: %i, fourcc: %4.4s\n", format_out->type, (char *)&format_out->encoding);
   fprintf(stderr, " bitrate: %i, framed: %i\n", format_out->bitrate,
           !!(format_out->flags & MMAL_ES_FORMAT_FLAG_FRAMED));
   fprintf(stderr, " extra data: %i, %p\n", format_out->extradata_size, format_out->extradata);
   fprintf(stderr, " width: %i, height: %i, (%i,%i,%i,%i)\n",
           format_out->es->video.width, format_out->es->video.height,
           format_out->es->video.crop.x, format_out->es->video.crop.y,
           format_out->es->video.crop.width, format_out->es->video.crop.height);
          
  
   context.queue = mmal_queue_create();
	
   decoder->output[0]->buffer_num = decoder->output[0]->buffer_num_recommended;
   decoder->output[0]->buffer_size = decoder->output[0]->buffer_size_recommended; 
   fprintf(stderr,"DECODER OUTPUT BUFFER SIZE %d NUM %d\n",decoder->output[0]->buffer_size_recommended,decoder->output[0]->buffer_num_recommended);
   
   
   status = mmal_port_format_commit(decoder->output[0]);
   CHECK_STATUS(status, "failed to commit output format");   
   
   status = mmal_port_enable(decoder->output[0], output_callback);
   CHECK_STATUS(status, "failed to enable output port"); 
   
   pool_out = mmal_pool_create(decoder->output[0]->buffer_num,
                               decoder->output[0]->buffer_size);
                               
   decoder->output[0]->userdata = (MMAL_PORT_USERDATA_T *)&context; 
   
   
   fprintf(stderr,"Press Enter to Continue\n");
   fprintf(stderr,"Press Enter again to stop\n");
   getchar();
    
   /* Start decoding */
   fprintf(stderr, "start decoding\n");
    
   int framecount=0;
   char c;
   int n, tem;
    
    
   tem = fcntl(0, F_GETFL, 0);
   fcntl (0, F_SETFL, (tem | O_NDELAY));
    
   av_init_packet(&pkt);
    
     
   //Send the sps and pps as first packet
   pkt.data=(uint8_t*)av_mallocz(video_dec_ctx->extradata_size);
   memcpy(pkt.data,video_dec_ctx->extradata,video_dec_ctx->extradata_size);
   pkt.size=video_dec_ctx->extradata_size; 
   
   run(&pkt);
   av_packet_unref(&pkt);
   
   //Send the rest of the packets in a loop
   while ((av_read_frame(fmt_ctx, &pkt) >= 0) && (framecount <100 )) {
        if (pkt.stream_index == video_stream_idx) {
           
            run(&pkt);
            av_packet_unref(&pkt);
        }
        framecount++;
        fprintf(stderr, "Frame number %d\n",framecount);
        
        
        n = read(0, &c, 1);
        if (n > 0) break;
      
   }
    
    
   fcntl(0, F_SETFL, tem);
   /* End decoding */
   fprintf(stderr, "end decoding\n");
    
   avformat_network_deinit();
   
   
   if (video_dec_ctx)
      avcodec_free_context(&video_dec_ctx);
   if (fmt_ctx)
      avformat_close_input(&fmt_ctx);
   
      
   mmal_port_disable(decoder->input[0]);
   mmal_port_disable(decoder->output[0]);
   mmal_port_disable(decoder->control); 
      
   mmal_port_flush(decoder->input[0]);
   mmal_port_flush(decoder->output[0]);
   mmal_port_flush(decoder->control); 
      
      
   if (decoder)
      mmal_component_destroy(decoder);
   if (pool_in)
      mmal_pool_destroy(pool_in);
   if (pool_out)
      mmal_pool_destroy(pool_out);
   if (context.queue)
      mmal_queue_destroy(context.queue);
    
     
}

[6by9]

So I got this working. I am getting output buffers in the correct size. The stumbling blocks were:

1. The sps and pps data must be sent in band as the first packet to the decoder. I had assumed that setting the input format's extradata fields were enough but that was not the case.

extradata should work and I thought was used by either OMXPlayer or Kodi. It's not one I really want to expend a lot of time investigating though.

2. The input and output ports have to be enabled AFTER the input and output pools are created. Otherwise you get an ENOMEM.

AFAIK That shouldn't be required, although I do tend to always create pools first so that everything is ready before enabling processing.

A question for the devs.
1. Do I need to worry about splitting packets for the encoder and marking the first packet as MMAL_BUFFER_HEADER_FLAG_FRAME_START and the last packet as MMAL_BUFFER_HEADER_FLAG_FRAME_END? I have not encountered this situation. I expect it might happen when the resolution of the video frame is high enough that the packet size is greater than input buffer size.

I assume that is a typo and you mean decoder (not encoder). (The encoder always requires full video frames to encode)
The decoder is quite happy with a random lump of an ES with startcodes, but if you provide frame end flags then it will reduce the latency by kicking the bitstream decoder earlier.

[cmisip]

Thanks 6by9. I did mean decoder but I was thinking about the encoder as well. I really appreciate the time you spend answering these questions.

Chris

[cmisip]

Just an FYI for anybody else trying to write something like this. Order matters very much with setting up the mmal components. I spent the last few hours trying to figure out why the video encoder component was failing on port format commit of the output port with an EINVAL error. The reason was I enabled the input_port before I called output port format commit. Hope this saves somebody some time figuring this out. This problem was only seen with the video encoder though. Curiosly, vc.ril.isp and h264 decoder did not have any errors when input port is enabled before output port format commit.

Chris

Code: Select all

//ENCODER---------------------------------------------------------------------------------------------
   
   status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder);
   CHECK_STATUS(status, "failed to create encoder");
   
     

   
   // CONTROL PORT SETTINGS
   encoder->control->userdata = (MMAL_PORT_USERDATA_T *)&context;
   status = mmal_port_enable(encoder->control, control_callback);
   CHECK_STATUS(status, "failed to enable control port");
   
   
   //INPUT PORT SETTINGS
   format_in = encoder->input[0]->format;
   format_in->type = MMAL_ES_TYPE_VIDEO;
   format_in->encoding = MMAL_ENCODING_I420;
   
   format_in->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_in->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_in->es->video.frame_rate.num = 30;
   format_in->es->video.frame_rate.den = 1;
   format_in->es->video.par.num = 1;
   format_in->es->video.par.den = 1;
   format_in->es->video.crop.width = iwidth;
   format_in->es->video.crop.height = iheight;
   
   
   display_format(&encoder->input[0],&format_in);
   
   encoder->input[0]->buffer_num = encoder->input[0]->buffer_num_recommended;
   encoder->input[0]->buffer_size = encoder->input[0]->buffer_size_recommended;
   fprintf(stderr,"encoder INPUT BUFFER SIZE %d NUM %d\n",encoder->input[0]->buffer_size,encoder->input[0]->buffer_num_recommended);
   
   status = mmal_port_format_commit(encoder->input[0]);
   CHECK_STATUS(status, "failed to commit encoder input format"); 
   
   //FINALIZE
   //status = mmal_port_enable(encoder->input[0], input_callback);  //DONT DO IT HERE.
   //CHECK_STATUS(status, "failed to enable input port");	
   
   pool_ine = mmal_pool_create(encoder->input[0]->buffer_num,
                              encoder->input[0]->buffer_size);
   

   // Store a reference to our context in each port (will be used during callbacks) */
   encoder->input[0]->userdata = (MMAL_PORT_USERDATA_T *)&context;
    
   
   //OUTPUT PORT SETTINGS
   format_out = encoder->output[0]->format;
   
   format_out->type = MMAL_ES_TYPE_VIDEO;
   format_out->encoding = MMAL_ENCODING_H264;
   
   format_out->es->video.width = VCOS_ALIGN_UP(iwidth, 32);
   format_out->es->video.height = VCOS_ALIGN_UP(iheight,16);
   format_out->es->video.frame_rate.num = 30;
   format_out->es->video.frame_rate.den = 1;
   format_out->es->video.par.num = 1; 
   format_out->es->video.par.den = 1;
   format_out->es->video.crop.width = iwidth;
   format_out->es->video.crop.height = iheight;
   
   
   display_format(&encoder->output[0],&format_out);
     
  
   encoder->output[0]->buffer_num = encoder->output[0]->buffer_num_min;
   encoder->output[0]->buffer_size = encoder->output[0]->buffer_size_min; 
   fprintf(stderr,"encoder OUTPUT BUFFER SIZE %d NUM %d\n",encoder->output[0]->buffer_size,encoder->output[0]->buffer_num_recommended);
   
   
   //FINALIZE
   
   status = mmal_port_format_commit(encoder->output[0]);
   CHECK_STATUS(status, "failed to commit encoder output format"); 
   
   status = mmal_port_parameter_set_boolean(encoder->output[0], MMAL_PARAMETER_VIDEO_ENCODE_INLINE_VECTORS, 1);
   CHECK_STATUS(status,"failed to request inline motion vectors from mmal encoder");
   
   pool_oute = mmal_pool_create(encoder->output[0]->buffer_num,
                               encoder->output[0]->buffer_size);
     
   context.equeue = mmal_queue_create();                            
   encoder->output[0]->userdata = (MMAL_PORT_USERDATA_T *)&context; 
   
   
   status = mmal_port_enable(encoder->input[0], input_callback);   //DO IT HERE INSTEAD, OR YOU GET EINVAL ON OUTPUT PORT FORMAT COMMIT
   CHECK_STATUS(status, "failed to enable input port");	
   status = mmal_port_enable(encoder->output[0], output_callback);
   CHECK_STATUS(status, "failed to enable output port"); 
   
   status = mmal_component_enable(encoder);
   CHECK_STATUS(status, "failed to enable encoder output format");
   
   
   
  

[6by9]

I thought that bit with video_encode was documented in the IL component docs (https://github.com/raspberrypi/firmware ... components, or hosted at http://www.jvcref.com/files/PI/document ... omponents/. I'm trying to get a copy hosted on the Pi website at the moment. He also has the MMAL docs from the doxygen markup at http://www.jvcref.com/files/PI/document ... 2015/html/).
As MMAL reuses the IL components, almost all of the IL documentation is relevant to MMAL as well.

video_encode is the only exception I can think of to being able to set port definitions/formats with ports other enabled. The comment in the source is

Code: Select all

         // NOTE: We DON'T allow the output port format to be set if the input port is active,
         //      as when in the data stream this change should take effect is ambiguous as we have an unknown
         //      amount of raw data buffered inside the component. By forcing the input port to be taken 
         //      disabled first we make this well defined.

This is mainly as the encoder has to do a format conversion on the frame before it can be encoded, and IIRC it returns the buffer once the conversion has occured, not the full encode.
Technically the ISP component can also have a queue of frames on the input port, but that is all totally controlled by the client as the input frames are only returned when the frame has been processed.
With video decode, any change to the output port format only refers to the format of new frames (you can't change resolution or similar), therefore there is no ambiguity there.

[cmisip]

I was able to get parallel components of video encoder and video resizer ( vc.ril.isp for format conversion only ) up and running. The decoder output is sent to the encoder and the resizer in parallel. I did not want to use the splitter because I wanted to save bandwidth.

As per your recommendation, the use of MMAL_BUFFER_HEADER_FLAG_FRAME_END should result in the decoding process to start earlier. I assumed that the use of this flag also required that format_in->flag be set.

Code: Select all

 format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;

As the input packet to the decoder will always be a full NAL unit, I set this with each input buffer. I hope that is correct.

Code: Select all

buffer->flags|=MMAL_BUFFER_HEADER_FLAG_FRAME_START;
buffer->flags|=MMAL_BUFFER_HEADER_FLAG_FRAME_END;

Thanks,
Chris

[6by9]

I can't remember if MMAL_ES_FORMAT_FLAG_FRAMED is actually needed - I don't think the decoder cares, but will always react to MMAL_BUFFER_HEADER_FLAG_FRAME_END.
MMAL_BUFFER_HEADER_FLAG_FRAME_START isn't going to do anything useful, but you are correct if you set it. If you remember that most of this maps down on to OpenMax IL components, IL has no FRAME_START flag, therefore it gets lost for all those components.

[cmisip]

Regarding the SPS and PPS data, it does not need to be sent in band as I thought. It is enough to set the extradata field. I decided to turn it off in zmc and it still worked. Must be some confusion in my earlier test programs.

Thanks,
Chris

[cmisip]

I disabled zero copy for now. I updated the code to shift some computational overhead over to the capture process. In particular, the capture process is now doing the motion detection and sending the alarm_pixels over to the analyse process via shared mem. Since this is happening in the capture process, it can guess when the analyze process might need jpeg frames for writing to disk, so the capture process using libjpeg-turbo preencodes the jpeg frames and sends them via shared mem. The capture process is also now able to save h264 packets to an mp4 file. Seems to be working fine except when I adjust the resolution to use a higher resolution rtsp stream, I get the Error 7 again. Any ideas what might be causing this? I posted the logs here for reference.

Thanks,
Chris

Code: Select all

Sep 01 23:27:21 zoneminder zma_m1[15646]: INF [zma_m1] [In mode 4/1, warming up]
Sep 01 23:27:21 zoneminder zmc_m1[15647]: INF [zmc_m1] [Starting Capture version 1.31.1]
Sep 01 23:27:21 zoneminder zmc_m1[15647]: INF [zmc_m1] [Priming capture from rtsp://mysv3c.chris.net/11]
Sep 01 23:27:22 zoneminder zmc_m1[15647]: INF [zmc_m1] [Stream open rtsp://mysv3c.chris.net/11]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Width 1280, Height 720, Codec->width 1280, Codec->height 720]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Decoder stats: 0, 0]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Decoder extradata size 23
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_decode:in:0(H264)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: H264
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 23, 0x616bf68
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_decode:in:0(H264) BUFFER SIZE 81920 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_decode:out:0(I420)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_decode:out:0(I420) BUFFER SIZE 1382400 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Encoder stats: 0, 0]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_encode:in:0(I420)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_encode:in:0(I420) BUFFER SIZE 1382400 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_encode:out:0(H264)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: H264
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.video_encode:out:0(H264) BUFFER SIZE 65536 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Resizer stats: 0, 0]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.isp:in:0(I420)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.isp:in:0(I420) BUFFER SIZE 1382400 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.isp:out:0(RGB3)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ type: 3, fourcc: RGB3
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [ width: 1280, height: 720, (0,0,1280,720)
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [PORT vc.ril.isp:out:0(RGB3) BUFFER SIZE 2764800 NUM 1
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Setting up the motion vector mask with numblocks 3600]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: INF [zmc_m1] [Done setting up zone vector mask]
Sep 01 23:27:26 zoneminder zmc_m1[15647]: WAR [zmc_m1] [ERROR: PORT: vc.ril.video_decode:ctr:0 Type: 7
Sep 01 23:27:26 zoneminder zmc_m1[15647]: WAR [zmc_m1] [ERROR: PORT: vc.ril.video_decode:ctr:0 Type: 7

At this point I reverted to 640x360 and everything is fine at the lower resolution. 


Sep 01 23:28:25 zoneminder zmdc[15241]: INF ['zma -m 1' sending stop to pid 15646 at 18/09/01 23:28:25]
Sep 01 23:28:25 zoneminder zma_m1[15646]: INF [zma_m1] [Got signal 15 (Terminated), exiting]
Sep 01 23:28:25 zoneminder zma_m1[15646]: ERR [zma_m1] [Got signal 6 (Aborted), crashing]
Sep 01 23:28:25 zoneminder zmdc[15241]: ERR ['zma -m 1' exited abnormally, exit status 6]
Sep 01 23:28:26 zoneminder zmdc[15241]: INF ['zmc -m 1' sending stop to pid 15647 at 18/09/01 23:28:25]
Sep 01 23:28:32 zoneminder zmdc[15241]: WAR ['zmc -m 1' has not stopped at 18/09/01 23:28:32. Sending KILL to pid 15647]
Sep 01 23:28:32 zoneminder zmdc[15241]: INF ['zmc -m 1' crashed, signal 8]
Sep 01 23:28:33 zoneminder zmdc[15241]: INF ['zmc -m 1' starting at 18/09/01 23:28:33, pid = 15674]
Sep 01 23:28:33 zoneminder zmdc[15674]: INF ['zmc -m 1' started at 18/09/01 23:28:33]
Sep 01 23:28:33 zoneminder zmdc[15241]: INF ['zma -m 1' starting at 18/09/01 23:28:33, pid = 15678]
Sep 01 23:28:33 zoneminder zmdc[15678]: INF ['zma -m 1' started at 18/09/01 23:28:33]
Sep 01 23:28:33 zoneminder zmdc[15241]: ERR ['zmc -m 1' exited abnormally, exit status 255]
Sep 01 23:28:33 zoneminder zmdc[15241]: INF [Starting pending process, zmc -m 1]
Sep 01 23:28:33 zoneminder zmdc[15241]: INF ['zmc -m 1' starting at 18/09/01 23:28:33, pid = 15679]
Sep 01 23:28:33 zoneminder zmdc[15679]: INF ['zmc -m 1' started at 18/09/01 23:28:33]
Sep 01 23:28:34 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:34 zoneminder zmc_m1[15679]: INF [zmc_m1] [Starting Capture version 1.31.1]
Sep 01 23:28:34 zoneminder zmc_m1[15679]: INF [zmc_m1] [Priming capture from rtsp://mysv3c.chris.net/12]
Sep 01 23:28:35 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:35 zoneminder zmc_m1[15679]: INF [zmc_m1] [Stream open rtsp://mysv3c.chris.net/12]
Sep 01 23:28:36 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:37 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:38 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:39 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:40 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:41 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Width 640, Height 360, Codec->width 640, Codec->height 360]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Decoder stats: 0, 0]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Decoder extradata size 23
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_decode:in:0(H264)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: H264
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 23, 0x60f0238
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_decode:in:0(H264) BUFFER SIZE 81920 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_decode:out:0(I420)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_decode:out:0(I420) BUFFER SIZE 353280 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Encoder stats: 0, 0]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_encode:in:0(I420)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_encode:in:0(I420) BUFFER SIZE 353280 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_encode:out:0(H264)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: H264
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.video_encode:out:0(H264) BUFFER SIZE 65536 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Resizer stats: 0, 0]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.isp:in:0(I420)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: I420
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.isp:in:0(I420) BUFFER SIZE 353280 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.isp:out:0(RGB3)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ type: 3, fourcc: RGB3
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ bitrate: 0, framed: 0
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ extra data: 0, (nil)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [ width: 640, height: 368, (0,0,640,360)
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [PORT vc.ril.isp:out:0(RGB3) BUFFER SIZE 706560 NUM 1
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [---------------------------------------------------
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Setting up the motion vector mask with numblocks 900]
Sep 01 23:28:41 zoneminder zmc_m1[15679]: INF [zmc_m1] [Done setting up zone vector mask]
Sep 01 23:28:42 zoneminder zma_m1[15678]: WAR [zma_m1] [Waiting for capture daemon]

Normal operation at this point forward. 

[cmisip]

You know what. I think the decoder input buffer size does not scale up properly with the higher resolution rtsp stream. It seems decoder->input[0]->buffer_size_recommended is not adequate for the job. It stays at 81920 despite the increased resolution.

Ffmpeg's mmal_dec uses 512*1024 for input buffer size regardless of the input resolution. Is there a way to programmatically derive an optimal buffer size? I don't want to use too much memory if I don't have to. I can also report that the system runs with just one input buffer.

Thanks,
Chris

[6by9]

Encoded buffer size is the elephant in the room that no one ever seems to really deal with. There is no formula for max buffer size.
MMAL and IL both support fragmenting the encoded frame over multiple buffers, hence the frame_end flag. They are happy with multiple 81920 byte fragments to a frame, hence why they don't change the buffer size.
V4L2 and Android both ignore the issue, but require that a frame fits within the buffer. When working with Android encode we worked on 512kB for <1080p, and 768kB for 1080p.
When I was looking at ffmpeg for writing to a container it required full frames but there appeared to be ways to extend the buffer should the initial allocation be too small.