Signed-off-by: Ting Fu <ting...@intel.com> --- libavfilter/dnn/dnn_backend_native.c | 74 +++++++++++++++---- libavfilter/dnn/dnn_backend_native.h | 5 ++ .../dnn/dnn_backend_native_layer_avgpool.c | 2 +- .../dnn/dnn_backend_native_layer_avgpool.h | 2 +- .../dnn/dnn_backend_native_layer_conv2d.c | 2 +- .../dnn/dnn_backend_native_layer_conv2d.h | 2 +- .../dnn_backend_native_layer_depth2space.c | 2 +- .../dnn_backend_native_layer_depth2space.h | 2 +- .../dnn/dnn_backend_native_layer_mathbinary.c | 10 ++- .../dnn/dnn_backend_native_layer_mathbinary.h | 2 +- .../dnn/dnn_backend_native_layer_mathunary.c | 10 ++- .../dnn/dnn_backend_native_layer_mathunary.h | 2 +- .../dnn/dnn_backend_native_layer_maximum.c | 2 +- .../dnn/dnn_backend_native_layer_maximum.h | 2 +- .../dnn/dnn_backend_native_layer_pad.c | 2 +- .../dnn/dnn_backend_native_layer_pad.h | 2 +- libavfilter/dnn/dnn_backend_native_layers.h | 2 +- 17 files changed, 90 insertions(+), 35 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index adc652a2c4..6ddffa54af 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -28,15 +28,30 @@ #include "dnn_backend_native_layer_conv2d.h" #include "dnn_backend_native_layers.h" +static const AVClass dnn_native_class = { + .class_name = "dnn_native", + .item_name = av_default_item_name, + .option = NULL, + .version = LIBAVUTIL_VERSION_INT, + .category = AV_CLASS_CATEGORY_FILTER, +}; + +NetworkContext network_ctx = { + .class = &dnn_native_class, +}; + static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; + NetworkContext *ctx = network->log_ctx; for (int i = 0; i < network->operands_num; ++i) { DnnOperand *oprd = &network->operands[i]; if (strcmp(oprd->name, input_name) == 0) { - if (oprd->type != DOT_INPUT) + if (oprd->type != DOT_INPUT) { + av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name); return DNN_ERROR; + } input->dt = oprd->data_type; av_assert0(oprd->dims[0] == 1); input->height = oprd->dims[1]; @@ -47,30 +62,37 @@ static DNNReturnType get_input_native(void *model, DNNData *input, const char *i } // do not find the input operand + av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name); return DNN_ERROR; } static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; + NetworkContext *ctx = network->log_ctx; DnnOperand *oprd = NULL; - if (network->layers_num <= 0 || network->operands_num <= 0) + if (network->layers_num <= 0 || network->operands_num <= 0) { + av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n"); return DNN_ERROR; + } /* inputs */ for (int i = 0; i < network->operands_num; ++i) { oprd = &network->operands[i]; if (strcmp(oprd->name, input_name) == 0) { - if (oprd->type != DOT_INPUT) + if (oprd->type != DOT_INPUT) { + av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name); return DNN_ERROR; + } break; } oprd = NULL; } - - if (!oprd) + if (!oprd) { + av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name); return DNN_ERROR; + } oprd->dims[0] = 1; oprd->dims[1] = input->height; @@ -79,11 +101,15 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const av_freep(&oprd->data); oprd->length = calculate_operand_data_length(oprd); - if (oprd->length <= 0) + if (oprd->length <= 0) { + av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n"); return DNN_ERROR; + } oprd->data = av_malloc(oprd->length); - if (!oprd->data) + if (!oprd->data) { + av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n"); return DNN_ERROR; + } input->data = oprd->data; @@ -91,8 +117,10 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const network->nb_output = 0; av_freep(&network->output_indexes); network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes)); - if (!network->output_indexes) + if (!network->output_indexes) { + av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for output\n"); return DNN_ERROR; + } for (uint32_t i = 0; i < nb_output; ++i) { const char *output_name = output_names[i]; @@ -105,8 +133,10 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const } } - if (network->nb_output != nb_output) + if (network->nb_output != nb_output) { + av_log(ctx, AV_LOG_ERROR, "Output(s) name are not all set correctly\n"); return DNN_ERROR; + } return DNN_SUCCESS; } @@ -128,6 +158,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *optio int32_t layer; DNNLayerType layer_type; + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ return NULL; } @@ -171,6 +202,8 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *optio if (!network){ goto fail; } + + network->log_ctx = &network_ctx; model->model = (void *)network; avio_seek(model_file_context, file_size - 8, SEEK_SET); @@ -258,20 +291,29 @@ fail: DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; + NetworkContext *ctx = network->log_ctx; int32_t layer; uint32_t nb = FFMIN(nb_output, network->nb_output); - if (network->layers_num <= 0 || network->operands_num <= 0) + if (network->layers_num <= 0 || network->operands_num <= 0) { + av_log(ctx, AV_LOG_ERROR, "Error network layer number: %d\n", network->layers_num); return DNN_ERROR; - if (!network->operands[0].data) + } + if (!network->operands[0].data) { + av_log(ctx, AV_LOG_ERROR, "Empty network input data\n"); return DNN_ERROR; + } - for (layer = 0; layer < network->layers_num; ++layer){ + for (layer = 0; layer < network->layers_num; ++layer) { DNNLayerType layer_type = network->layers[layer].type; - layer_funcs[layer_type].pf_exec(network->operands, - network->layers[layer].input_operand_indexes, - network->layers[layer].output_operand_index, - network->layers[layer].params); + if (layer_funcs[layer_type].pf_exec(network->operands, + network->layers[layer].input_operand_indexes, + network->layers[layer].output_operand_index, + network->layers[layer].params, + network->log_ctx) == DNN_ERROR) { + av_log(ctx, AV_LOG_ERROR, "Failed to execuet network operands\n"); + return DNN_ERROR; + } } for (uint32_t i = 0; i < nb; ++i) { diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h index b455e44992..954a4187d1 100644 --- a/libavfilter/dnn/dnn_backend_native.h +++ b/libavfilter/dnn/dnn_backend_native.h @@ -106,8 +106,13 @@ typedef struct InputParams{ int height, width, channels; } InputParams; +typedef struct NetworkContext { + const AVClass *class; +} NetworkContext; + // Represents simple feed-forward convolutional network. typedef struct ConvolutionalNetwork{ + NetworkContext *log_ctx; Layer *layers; int32_t layers_num; DnnOperand *operands; diff --git a/libavfilter/dnn/dnn_backend_native_layer_avgpool.c b/libavfilter/dnn/dnn_backend_native_layer_avgpool.c index d745c35b4a..98d670f965 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_avgpool.c +++ b/libavfilter/dnn/dnn_backend_native_layer_avgpool.c @@ -56,7 +56,7 @@ int dnn_load_layer_avg_pool(Layer *layer, AVIOContext *model_file_context, int f } int dnn_execute_layer_avg_pool(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { float *output; int height_end, width_end, height_radius, width_radius, output_height, output_width, kernel_area; diff --git a/libavfilter/dnn/dnn_backend_native_layer_avgpool.h b/libavfilter/dnn/dnn_backend_native_layer_avgpool.h index 8e31ddb7c8..7c7623803a 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_avgpool.h +++ b/libavfilter/dnn/dnn_backend_native_layer_avgpool.h @@ -35,6 +35,6 @@ typedef struct AvgPoolParams{ int dnn_load_layer_avg_pool(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_avg_pool(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c index a2202e4073..1b898813a9 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c +++ b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c @@ -89,7 +89,7 @@ int dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int fil } int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { float *output; int32_t input_operand_index = input_operand_indexes[0]; diff --git a/libavfilter/dnn/dnn_backend_native_layer_conv2d.h b/libavfilter/dnn/dnn_backend_native_layer_conv2d.h index b240b7ef6b..8cee63e558 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_conv2d.h +++ b/libavfilter/dnn/dnn_backend_native_layer_conv2d.h @@ -37,5 +37,5 @@ typedef struct ConvolutionalParams{ int dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c index 2c8bddf23d..54fee37406 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c +++ b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c @@ -50,7 +50,7 @@ int dnn_load_layer_depth2space(Layer *layer, AVIOContext *model_file_context, in } int dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { float *output; const DepthToSpaceParams *params = (const DepthToSpaceParams *)parameters; diff --git a/libavfilter/dnn/dnn_backend_native_layer_depth2space.h b/libavfilter/dnn/dnn_backend_native_layer_depth2space.h index b2901e0141..6a6f52f722 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_depth2space.h +++ b/libavfilter/dnn/dnn_backend_native_layer_depth2space.h @@ -36,6 +36,6 @@ typedef struct DepthToSpaceParams{ int dnn_load_layer_depth2space(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c index dd42c329a9..413120d0bd 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c @@ -77,7 +77,7 @@ int dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, in } int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { const DnnOperand *input = &operands[input_operand_indexes[0]]; DnnOperand *output = &operands[output_operand_index]; @@ -91,11 +91,15 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope output->data_type = input->data_type; output->length = calculate_operand_data_length(output); - if (output->length <= 0) + if (output->length <= 0) { + av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n"); return DNN_ERROR; + } output->data = av_realloc(output->data, output->length); - if (!output->data) + if (!output->data) { + av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n"); return DNN_ERROR; + } dims_count = calculate_operand_dims_count(output); src = input->data; diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h index 0acf3b0ea0..b2a0105fe6 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h @@ -48,6 +48,6 @@ typedef struct DnnLayerMathBinaryParams{ int dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathunary.c b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c index 58ee0e9d3d..4758d147f2 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathunary.c +++ b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c @@ -53,7 +53,7 @@ int dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int } int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { const DnnOperand *input = &operands[input_operand_indexes[0]]; DnnOperand *output = &operands[output_operand_index]; @@ -67,11 +67,15 @@ int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_oper output->data_type = input->data_type; output->length = calculate_operand_data_length(output); - if (output->length <= 0) + if (output->length <= 0) { + av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n"); return DNN_ERROR; + } output->data = av_realloc(output->data, output->length); - if (!output->data) + if (!output->data) { + av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n"); return DNN_ERROR; + } dims_count = calculate_operand_dims_count(output); src = input->data; diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathunary.h b/libavfilter/dnn/dnn_backend_native_layer_mathunary.h index d6a61effd5..ba16c06912 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathunary.h +++ b/libavfilter/dnn/dnn_backend_native_layer_mathunary.h @@ -55,6 +55,6 @@ typedef struct DnnLayerMathUnaryParams{ int dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_maximum.c b/libavfilter/dnn/dnn_backend_native_layer_maximum.c index cdddfdd87b..98323abff3 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_maximum.c +++ b/libavfilter/dnn/dnn_backend_native_layer_maximum.c @@ -50,7 +50,7 @@ int dnn_load_layer_maximum(Layer *layer, AVIOContext *model_file_context, int fi } int dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { const DnnOperand *input = &operands[input_operand_indexes[0]]; DnnOperand *output = &operands[output_operand_index]; diff --git a/libavfilter/dnn/dnn_backend_native_layer_maximum.h b/libavfilter/dnn/dnn_backend_native_layer_maximum.h index c049c63fd8..b8c25da958 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_maximum.h +++ b/libavfilter/dnn/dnn_backend_native_layer_maximum.h @@ -39,6 +39,6 @@ typedef struct DnnLayerMaximumParams{ int dnn_load_layer_maximum(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.c b/libavfilter/dnn/dnn_backend_native_layer_pad.c index feaab001e8..01d1edcc6e 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_pad.c +++ b/libavfilter/dnn/dnn_backend_native_layer_pad.c @@ -76,7 +76,7 @@ static int after_get_buddy(int given, int border, LayerPadModeParam mode) } int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters) + int32_t output_operand_index, const void *parameters, NetworkContext *ctx) { int32_t before_paddings; int32_t after_paddings; diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.h b/libavfilter/dnn/dnn_backend_native_layer_pad.h index 18e05bdd5c..bf84924a68 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_pad.h +++ b/libavfilter/dnn/dnn_backend_native_layer_pad.h @@ -38,6 +38,6 @@ typedef struct LayerPadParams{ int dnn_load_layer_pad(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); #endif diff --git a/libavfilter/dnn/dnn_backend_native_layers.h b/libavfilter/dnn/dnn_backend_native_layers.h index b696e9c6fa..f6dcdc6ecb 100644 --- a/libavfilter/dnn/dnn_backend_native_layers.h +++ b/libavfilter/dnn/dnn_backend_native_layers.h @@ -25,7 +25,7 @@ #include "dnn_backend_native.h" typedef int (*LAYER_EXEC_FUNC)(DnnOperand *operands, const int32_t *input_operand_indexes, - int32_t output_operand_index, const void *parameters); + int32_t output_operand_index, const void *parameters, NetworkContext *ctx); typedef int (*LAYER_LOAD_FUNC)(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num); typedef struct LayerFunc { -- 2.17.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".