/* * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "Conv2DOperationConverter.h" #include #include "OperationConverterResolver.h" #include "SubGraphContext.h" namespace android { namespace nn { Result> Conv2DOperationConverter::getConv2DInputs( const Operation& operation, SubGraphContext* context) const { NN_RET_CHECK(isOperandConstant( context->getSubgraph()->operands[operation.inputs[kFilterTensorIdx]])); NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kInputTensorIdx])); // TFLite does not support asymmetric tensors for convolution filters NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kFilterTensorIdx], true /* makeSymmetric */)); NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kBiasTensorIdx])); std::vector inputs{ context->getTensorIdxFromOperandIdx(operation.inputs[kInputTensorIdx]), context->getTensorIdxFromOperandIdx(operation.inputs[kFilterTensorIdx]), context->getTensorIdxFromOperandIdx(operation.inputs[kBiasTensorIdx])}; return inputs; } Result> Conv2DOperationConverter::getConv2DOutputs( const Operation& operation, SubGraphContext* context) const { NN_TRY(context->createTensorFlatbufferFromOperand(operation.outputs[kOutputTensorIdx])); std::vector outputs{ context->getTensorIdxFromOperandIdx(operation.outputs[kOutputTensorIdx])}; return outputs; } Result Conv2DOperationConverter::decomposeExplicitPadding(const Operation& operation, SubGraphContext* context) const { const Model::Subgraph* subgraph = context->getSubgraph(); const Operand& inputOperand = subgraph->operands[operation.inputs[0]]; // add opcode for PAD if it does not exist yet uint32_t opCodeIdx = context->addOpCode(OperationType::PAD); // pad options auto padOptionsFlatbuffer = tflite::CreatePadOptions(context->getBuilder()); // check to make sure padding Operands are constants const Operand& frontWidthPaddingOperand = subgraph->operands[operation.inputs[3]]; const Operand& backWidthPaddingOperand = subgraph->operands[operation.inputs[4]]; const Operand& frontHeightPaddingOperand = subgraph->operands[operation.inputs[5]]; const Operand& backHeightPaddingOperand = subgraph->operands[operation.inputs[6]]; NN_RET_CHECK(isOperandConstant(frontWidthPaddingOperand)); NN_RET_CHECK(isOperandConstant(backWidthPaddingOperand)); NN_RET_CHECK(isOperandConstant(frontHeightPaddingOperand)); NN_RET_CHECK(isOperandConstant(backHeightPaddingOperand)); // get padding params int32_t frontHeightPadding = context->getConstantScalar(frontHeightPaddingOperand); int32_t backHeightPadding = context->getConstantScalar(backHeightPaddingOperand); int32_t frontWidthPadding = context->getConstantScalar(frontWidthPaddingOperand); int32_t backWidthPadding = context->getConstantScalar(backWidthPaddingOperand); // build padding buffer const Dimensions& dims = inputOperand.dimensions; int numDimensionsInput = static_cast(dims.size()); std::vector paddingData(numDimensionsInput * 2, 0); paddingData[2] = frontHeightPadding; paddingData[3] = backHeightPadding; paddingData[4] = frontWidthPadding; paddingData[5] = backWidthPadding; uint32_t paddingBufferIdx = context->addBufferFromData( reinterpret_cast(paddingData.data()), paddingData.size() * sizeof(int32_t)); // create new tensor for padding std::vector padShape{numDimensionsInput, 2}; auto padTensor = tflite::CreateTensorDirect(context->getBuilder(), &padShape /* shape */, tflite::TensorType::TensorType_INT32 /* type */, paddingBufferIdx /* buffer */); int padTensorIdx = context->addTensorFlatbuffer(padTensor); // add inputs for padding operation std::vector padInputs = {context->getTensorIdxFromOperandIdx(operation.inputs[0]), padTensorIdx}; // get dimensions of output of pad operation std::vector padToConv2dShape(dims.begin(), dims.end()); // keep unknown height and width dimensions unknown padToConv2dShape[1] = padToConv2dShape[1] != 0 ? frontHeightPadding + padToConv2dShape[1] + backHeightPadding : -1; padToConv2dShape[2] = padToConv2dShape[2] != 0 ? frontWidthPadding + padToConv2dShape[2] + backWidthPadding : -1; replaceZeroDimensions(&padToConv2dShape); // build quantization parameters std::vector scaleVector{inputOperand.scale}; std::vector zeroPointVector{inputOperand.zeroPoint}; // min and max used to convert TFLite models to TF models, so it is unused in this case and can // be set to 0 std::vector minVector{0}; std::vector maxVector{0}; auto quantizationParams = tflite::CreateQuantizationParametersDirect( context->getBuilder(), &minVector /* min */, &maxVector /* max */, &scaleVector /* scale */, &zeroPointVector /* zero_point */, tflite::QuantizationDetails::QuantizationDetails_NONE /* details_type */); // create new tensor to be output of pad & input for conv2d auto padToConv2dTensor = tflite::CreateTensorDirect( context->getBuilder(), &padToConv2dShape /* shape */, NN_TRY(getTensorFlatbufferOperandType(inputOperand.type)) /* type */, 0 /* buffer */, 0 /* name */, quantizationParams /* quantization */); int padToConv2dTensorIdx = context->addTensorFlatbuffer(padToConv2dTensor); // set output for padding operation and add to operators std::vector padOutputs{padToConv2dTensorIdx}; OperatorFlatbuffer padOp = tflite::CreateOperatorDirect( context->getBuilder(), opCodeIdx, &padInputs, &padOutputs, tflite::BuiltinOptions::BuiltinOptions_PadOptions, padOptionsFlatbuffer.Union()); context->addOperatorFlatbuffer(padOp); // Return tensor index of pad output created return padToConv2dTensorIdx; } Result Conv2DOperationConverter::convert(const Operation& operation, SubGraphContext* context) const { const Model::Subgraph* subgraph = context->getSubgraph(); // add opcode for CONV_2D if not added yet uint32_t opCodeIdx = context->addOpCode(OperationType::CONV_2D); // if there are less than 8 inputs or the input at the 7th index is a BOOL, there is implicit // padding bool isImplicitPadding = false; if (operation.inputs.size() < 8 || subgraph->operands[operation.inputs[7]].type == OperandType::BOOL) { isImplicitPadding = true; } std::vector inputs = NN_TRY(getConv2DInputs(operation, context)); std::vector outputs = NN_TRY(getConv2DOutputs(operation, context)); // if explicit padding, we need to decompose the operation to a separate padding op and a conv2d // op if (!isImplicitPadding) { auto padOpIdx = NN_TRY(decomposeExplicitPadding(operation, context)); inputs[0] = padOpIdx; } int baseOptionsIdx = 4; tflite::Padding padding; if (isImplicitPadding) { const Operand& paddingTypeOperand = subgraph->operands[operation.inputs[3]]; NN_RET_CHECK(isOperandConstant(paddingTypeOperand)); int32_t paddingType = context->getConstantScalar(paddingTypeOperand); padding = getTFLitePadding(paddingType); } else { padding = tflite::Padding::Padding_VALID; baseOptionsIdx = 7; } // check if stride and activation Operands are constant const Operand& strideWOperand = subgraph->operands[operation.inputs[baseOptionsIdx + kStrideWOffset]]; const Operand& strideHOperand = subgraph->operands[operation.inputs[baseOptionsIdx + kStrideHOffset]]; const Operand& activationOperand = subgraph->operands[operation.inputs[baseOptionsIdx + kActivationOffset]]; NN_RET_CHECK(isOperandConstant(strideWOperand)); NN_RET_CHECK(isOperandConstant(strideHOperand)); NN_RET_CHECK(isOperandConstant(activationOperand)); // get strides and activation int32_t strideW = context->getConstantScalar(strideWOperand); int32_t strideH = context->getConstantScalar(strideHOperand); FusedActivationFunc activation = static_cast( context->getConstantScalar(activationOperand)); // check for nchw int isNchwIdx = baseOptionsIdx + kIsNchwOffset; if (operation.inputs.size() > static_cast(isNchwIdx)) { const Operand& isNchwOperand = subgraph->operands[operation.inputs[isNchwIdx]]; NN_RET_CHECK(isOperandConstant(isNchwOperand)); bool isNchw = context->getConstantScalar(isNchwOperand); NN_RET_CHECK(!isNchw) << "TFLite does not support NCHW formatted input tensors"; } // dilations int dilationWIdx = baseOptionsIdx + kDilationWOffset; int dilationHIdx = baseOptionsIdx + kDilationHOffset; // default dilation factors are 1 int32_t dilationW = 1; int32_t dilationH = 1; if (operation.inputs.size() > static_cast(dilationWIdx)) { const Operand& dilationWOperand = subgraph->operands[operation.inputs[dilationWIdx]]; NN_RET_CHECK(isOperandConstant(dilationWOperand)); dilationW = context->getConstantScalar(dilationWOperand); } if (operation.inputs.size() > static_cast(dilationHIdx)) { const Operand& dilationHOperand = subgraph->operands[operation.inputs[dilationHIdx]]; NN_RET_CHECK(isOperandConstant(dilationHOperand)); dilationH = context->getConstantScalar(dilationHOperand); } flatbuffers::Offset optionsFlatbuffer = tflite::CreateConv2DOptions( context->getBuilder(), padding, strideW, strideH, NN_TRY(getTfliteActivation(activation)) /* fused_activation_function */, dilationW, dilationH); auto operatorFlatbuffer = tflite::CreateOperatorDirect( context->getBuilder() /* builder */, opCodeIdx /* opcode_index */, &inputs /* inputs */, &outputs /* outputs */, tflite::BuiltinOptions::BuiltinOptions_Conv2DOptions /* builtin_options_type */, optionsFlatbuffer.Union() /* builtin_options */); context->addOperatorFlatbuffer(operatorFlatbuffer); return {}; } NN_REGISTER_OPERATION_CONVERTER(CONV_2D, Conv2DOperationConverter); } // namespace nn } // namespace android