mirror of
https://github.com/Tencent/ncnn.git
synced 2026-10-17 04:41:47 +08:00
Layla
2533 lines
82 KiB
C++
2533 lines
82 KiB
C++
// Copyright 2019 BUG1989 (https://github.com/BUG1989/) Long-term support.
|
|
// Copyright 2019 JansonZhu (https://github.com/JansonZhu) Implemented the function of entropy calibration.
|
|
// Copyright 2021 Tencent
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
#ifdef _MSC_VER
|
|
#define _CRT_SECURE_NO_DEPRECATE
|
|
#endif
|
|
|
|
#include <float.h>
|
|
#include <limits.h>
|
|
#include <math.h>
|
|
#include <stdio.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#if defined(USE_NCNN_SIMPLEOCV)
|
|
#include "simpleocv.h"
|
|
#elif defined(USE_LOCAL_IMREADWRITE)
|
|
#include "imreadwrite.h"
|
|
#else
|
|
#include <opencv2/core/core.hpp>
|
|
#include <opencv2/highgui/highgui.hpp>
|
|
#endif
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
// npy format header
|
|
#include "npy.hpp"
|
|
|
|
// ncnn public header
|
|
#include "benchmark.h"
|
|
#include "cpu.h"
|
|
#include "net.h"
|
|
|
|
// ncnn private header
|
|
#include "layer/convolution.h"
|
|
#include "layer/convolutiondepthwise.h"
|
|
#include "layer/innerproduct.h"
|
|
#include "layer/gemm.h"
|
|
#include "layer/embed.h"
|
|
#include "layer/multiheadattention.h"
|
|
#include "layer/rnn.h"
|
|
#include "layer/lstm.h"
|
|
#include "layer/gru.h"
|
|
|
|
class QuantBlobStat
|
|
{
|
|
public:
|
|
QuantBlobStat()
|
|
{
|
|
threshold = 0.f;
|
|
absmax = 0.f;
|
|
percentile = 0.f;
|
|
zero_count = 0;
|
|
total = 0;
|
|
}
|
|
|
|
public:
|
|
float threshold;
|
|
float absmax;
|
|
float percentile;
|
|
uint64_t zero_count;
|
|
|
|
// ACIQ
|
|
int total;
|
|
|
|
// KL
|
|
std::vector<uint64_t> histogram;
|
|
std::vector<float> histogram_normed;
|
|
};
|
|
|
|
class QuantMHAStat
|
|
{
|
|
public:
|
|
ncnn::Mat q_weight_scales;
|
|
ncnn::Mat k_weight_scales;
|
|
ncnn::Mat v_weight_scales;
|
|
float out_weight_scale;
|
|
};
|
|
|
|
class QuantGemmStat
|
|
{
|
|
public:
|
|
QuantGemmStat()
|
|
: B_weight_scale(1.f)
|
|
{
|
|
}
|
|
|
|
ncnn::Mat A_weight_scales;
|
|
float B_weight_scale;
|
|
};
|
|
|
|
// rnn, gru, lstm
|
|
class QuantRecurrentStat
|
|
{
|
|
public:
|
|
ncnn::Mat weight_xc_scales;
|
|
ncnn::Mat weight_hc_scales;
|
|
};
|
|
|
|
class QuantNet : public ncnn::Net
|
|
{
|
|
public:
|
|
QuantNet();
|
|
|
|
std::vector<ncnn::Blob>& blobs;
|
|
std::vector<ncnn::Layer*>& layers;
|
|
|
|
public:
|
|
std::vector<std::vector<std::string> > listspaths;
|
|
std::vector<std::vector<float> > means;
|
|
std::vector<std::vector<float> > norms;
|
|
std::vector<std::vector<int> > shapes;
|
|
std::vector<int> type_to_pixels;
|
|
int quantize_num_threads;
|
|
int file_type;
|
|
bool use_calibration_dataset;
|
|
|
|
public:
|
|
int init();
|
|
void print_quant_info() const;
|
|
int save_table(const char* tablepath);
|
|
void initialize_static_weight_scales(bool initialize_conv_weight_scales = false);
|
|
int build_activation_histogram(int num_histogram_bins);
|
|
int quantize_KL();
|
|
int quantize_percentile(const std::vector<float>& percentiles);
|
|
int quantize_ACIQ();
|
|
int quantize_EQ();
|
|
|
|
public:
|
|
std::vector<int> input_blobs;
|
|
std::vector<int> conv_layers;
|
|
std::vector<int> conv_bottom_blobs;
|
|
std::vector<int> conv_top_blobs;
|
|
std::vector<int> embed_layers;
|
|
std::vector<int> gemm_layers;
|
|
std::vector<int> mha_layers;
|
|
std::vector<int> rnn_layers;
|
|
std::vector<int> lstm_layers;
|
|
std::vector<int> gru_layers;
|
|
|
|
// result
|
|
std::vector<QuantBlobStat> quant_blob_stats;
|
|
std::vector<ncnn::Mat> weight_scales;
|
|
std::vector<ncnn::Mat> bottom_blob_scales;
|
|
std::vector<float> embed_weight_scales;
|
|
std::vector<QuantGemmStat> gemm_stats;
|
|
std::vector<QuantMHAStat> mha_stats;
|
|
std::vector<QuantRecurrentStat> rnn_stats;
|
|
std::vector<QuantRecurrentStat> lstm_stats;
|
|
std::vector<QuantRecurrentStat> gru_stats;
|
|
};
|
|
|
|
QuantNet::QuantNet()
|
|
: blobs(mutable_blobs()), layers(mutable_layers())
|
|
{
|
|
quantize_num_threads = ncnn::get_cpu_count();
|
|
use_calibration_dataset = false;
|
|
}
|
|
|
|
int QuantNet::init()
|
|
{
|
|
// find all input layers
|
|
for (int i = 0; i < (int)layers.size(); i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[i];
|
|
if (layer->type == "Input")
|
|
{
|
|
input_blobs.push_back(layer->tops[0]);
|
|
}
|
|
}
|
|
|
|
// find all conv layers
|
|
for (int i = 0; i < (int)layers.size(); i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[i];
|
|
if (layer->type == "Convolution" || layer->type == "ConvolutionDepthWise" || layer->type == "InnerProduct")
|
|
{
|
|
conv_layers.push_back(i);
|
|
conv_bottom_blobs.push_back(layer->bottoms[0]);
|
|
conv_top_blobs.push_back(layer->tops[0]);
|
|
}
|
|
|
|
// find embed layers
|
|
else if (layer->type == "Embed")
|
|
{
|
|
embed_layers.push_back(i);
|
|
}
|
|
else if (layer->type == "Gemm")
|
|
{
|
|
gemm_layers.push_back(i);
|
|
}
|
|
|
|
// find all mha layers
|
|
else if (layer->type == "MultiHeadAttention")
|
|
{
|
|
mha_layers.push_back(i);
|
|
}
|
|
else if (layer->type == "RNN")
|
|
{
|
|
rnn_layers.push_back(i);
|
|
}
|
|
else if (layer->type == "LSTM")
|
|
{
|
|
lstm_layers.push_back(i);
|
|
}
|
|
else if (layer->type == "GRU")
|
|
{
|
|
gru_layers.push_back(i);
|
|
}
|
|
}
|
|
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
const int embed_layer_count = (int)embed_layers.size();
|
|
const int gemm_layer_count = (int)gemm_layers.size();
|
|
const int mha_layer_count = (int)mha_layers.size();
|
|
const int rnn_layer_count = (int)rnn_layers.size();
|
|
const int lstm_layer_count = (int)lstm_layers.size();
|
|
const int gru_layer_count = (int)gru_layers.size();
|
|
|
|
quant_blob_stats.resize(conv_bottom_blob_count);
|
|
weight_scales.resize(conv_layer_count);
|
|
bottom_blob_scales.resize(conv_bottom_blob_count);
|
|
embed_weight_scales.resize(embed_layer_count);
|
|
gemm_stats.resize(gemm_layer_count);
|
|
mha_stats.resize(mha_layer_count);
|
|
rnn_stats.resize(rnn_layer_count);
|
|
lstm_stats.resize(lstm_layer_count);
|
|
gru_stats.resize(gru_layer_count);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int QuantNet::save_table(const char* tablepath)
|
|
{
|
|
FILE* fp = fopen(tablepath, "wb");
|
|
if (!fp)
|
|
{
|
|
fprintf(stderr, "fopen %s failed\n", tablepath);
|
|
return -1;
|
|
}
|
|
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
const int embed_layer_count = (int)embed_layers.size();
|
|
const int gemm_layer_count = (int)gemm_layers.size();
|
|
const int mha_layer_count = (int)mha_layers.size();
|
|
const int rnn_layer_count = (int)rnn_layers.size();
|
|
const int lstm_layer_count = (int)lstm_layers.size();
|
|
const int gru_layer_count = (int)gru_layers.size();
|
|
|
|
fprintf(stdout, "param:%d\n", use_calibration_dataset ? conv_layer_count : 0);
|
|
|
|
for (int i = 0; use_calibration_dataset && i < conv_layer_count; i++)
|
|
{
|
|
const ncnn::Mat& weight_scale = weight_scales[i];
|
|
|
|
fprintf(fp, "%s_param_0 ", layers[conv_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_scale.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_scale[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
for (int i = 0; use_calibration_dataset && i < conv_bottom_blob_count; i++)
|
|
{
|
|
const ncnn::Mat& bottom_blob_scale = bottom_blob_scales[i];
|
|
|
|
fprintf(fp, "%s ", layers[conv_layers[i]]->name.c_str());
|
|
for (int j = 0; j < bottom_blob_scale.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", bottom_blob_scale[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", embed_layer_count);
|
|
for (int i = 0; i < embed_layer_count; i++)
|
|
{
|
|
fprintf(fp, "%s_param_0 ", layers[embed_layers[i]]->name.c_str());
|
|
fprintf(fp, "%f ", embed_weight_scales[i]);
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", gemm_layer_count);
|
|
for (int i = 0; i < gemm_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[gemm_layers[i]];
|
|
const ncnn::Gemm* gemm = (const ncnn::Gemm*)layer;
|
|
const QuantGemmStat& stat = gemm_stats[i];
|
|
|
|
if (gemm->constantA)
|
|
{
|
|
fprintf(fp, "%s_param_0 ", layer->name.c_str());
|
|
for (int j = 0; j < stat.A_weight_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", stat.A_weight_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
if (gemm->constantB)
|
|
{
|
|
fprintf(fp, "%s_param_1 ", layer->name.c_str());
|
|
fprintf(fp, "%f ", stat.B_weight_scale);
|
|
fprintf(fp, "\n");
|
|
}
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", mha_layer_count);
|
|
for (int i = 0; i < mha_layer_count; i++)
|
|
{
|
|
// q_weight
|
|
const ncnn::Mat q_weight_scales = mha_stats[i].q_weight_scales;
|
|
fprintf(fp, "%s_param_0 ", layers[mha_layers[i]]->name.c_str());
|
|
for (int j = 0; j < q_weight_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", q_weight_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
// k_weight
|
|
const ncnn::Mat k_weight_scales = mha_stats[i].k_weight_scales;
|
|
fprintf(fp, "%s_param_1 ", layers[mha_layers[i]]->name.c_str());
|
|
for (int j = 0; j < k_weight_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", k_weight_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
// v_weight
|
|
const ncnn::Mat v_weight_scales = mha_stats[i].v_weight_scales;
|
|
fprintf(fp, "%s_param_2 ", layers[mha_layers[i]]->name.c_str());
|
|
for (int j = 0; j < v_weight_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", v_weight_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
// out_weight
|
|
fprintf(fp, "%s_param_3 ", layers[mha_layers[i]]->name.c_str());
|
|
fprintf(fp, "%f ", mha_stats[i].out_weight_scale);
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", rnn_layer_count);
|
|
for (int i = 0; i < rnn_layer_count; i++)
|
|
{
|
|
const ncnn::Mat weight_xc_scales = rnn_stats[i].weight_xc_scales;
|
|
fprintf(fp, "%s_param_0 ", layers[rnn_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_xc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_xc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
const ncnn::Mat weight_hc_scales = rnn_stats[i].weight_hc_scales;
|
|
fprintf(fp, "%s_param_1 ", layers[rnn_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_hc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_hc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", lstm_layer_count);
|
|
for (int i = 0; i < lstm_layer_count; i++)
|
|
{
|
|
const ncnn::Mat weight_xc_scales = lstm_stats[i].weight_xc_scales;
|
|
fprintf(fp, "%s_param_0 ", layers[lstm_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_xc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_xc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
const ncnn::Mat weight_hc_scales = lstm_stats[i].weight_hc_scales;
|
|
fprintf(fp, "%s_param_1 ", layers[lstm_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_hc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_hc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fprintf(stdout, "param:%d\n", gru_layer_count);
|
|
for (int i = 0; i < gru_layer_count; i++)
|
|
{
|
|
const ncnn::Mat weight_xc_scales = gru_stats[i].weight_xc_scales;
|
|
fprintf(fp, "%s_param_0 ", layers[gru_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_xc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_xc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
|
|
const ncnn::Mat weight_hc_scales = gru_stats[i].weight_hc_scales;
|
|
fprintf(fp, "%s_param_1 ", layers[gru_layers[i]]->name.c_str());
|
|
for (int j = 0; j < weight_hc_scales.w; j++)
|
|
{
|
|
fprintf(fp, "%f ", weight_hc_scales[j]);
|
|
}
|
|
fprintf(fp, "\n");
|
|
}
|
|
|
|
fclose(fp);
|
|
|
|
fprintf(stderr, "ncnn int8 calibration table create success, best wish for your int8 inference has a low accuracy loss...\\(^0^)/...233...\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
void QuantNet::print_quant_info() const
|
|
{
|
|
if (!use_calibration_dataset)
|
|
return;
|
|
|
|
for (int i = 0; i < (int)conv_bottom_blobs.size(); i++)
|
|
{
|
|
const QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
float scale = 127 / stat.threshold;
|
|
|
|
if (stat.percentile == 0.f)
|
|
fprintf(stderr, "%-40s : max = %-15f threshold = %-15f scale = %-15f\n", layers[conv_layers[i]]->name.c_str(), stat.absmax, stat.threshold, scale);
|
|
else
|
|
fprintf(stderr, "%-40s : max = %-15f threshold = %-15f scale = %-15f percentile = %-15f\n", layers[conv_layers[i]]->name.c_str(), stat.absmax, stat.threshold, scale, stat.percentile);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Read npy file
|
|
* shape is input as [w,h,...]
|
|
* @return ncnn::Mat
|
|
*/
|
|
|
|
inline ncnn::Mat read_npy(const std::vector<int>& shape, const std::string& npypath)
|
|
{
|
|
npy::npy_data<float> d;
|
|
try
|
|
{
|
|
d = npy::read_npy<float>(npypath);
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
fprintf(stderr, "npy::read_npy exception: %s\n", e.what());
|
|
std::exit(EXIT_FAILURE);
|
|
}
|
|
|
|
std::vector<unsigned long> npy_shape = d.shape;
|
|
size_t dims = shape.size();
|
|
|
|
if (dims != npy_shape.size())
|
|
{
|
|
fprintf(stderr, "expect %d dims, but got: %d\n", (int)dims, (int)npy_shape.size());
|
|
std::exit(EXIT_FAILURE);
|
|
}
|
|
|
|
for (size_t i = 0; i < dims; ++i)
|
|
{
|
|
if (static_cast<unsigned long>(shape[i]) != npy_shape[dims - 1 - i])
|
|
{
|
|
fprintf(stderr, "shape mismatch!\n");
|
|
std::exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
switch (dims)
|
|
{
|
|
case 1:
|
|
return ncnn::Mat(shape[0], (void*)(d.data.data())).reshape(shape[0]).clone();
|
|
case 2:
|
|
return ncnn::Mat(shape[0] * shape[1], (void*)(d.data.data())).reshape(shape[0], shape[1]).clone();
|
|
case 3:
|
|
return ncnn::Mat(shape[0] * shape[1] * shape[2], (void*)(d.data.data())).reshape(shape[0], shape[1], shape[2]).clone();
|
|
case 4:
|
|
return ncnn::Mat(shape[0] * shape[1] * shape[2] * shape[3], (void*)(d.data.data())).reshape(shape[0], shape[1], shape[2], shape[3]).clone();
|
|
default:
|
|
fprintf(stderr, "dims:%d illegal!", (int)dims);
|
|
return ncnn::Mat();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Read and resize image
|
|
* shape is input as [w,h,...]
|
|
* if w and h both are given, image will be resized to exactly size.
|
|
* if w and h both are zero or negative, image will not be resized.
|
|
* if only h is zero or negative, image's width will scaled resize to w, keeping aspect ratio.
|
|
* if only w is zero or negative, image's height will scaled resize to h
|
|
* @return ncnn::Mat
|
|
*/
|
|
|
|
inline ncnn::Mat read_and_resize_image(const std::vector<int>& shape, const std::string& imagepath, int pixel_convert_type)
|
|
{
|
|
int target_w = shape[0];
|
|
int target_h = shape[1];
|
|
cv::Mat bgr = cv::imread(imagepath, 1);
|
|
if (target_h <= 0 && target_w <= 0)
|
|
{
|
|
return ncnn::Mat::from_pixels(bgr.data, pixel_convert_type, bgr.cols, bgr.rows);
|
|
}
|
|
if (target_h <= 0 || target_w <= 0)
|
|
{
|
|
float scale = 1.0;
|
|
if (target_h <= 0)
|
|
{
|
|
scale = 1.0 * bgr.cols / target_w;
|
|
target_h = int(1.0 * bgr.rows / scale);
|
|
}
|
|
if (target_w <= 0)
|
|
{
|
|
scale = 1.0 * bgr.rows / target_h;
|
|
target_w = int(1.0 * bgr.cols / scale);
|
|
}
|
|
}
|
|
return ncnn::Mat::from_pixels_resize(bgr.data, pixel_convert_type, bgr.cols, bgr.rows, target_w, target_h);
|
|
}
|
|
|
|
void QuantNet::initialize_static_weight_scales(bool initialize_conv_weight_scales)
|
|
{
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int embed_layer_count = (int)embed_layers.size();
|
|
const int gemm_layer_count = (int)gemm_layers.size();
|
|
const int mha_layer_count = (int)mha_layers.size();
|
|
const int rnn_layer_count = (int)rnn_layers.size();
|
|
const int lstm_layer_count = (int)lstm_layers.size();
|
|
const int gru_layer_count = (int)gru_layers.size();
|
|
|
|
if (initialize_conv_weight_scales && use_calibration_dataset)
|
|
{
|
|
// initialize conv weight scales
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[conv_layers[i]];
|
|
|
|
if (layer->type == "Convolution")
|
|
{
|
|
const ncnn::Convolution* convolution = (const ncnn::Convolution*)layer;
|
|
|
|
const int num_output = convolution->num_output;
|
|
const int kernel_w = convolution->kernel_w;
|
|
const int kernel_h = convolution->kernel_h;
|
|
const int dilation_w = convolution->dilation_w;
|
|
const int dilation_h = convolution->dilation_h;
|
|
const int stride_w = convolution->stride_w;
|
|
const int stride_h = convolution->stride_h;
|
|
|
|
const int weight_data_size_output = convolution->weight_data_size / num_output;
|
|
|
|
// int8 winograd F43 needs weight data to use 6bit quantization
|
|
// TODO proper condition for winograd 3x3 int8
|
|
bool quant_6bit = false;
|
|
if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
|
|
quant_6bit = true;
|
|
|
|
weight_scales[i].create(num_output);
|
|
|
|
for (int n = 0; n < num_output; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = convolution->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
if (quant_6bit)
|
|
{
|
|
weight_scales[i][n] = 31 / absmax;
|
|
}
|
|
else
|
|
{
|
|
weight_scales[i][n] = 127 / absmax;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (layer->type == "ConvolutionDepthWise")
|
|
{
|
|
const ncnn::ConvolutionDepthWise* convolutiondepthwise = (const ncnn::ConvolutionDepthWise*)layer;
|
|
|
|
const int group = convolutiondepthwise->group;
|
|
const int weight_data_size_output = convolutiondepthwise->weight_data_size / group;
|
|
|
|
std::vector<float> scales;
|
|
|
|
weight_scales[i].create(group);
|
|
|
|
for (int n = 0; n < group; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = convolutiondepthwise->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
weight_scales[i][n] = 127 / absmax;
|
|
}
|
|
}
|
|
|
|
if (layer->type == "InnerProduct")
|
|
{
|
|
const ncnn::InnerProduct* innerproduct = (const ncnn::InnerProduct*)layer;
|
|
|
|
const int num_output = innerproduct->num_output;
|
|
const int weight_data_size_output = innerproduct->weight_data_size / num_output;
|
|
|
|
weight_scales[i].create(num_output);
|
|
|
|
for (int n = 0; n < num_output; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = innerproduct->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
weight_scales[i][n] = 127 / absmax;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// initialize gemm weight scales
|
|
for (int i = 0; i < gemm_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[gemm_layers[i]];
|
|
const ncnn::Gemm* gemm = (const ncnn::Gemm*)layer;
|
|
QuantGemmStat& stat = gemm_stats[i];
|
|
|
|
if (gemm->constantA)
|
|
{
|
|
ncnn::Mat A_data = gemm->A_data;
|
|
int transA = gemm->transA;
|
|
if (transA == 1)
|
|
{
|
|
ncnn::Mat A_data_transposed(gemm->constantK * gemm->constantM);
|
|
for (int m = 0; m < gemm->constantM; m++)
|
|
{
|
|
float* ptr = (float*)A_data_transposed + m * gemm->constantK;
|
|
for (int k = 0; k < gemm->constantK; k++)
|
|
{
|
|
ptr[k] = gemm->A_data[k * gemm->constantM + m];
|
|
}
|
|
}
|
|
A_data = A_data_transposed;
|
|
}
|
|
|
|
stat.A_weight_scales.create(gemm->constantM);
|
|
for (int m = 0; m < gemm->constantM; m++)
|
|
{
|
|
float absmax = 0.f;
|
|
const float* ptr = (const float*)A_data + m * gemm->constantK;
|
|
for (int k = 0; k < gemm->constantK; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[k]));
|
|
}
|
|
stat.A_weight_scales[m] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
|
|
if (gemm->constantB)
|
|
{
|
|
const float* ptr = (const float*)gemm->B_data;
|
|
float absmax = 0.f;
|
|
const int b_data_size = gemm->B_data.w * gemm->B_data.h;
|
|
for (int j = 0; j < b_data_size; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[j]));
|
|
}
|
|
|
|
stat.B_weight_scale = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
|
|
// initialize embed weight scales
|
|
for (int i = 0; i < embed_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[embed_layers[i]];
|
|
const ncnn::Embed* embed = (const ncnn::Embed*)layer;
|
|
const float* ptr = embed->weight_data;
|
|
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < embed->weight_data.w; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[j]));
|
|
}
|
|
embed_weight_scales[i] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
|
|
// initialize mha weight scales
|
|
for (int i = 0; i < mha_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[mha_layers[i]];
|
|
const ncnn::MultiHeadAttention* mha = (const ncnn::MultiHeadAttention*)layer;
|
|
|
|
const int qdim = mha->weight_data_size / mha->embed_dim;
|
|
mha_stats[i].q_weight_scales.create(mha->embed_dim);
|
|
for (int j = 0; j < mha->embed_dim; j++)
|
|
{
|
|
float q_absmax = 0.f;
|
|
|
|
const float* q_ptr = (const float*)mha->q_weight_data + j * qdim;
|
|
for (int k = 0; k < qdim; k++)
|
|
{
|
|
q_absmax = std::max(q_absmax, (float)fabs(q_ptr[k]));
|
|
}
|
|
mha_stats[i].q_weight_scales[j] = q_absmax == 0.f ? 1.f : 127 / q_absmax;
|
|
}
|
|
|
|
const int kdim = mha->kdim;
|
|
mha_stats[i].k_weight_scales.create(mha->embed_dim);
|
|
for (int j = 0; j < mha->embed_dim; j++)
|
|
{
|
|
float k_absmax = 0.f;
|
|
|
|
const float* k_ptr = (const float*)mha->k_weight_data + j * kdim;
|
|
for (int k = 0; k < kdim; k++)
|
|
{
|
|
k_absmax = std::max(k_absmax, (float)fabs(k_ptr[k]));
|
|
}
|
|
mha_stats[i].k_weight_scales[j] = k_absmax == 0.f ? 1.f : 127 / k_absmax;
|
|
}
|
|
|
|
const int vdim = mha->vdim;
|
|
mha_stats[i].v_weight_scales.create(mha->embed_dim);
|
|
for (int j = 0; j < mha->embed_dim; j++)
|
|
{
|
|
float v_absmax = 0.f;
|
|
|
|
const float* v_ptr = (const float*)mha->v_weight_data + j * vdim;
|
|
for (int k = 0; k < vdim; k++)
|
|
{
|
|
v_absmax = std::max(v_absmax, (float)fabs(v_ptr[k]));
|
|
}
|
|
mha_stats[i].v_weight_scales[j] = v_absmax == 0.f ? 1.f : 127 / v_absmax;
|
|
}
|
|
|
|
const float* o_ptr = (const float*)mha->out_weight_data;
|
|
float o_absmax = 0.f;
|
|
for (int k = 0; k < mha->out_weight_data.w; k++)
|
|
{
|
|
o_absmax = std::max(o_absmax, (float)fabs(o_ptr[k]));
|
|
}
|
|
mha_stats[i].out_weight_scale = o_absmax == 0.f ? 1.f : 127 / o_absmax;
|
|
}
|
|
|
|
// initialize rnn weight scales
|
|
for (int i = 0; i < rnn_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[rnn_layers[i]];
|
|
const ncnn::RNN* rnn = (const ncnn::RNN*)layer;
|
|
|
|
const int num_directions = rnn->direction == 2 ? 2 : 1;
|
|
const int size = rnn->weight_data_size / num_directions / rnn->num_output;
|
|
|
|
rnn_stats[i].weight_xc_scales.create(rnn->num_output * num_directions);
|
|
rnn_stats[i].weight_hc_scales.create(rnn->num_output * num_directions);
|
|
|
|
for (int d = 0; d < num_directions; d++)
|
|
{
|
|
for (int q = 0; q < rnn->num_output; q++)
|
|
{
|
|
{
|
|
const float* weight_xc_ptr = rnn->weight_xc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < size; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_xc_ptr[j]));
|
|
}
|
|
rnn_stats[i].weight_xc_scales[d * rnn->num_output + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
|
|
{
|
|
const float* weight_hc_ptr = rnn->weight_hc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < rnn->num_output; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_hc_ptr[j]));
|
|
}
|
|
rnn_stats[i].weight_hc_scales[d * rnn->num_output + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// initialize lstm weight scales
|
|
for (int i = 0; i < lstm_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[lstm_layers[i]];
|
|
const ncnn::LSTM* lstm = (const ncnn::LSTM*)layer;
|
|
|
|
const int num_directions = lstm->direction == 2 ? 2 : 1;
|
|
const int size = lstm->weight_data_size / num_directions / lstm->hidden_size / 4;
|
|
|
|
lstm_stats[i].weight_xc_scales.create(lstm->hidden_size * 4 * num_directions);
|
|
lstm_stats[i].weight_hc_scales.create(lstm->hidden_size * 4 * num_directions);
|
|
|
|
for (int d = 0; d < num_directions; d++)
|
|
{
|
|
for (int q = 0; q < lstm->hidden_size * 4; q++)
|
|
{
|
|
{
|
|
const float* weight_xc_ptr = lstm->weight_xc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < size; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_xc_ptr[j]));
|
|
}
|
|
lstm_stats[i].weight_xc_scales[d * lstm->hidden_size * 4 + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
|
|
{
|
|
const float* weight_hc_ptr = lstm->weight_hc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < lstm->num_output; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_hc_ptr[j]));
|
|
}
|
|
lstm_stats[i].weight_hc_scales[d * lstm->hidden_size * 4 + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// initialize gru weight scales
|
|
for (int i = 0; i < gru_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[gru_layers[i]];
|
|
const ncnn::GRU* gru = (const ncnn::GRU*)layer;
|
|
|
|
const int num_directions = gru->direction == 2 ? 2 : 1;
|
|
const int size = gru->weight_data_size / num_directions / gru->num_output / 3;
|
|
|
|
gru_stats[i].weight_xc_scales.create(gru->num_output * 3 * num_directions);
|
|
gru_stats[i].weight_hc_scales.create(gru->num_output * 3 * num_directions);
|
|
|
|
for (int d = 0; d < num_directions; d++)
|
|
{
|
|
for (int q = 0; q < gru->num_output * 3; q++)
|
|
{
|
|
{
|
|
const float* weight_xc_ptr = gru->weight_xc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < size; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_xc_ptr[j]));
|
|
}
|
|
gru_stats[i].weight_xc_scales[d * gru->num_output * 3 + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
|
|
{
|
|
const float* weight_hc_ptr = gru->weight_hc_data.channel(d).row(q);
|
|
float absmax = 0.f;
|
|
for (int j = 0; j < gru->num_output; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_hc_ptr[j]));
|
|
}
|
|
gru_stats[i].weight_hc_scales[d * gru->num_output * 3 + q] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static float compute_kl_divergence(const std::vector<float>& a, const std::vector<float>& b)
|
|
{
|
|
const size_t length = a.size();
|
|
|
|
float result = 0;
|
|
for (size_t i = 0; i < length; i++)
|
|
{
|
|
result += a[i] * log(a[i] / b[i]);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
int QuantNet::build_activation_histogram(int num_histogram_bins)
|
|
{
|
|
const int input_blob_count = (int)input_blobs.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
const int file_count = (int)listspaths[0].size();
|
|
|
|
std::vector<ncnn::UnlockedPoolAllocator> blob_allocators(quantize_num_threads);
|
|
std::vector<ncnn::UnlockedPoolAllocator> workspace_allocators(quantize_num_threads);
|
|
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_bottom_blob_count; i++)
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
stat.threshold = 0.f;
|
|
stat.absmax = 0.f;
|
|
stat.percentile = 0.f;
|
|
stat.zero_count = 0;
|
|
stat.histogram.clear();
|
|
stat.histogram_normed.clear();
|
|
}
|
|
|
|
// count the absmax
|
|
#pragma omp parallel for num_threads(quantize_num_threads) schedule(static, 1)
|
|
for (int i = 0; i < file_count; i++)
|
|
{
|
|
if (i % 100 == 0)
|
|
{
|
|
fprintf(stderr, "count the absmax %.2f%% [ %d / %d ]\n", i * 100.f / file_count, i, file_count);
|
|
}
|
|
|
|
ncnn::Extractor ex = create_extractor();
|
|
ex.set_light_mode(true);
|
|
|
|
const int thread_num = ncnn::get_omp_thread_num();
|
|
ex.set_blob_allocator(&blob_allocators[thread_num]);
|
|
ex.set_workspace_allocator(&workspace_allocators[thread_num]);
|
|
|
|
for (int j = 0; j < input_blob_count; j++)
|
|
{
|
|
ncnn::Mat in;
|
|
|
|
if (0 == file_type)
|
|
{
|
|
const int type_to_pixel = type_to_pixels[j];
|
|
const std::vector<float>& mean_vals = means[j];
|
|
const std::vector<float>& norm_vals = norms[j];
|
|
|
|
int pixel_convert_type = ncnn::Mat::PIXEL_BGR;
|
|
if (type_to_pixel != pixel_convert_type)
|
|
{
|
|
pixel_convert_type = pixel_convert_type | (type_to_pixel << ncnn::Mat::PIXEL_CONVERT_SHIFT);
|
|
}
|
|
in = read_and_resize_image(shapes[j], listspaths[j][i], pixel_convert_type);
|
|
in.substract_mean_normalize(mean_vals.data(), norm_vals.data());
|
|
}
|
|
else
|
|
{
|
|
in = read_npy(shapes[j], listspaths[j][i]);
|
|
}
|
|
|
|
ex.input(input_blobs[j], in);
|
|
}
|
|
|
|
for (int j = 0; j < conv_bottom_blob_count; j++)
|
|
{
|
|
ncnn::Mat out;
|
|
ex.extract(conv_bottom_blobs[j], out);
|
|
|
|
// count absmax
|
|
{
|
|
float absmax = 0.f;
|
|
|
|
const int outc = out.c;
|
|
const int outsize = out.w * out.h;
|
|
for (int p = 0; p < outc; p++)
|
|
{
|
|
const float* ptr = out.channel(p);
|
|
for (int k = 0; k < outsize; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[k]));
|
|
}
|
|
}
|
|
|
|
#pragma omp critical
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[j];
|
|
stat.absmax = std::max(stat.absmax, absmax);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// initialize histogram
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_bottom_blob_count; i++)
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
stat.histogram.resize(num_histogram_bins, 0);
|
|
stat.histogram_normed.resize(num_histogram_bins, 0);
|
|
}
|
|
|
|
// build histogram
|
|
#pragma omp parallel for num_threads(quantize_num_threads) schedule(static, 1)
|
|
for (int i = 0; i < file_count; i++)
|
|
{
|
|
if (i % 100 == 0)
|
|
{
|
|
fprintf(stderr, "build histogram %.2f%% [ %d / %d ]\n", i * 100.f / file_count, i, file_count);
|
|
}
|
|
|
|
ncnn::Extractor ex = create_extractor();
|
|
ex.set_light_mode(true);
|
|
|
|
const int thread_num = ncnn::get_omp_thread_num();
|
|
ex.set_blob_allocator(&blob_allocators[thread_num]);
|
|
ex.set_workspace_allocator(&workspace_allocators[thread_num]);
|
|
|
|
for (int j = 0; j < input_blob_count; j++)
|
|
{
|
|
ncnn::Mat in;
|
|
|
|
if (0 == file_type)
|
|
{
|
|
const int type_to_pixel = type_to_pixels[j];
|
|
const std::vector<float>& mean_vals = means[j];
|
|
const std::vector<float>& norm_vals = norms[j];
|
|
|
|
int pixel_convert_type = ncnn::Mat::PIXEL_BGR;
|
|
if (type_to_pixel != pixel_convert_type)
|
|
{
|
|
pixel_convert_type = pixel_convert_type | (type_to_pixel << ncnn::Mat::PIXEL_CONVERT_SHIFT);
|
|
}
|
|
in = read_and_resize_image(shapes[j], listspaths[j][i], pixel_convert_type);
|
|
in.substract_mean_normalize(mean_vals.data(), norm_vals.data());
|
|
}
|
|
else
|
|
{
|
|
in = read_npy(shapes[j], listspaths[j][i]);
|
|
}
|
|
|
|
ex.input(input_blobs[j], in);
|
|
}
|
|
|
|
for (int j = 0; j < conv_bottom_blob_count; j++)
|
|
{
|
|
ncnn::Mat out;
|
|
ex.extract(conv_bottom_blobs[j], out);
|
|
|
|
// count histogram bin
|
|
{
|
|
const float absmax = quant_blob_stats[j].absmax;
|
|
|
|
std::vector<uint64_t> histogram(num_histogram_bins, 0);
|
|
uint64_t zero_count = 0;
|
|
|
|
const int outc = out.c;
|
|
const int outsize = out.w * out.h;
|
|
for (int p = 0; p < outc; p++)
|
|
{
|
|
const float* ptr = out.channel(p);
|
|
for (int k = 0; k < outsize; k++)
|
|
{
|
|
if (ptr[k] == 0.f)
|
|
{
|
|
zero_count++;
|
|
continue;
|
|
}
|
|
|
|
const int index = std::min((int)(fabs(ptr[k]) / absmax * num_histogram_bins), (num_histogram_bins - 1));
|
|
|
|
histogram[index] += 1;
|
|
}
|
|
}
|
|
|
|
#pragma omp critical
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[j];
|
|
|
|
stat.zero_count += zero_count;
|
|
for (int k = 0; k < num_histogram_bins; k++)
|
|
{
|
|
stat.histogram[k] += histogram[k];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int QuantNet::quantize_KL()
|
|
{
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
|
|
initialize_static_weight_scales(true);
|
|
|
|
if (conv_layer_count == 0 || !use_calibration_dataset)
|
|
return 0;
|
|
|
|
const int num_histogram_bins = 2048;
|
|
|
|
build_activation_histogram(num_histogram_bins);
|
|
|
|
// using kld to find the best threshold value
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_bottom_blob_count; i++)
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
// normalize histogram bin
|
|
{
|
|
uint64_t sum = 0;
|
|
for (int j = 0; j < num_histogram_bins; j++)
|
|
{
|
|
sum += stat.histogram[j];
|
|
}
|
|
|
|
for (int j = 0; j < num_histogram_bins; j++)
|
|
{
|
|
stat.histogram_normed[j] = (float)(stat.histogram[j] / (double)sum);
|
|
}
|
|
}
|
|
|
|
const int target_bin = 128;
|
|
|
|
int target_threshold = target_bin;
|
|
float min_kl_divergence = FLT_MAX;
|
|
|
|
for (int threshold = target_bin; threshold < num_histogram_bins; threshold++)
|
|
{
|
|
const float kl_eps = 0.0001f;
|
|
|
|
std::vector<float> clip_distribution(threshold, kl_eps);
|
|
{
|
|
for (int j = 0; j < threshold; j++)
|
|
{
|
|
clip_distribution[j] += stat.histogram_normed[j];
|
|
}
|
|
for (int j = threshold; j < num_histogram_bins; j++)
|
|
{
|
|
clip_distribution[threshold - 1] += stat.histogram_normed[j];
|
|
}
|
|
}
|
|
|
|
const float num_per_bin = (float)threshold / target_bin;
|
|
|
|
std::vector<float> quantize_distribution(target_bin, 0.f);
|
|
{
|
|
{
|
|
const float end = num_per_bin;
|
|
|
|
const int right_lower = (int)floor(end);
|
|
const float right_scale = end - right_lower;
|
|
|
|
if (right_scale > 0)
|
|
{
|
|
quantize_distribution[0] += right_scale * stat.histogram_normed[right_lower];
|
|
}
|
|
|
|
for (int k = 0; k < right_lower; k++)
|
|
{
|
|
quantize_distribution[0] += stat.histogram_normed[k];
|
|
}
|
|
|
|
quantize_distribution[0] /= right_lower + right_scale;
|
|
}
|
|
for (int j = 1; j < target_bin - 1; j++)
|
|
{
|
|
const float start = j * num_per_bin;
|
|
const float end = (j + 1) * num_per_bin;
|
|
|
|
const int left_upper = (int)ceil(start);
|
|
const float left_scale = left_upper - start;
|
|
|
|
const int right_lower = (int)floor(end);
|
|
const float right_scale = end - right_lower;
|
|
|
|
if (left_scale > 0)
|
|
{
|
|
quantize_distribution[j] += left_scale * stat.histogram_normed[left_upper - 1];
|
|
}
|
|
|
|
if (right_scale > 0)
|
|
{
|
|
quantize_distribution[j] += right_scale * stat.histogram_normed[right_lower];
|
|
}
|
|
|
|
for (int k = left_upper; k < right_lower; k++)
|
|
{
|
|
quantize_distribution[j] += stat.histogram_normed[k];
|
|
}
|
|
|
|
quantize_distribution[j] /= right_lower - left_upper + left_scale + right_scale;
|
|
}
|
|
{
|
|
const float start = threshold - num_per_bin;
|
|
|
|
const int left_upper = (int)ceil(start);
|
|
const float left_scale = left_upper - start;
|
|
|
|
if (left_scale > 0)
|
|
{
|
|
quantize_distribution[target_bin - 1] += left_scale * stat.histogram_normed[left_upper - 1];
|
|
}
|
|
|
|
for (int k = left_upper; k < threshold; k++)
|
|
{
|
|
quantize_distribution[target_bin - 1] += stat.histogram_normed[k];
|
|
}
|
|
|
|
quantize_distribution[target_bin - 1] /= threshold - left_upper + left_scale;
|
|
}
|
|
}
|
|
|
|
std::vector<float> expand_distribution(threshold, kl_eps);
|
|
{
|
|
{
|
|
const float end = num_per_bin;
|
|
|
|
const int right_lower = (int)floor(end);
|
|
const float right_scale = end - right_lower;
|
|
|
|
if (right_scale > 0)
|
|
{
|
|
expand_distribution[right_lower] += right_scale * quantize_distribution[0];
|
|
}
|
|
|
|
for (int k = 0; k < right_lower; k++)
|
|
{
|
|
expand_distribution[k] += quantize_distribution[0];
|
|
}
|
|
}
|
|
for (int j = 1; j < target_bin - 1; j++)
|
|
{
|
|
const float start = j * num_per_bin;
|
|
const float end = (j + 1) * num_per_bin;
|
|
|
|
const int left_upper = (int)ceil(start);
|
|
const float left_scale = left_upper - start;
|
|
|
|
const int right_lower = (int)floor(end);
|
|
const float right_scale = end - right_lower;
|
|
|
|
if (left_scale > 0)
|
|
{
|
|
expand_distribution[left_upper - 1] += left_scale * quantize_distribution[j];
|
|
}
|
|
|
|
if (right_scale > 0)
|
|
{
|
|
expand_distribution[right_lower] += right_scale * quantize_distribution[j];
|
|
}
|
|
|
|
for (int k = left_upper; k < right_lower; k++)
|
|
{
|
|
expand_distribution[k] += quantize_distribution[j];
|
|
}
|
|
}
|
|
{
|
|
const float start = threshold - num_per_bin;
|
|
|
|
const int left_upper = (int)ceil(start);
|
|
const float left_scale = left_upper - start;
|
|
|
|
if (left_scale > 0)
|
|
{
|
|
expand_distribution[left_upper - 1] += left_scale * quantize_distribution[target_bin - 1];
|
|
}
|
|
|
|
for (int k = left_upper; k < threshold; k++)
|
|
{
|
|
expand_distribution[k] += quantize_distribution[target_bin - 1];
|
|
}
|
|
}
|
|
}
|
|
|
|
// kl
|
|
const float kl_divergence = compute_kl_divergence(clip_distribution, expand_distribution);
|
|
|
|
// the best num of bin
|
|
if (kl_divergence < min_kl_divergence)
|
|
{
|
|
min_kl_divergence = kl_divergence;
|
|
target_threshold = threshold;
|
|
}
|
|
}
|
|
|
|
stat.threshold = (target_threshold + 0.5f) * stat.absmax / num_histogram_bins;
|
|
float scale = 127 / stat.threshold;
|
|
|
|
bottom_blob_scales[i].create(1);
|
|
bottom_blob_scales[i][0] = scale;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int QuantNet::quantize_percentile(const std::vector<float>& percentiles)
|
|
{
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
|
|
initialize_static_weight_scales(true);
|
|
|
|
if (conv_layer_count == 0 || !use_calibration_dataset)
|
|
return 0;
|
|
|
|
const int num_histogram_bins = 2048;
|
|
|
|
build_activation_histogram(num_histogram_bins);
|
|
|
|
// using percentile to find the threshold value
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_bottom_blob_count; i++)
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
uint64_t sum = stat.zero_count;
|
|
for (int j = 0; j < num_histogram_bins; j++)
|
|
{
|
|
sum += stat.histogram[j];
|
|
}
|
|
|
|
if (sum == 0 || stat.absmax == 0.f)
|
|
{
|
|
stat.threshold = 127.f;
|
|
stat.percentile = 1.f;
|
|
bottom_blob_scales[i].create(1);
|
|
bottom_blob_scales[i][0] = 1.f;
|
|
continue;
|
|
}
|
|
|
|
double min_error = DBL_MAX;
|
|
int target_threshold = num_histogram_bins - 1;
|
|
float target_percentile = 1.f;
|
|
|
|
for (int p = 0; p < (int)percentiles.size(); p++)
|
|
{
|
|
uint64_t target_count = (uint64_t)ceil(sum * (double)percentiles[p]);
|
|
if (target_count == 0)
|
|
target_count = 1;
|
|
|
|
uint64_t cumsum = stat.zero_count;
|
|
int threshold = num_histogram_bins - 1;
|
|
|
|
for (int j = 0; j < num_histogram_bins; j++)
|
|
{
|
|
cumsum += stat.histogram[j];
|
|
if (cumsum >= target_count)
|
|
{
|
|
threshold = j;
|
|
break;
|
|
}
|
|
}
|
|
|
|
const double threshold_value = (threshold + 0.5) * stat.absmax / num_histogram_bins;
|
|
const double scale = 127 / threshold_value;
|
|
double error = 0.0;
|
|
|
|
for (int j = 0; j < num_histogram_bins; j++)
|
|
{
|
|
if (stat.histogram[j] == 0)
|
|
continue;
|
|
|
|
const double v = (j + 0.5) * stat.absmax / num_histogram_bins;
|
|
const double v_clipped = std::min(v, threshold_value);
|
|
const int q = std::min((int)floor(v_clipped * scale + 0.5), 127);
|
|
const double v_dequant = q / scale;
|
|
const double diff = v - v_dequant;
|
|
|
|
error += stat.histogram[j] * diff * diff;
|
|
}
|
|
|
|
if (error < min_error)
|
|
{
|
|
min_error = error;
|
|
target_threshold = threshold;
|
|
target_percentile = percentiles[p];
|
|
}
|
|
}
|
|
|
|
stat.threshold = (target_threshold + 0.5f) * stat.absmax / num_histogram_bins;
|
|
stat.percentile = target_percentile;
|
|
float scale = 127 / stat.threshold;
|
|
|
|
bottom_blob_scales[i].create(1);
|
|
bottom_blob_scales[i][0] = scale;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static float compute_aciq_gaussian_clip(float absmax, int N, int num_bits = 8)
|
|
{
|
|
const float alpha_gaussian[8] = {0, 1.71063519, 2.15159277, 2.55913646, 2.93620062, 3.28691474, 3.6151146, 3.92403714};
|
|
|
|
const double gaussian_const = (0.5 * 0.35) * (1 + sqrt(3.14159265358979323846 * log(4)));
|
|
|
|
double std = (absmax * 2 * gaussian_const) / sqrt(2 * log(N));
|
|
|
|
return (float)(alpha_gaussian[num_bits - 1] * std);
|
|
}
|
|
|
|
int QuantNet::quantize_ACIQ()
|
|
{
|
|
const int input_blob_count = (int)input_blobs.size();
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
|
|
if (use_calibration_dataset)
|
|
{
|
|
// initialize conv weight scales
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_layer_count; i++)
|
|
{
|
|
const ncnn::Layer* layer = layers[conv_layers[i]];
|
|
|
|
if (layer->type == "Convolution")
|
|
{
|
|
const ncnn::Convolution* convolution = (const ncnn::Convolution*)layer;
|
|
|
|
const int num_output = convolution->num_output;
|
|
const int kernel_w = convolution->kernel_w;
|
|
const int kernel_h = convolution->kernel_h;
|
|
const int dilation_w = convolution->dilation_w;
|
|
const int dilation_h = convolution->dilation_h;
|
|
const int stride_w = convolution->stride_w;
|
|
const int stride_h = convolution->stride_h;
|
|
|
|
const int weight_data_size_output = convolution->weight_data_size / num_output;
|
|
|
|
// int8 winograd F43 needs weight data to use 6bit quantization
|
|
// TODO proper condition for winograd 3x3 int8
|
|
bool quant_6bit = false;
|
|
if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
|
|
quant_6bit = true;
|
|
|
|
weight_scales[i].create(num_output);
|
|
|
|
for (int n = 0; n < num_output; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = convolution->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
if (quant_6bit)
|
|
{
|
|
const float threshold = compute_aciq_gaussian_clip(absmax, weight_data_size_output, 6);
|
|
weight_scales[i][n] = 31 / threshold;
|
|
}
|
|
else
|
|
{
|
|
const float threshold = compute_aciq_gaussian_clip(absmax, weight_data_size_output);
|
|
weight_scales[i][n] = 127 / threshold;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (layer->type == "ConvolutionDepthWise")
|
|
{
|
|
const ncnn::ConvolutionDepthWise* convolutiondepthwise = (const ncnn::ConvolutionDepthWise*)layer;
|
|
|
|
const int group = convolutiondepthwise->group;
|
|
const int weight_data_size_output = convolutiondepthwise->weight_data_size / group;
|
|
|
|
std::vector<float> scales;
|
|
|
|
weight_scales[i].create(group);
|
|
|
|
for (int n = 0; n < group; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = convolutiondepthwise->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
const float threshold = compute_aciq_gaussian_clip(absmax, weight_data_size_output);
|
|
weight_scales[i][n] = 127 / threshold;
|
|
}
|
|
}
|
|
|
|
if (layer->type == "InnerProduct")
|
|
{
|
|
const ncnn::InnerProduct* innerproduct = (const ncnn::InnerProduct*)layer;
|
|
|
|
const int num_output = innerproduct->num_output;
|
|
const int weight_data_size_output = innerproduct->weight_data_size / num_output;
|
|
|
|
weight_scales[i].create(num_output);
|
|
|
|
for (int n = 0; n < num_output; n++)
|
|
{
|
|
const ncnn::Mat weight_data_n = innerproduct->weight_data.range(weight_data_size_output * n, weight_data_size_output);
|
|
|
|
float absmax = 0.f;
|
|
for (int k = 0; k < weight_data_size_output; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(weight_data_n[k]));
|
|
}
|
|
|
|
const float threshold = compute_aciq_gaussian_clip(absmax, weight_data_size_output);
|
|
weight_scales[i][n] = 127 / threshold;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
initialize_static_weight_scales();
|
|
|
|
if (conv_layer_count == 0 || !use_calibration_dataset)
|
|
return 0;
|
|
|
|
const int file_count = (int)listspaths[0].size();
|
|
|
|
std::vector<ncnn::UnlockedPoolAllocator> blob_allocators(quantize_num_threads);
|
|
std::vector<ncnn::UnlockedPoolAllocator> workspace_allocators(quantize_num_threads);
|
|
|
|
// count the absmax
|
|
#pragma omp parallel for num_threads(quantize_num_threads) schedule(static, 1)
|
|
for (int i = 0; i < file_count; i++)
|
|
{
|
|
if (i % 100 == 0)
|
|
{
|
|
fprintf(stderr, "count the absmax %.2f%% [ %d / %d ]\n", i * 100.f / file_count, i, file_count);
|
|
}
|
|
|
|
ncnn::Extractor ex = create_extractor();
|
|
ex.set_light_mode(true);
|
|
|
|
const int thread_num = ncnn::get_omp_thread_num();
|
|
ex.set_blob_allocator(&blob_allocators[thread_num]);
|
|
ex.set_workspace_allocator(&workspace_allocators[thread_num]);
|
|
|
|
for (int j = 0; j < input_blob_count; j++)
|
|
{
|
|
ncnn::Mat in;
|
|
|
|
if (0 == file_type)
|
|
{
|
|
const int type_to_pixel = type_to_pixels[j];
|
|
const std::vector<float>& mean_vals = means[j];
|
|
const std::vector<float>& norm_vals = norms[j];
|
|
|
|
int pixel_convert_type = ncnn::Mat::PIXEL_BGR;
|
|
if (type_to_pixel != pixel_convert_type)
|
|
{
|
|
pixel_convert_type = pixel_convert_type | (type_to_pixel << ncnn::Mat::PIXEL_CONVERT_SHIFT);
|
|
}
|
|
in = read_and_resize_image(shapes[j], listspaths[j][i], pixel_convert_type);
|
|
in.substract_mean_normalize(mean_vals.data(), norm_vals.data());
|
|
}
|
|
else
|
|
{
|
|
in = read_npy(shapes[j], listspaths[j][i]);
|
|
}
|
|
|
|
ex.input(input_blobs[j], in);
|
|
}
|
|
|
|
for (int j = 0; j < conv_bottom_blob_count; j++)
|
|
{
|
|
ncnn::Mat out;
|
|
ex.extract(conv_bottom_blobs[j], out);
|
|
|
|
// count absmax
|
|
{
|
|
float absmax = 0.f;
|
|
|
|
const int outc = out.c;
|
|
const int outsize = out.w * out.h;
|
|
for (int p = 0; p < outc; p++)
|
|
{
|
|
const float* ptr = out.channel(p);
|
|
for (int k = 0; k < outsize; k++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[k]));
|
|
}
|
|
}
|
|
|
|
#pragma omp critical
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[j];
|
|
stat.absmax = std::max(stat.absmax, absmax);
|
|
stat.total = outc * outsize;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// alpha gaussian
|
|
#pragma omp parallel for num_threads(quantize_num_threads)
|
|
for (int i = 0; i < conv_bottom_blob_count; i++)
|
|
{
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
|
|
stat.threshold = compute_aciq_gaussian_clip(stat.absmax, stat.total);
|
|
float scale = 127 / stat.threshold;
|
|
|
|
bottom_blob_scales[i].create(1);
|
|
bottom_blob_scales[i][0] = scale;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static float cosine_similarity(const ncnn::Mat& a, const ncnn::Mat& b)
|
|
{
|
|
const int chanenls = a.c;
|
|
const int size = a.w * a.h;
|
|
|
|
float sa = 0;
|
|
float sb = 0;
|
|
float sum = 0;
|
|
|
|
for (int p = 0; p < chanenls; p++)
|
|
{
|
|
const float* pa = a.channel(p);
|
|
const float* pb = b.channel(p);
|
|
|
|
for (int i = 0; i < size; i++)
|
|
{
|
|
sa += pa[i] * pa[i];
|
|
sb += pb[i] * pb[i];
|
|
sum += pa[i] * pb[i];
|
|
}
|
|
}
|
|
|
|
float sim = (float)sum / sqrt(sa) / sqrt(sb);
|
|
|
|
return sim;
|
|
}
|
|
|
|
static int get_layer_param(const ncnn::Layer* layer, ncnn::ParamDict& pd)
|
|
{
|
|
if (layer->type == "Convolution")
|
|
{
|
|
ncnn::Convolution* convolution = (ncnn::Convolution*)layer;
|
|
|
|
pd.set(0, convolution->num_output);
|
|
pd.set(1, convolution->kernel_w);
|
|
pd.set(11, convolution->kernel_h);
|
|
pd.set(2, convolution->dilation_w);
|
|
pd.set(12, convolution->dilation_h);
|
|
pd.set(3, convolution->stride_w);
|
|
pd.set(13, convolution->stride_h);
|
|
pd.set(4, convolution->pad_left);
|
|
pd.set(15, convolution->pad_right);
|
|
pd.set(14, convolution->pad_top);
|
|
pd.set(16, convolution->pad_bottom);
|
|
pd.set(18, convolution->pad_value);
|
|
pd.set(5, convolution->bias_term);
|
|
pd.set(6, convolution->weight_data_size);
|
|
pd.set(8, convolution->int8_scale_term);
|
|
pd.set(9, convolution->activation_type);
|
|
pd.set(10, convolution->activation_params);
|
|
}
|
|
else if (layer->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolutiondepthwise = (ncnn::ConvolutionDepthWise*)layer;
|
|
|
|
pd.set(0, convolutiondepthwise->num_output);
|
|
pd.set(1, convolutiondepthwise->kernel_w);
|
|
pd.set(11, convolutiondepthwise->kernel_h);
|
|
pd.set(2, convolutiondepthwise->dilation_w);
|
|
pd.set(12, convolutiondepthwise->dilation_h);
|
|
pd.set(3, convolutiondepthwise->stride_w);
|
|
pd.set(13, convolutiondepthwise->stride_h);
|
|
pd.set(4, convolutiondepthwise->pad_left);
|
|
pd.set(15, convolutiondepthwise->pad_right);
|
|
pd.set(14, convolutiondepthwise->pad_top);
|
|
pd.set(16, convolutiondepthwise->pad_bottom);
|
|
pd.set(18, convolutiondepthwise->pad_value);
|
|
pd.set(5, convolutiondepthwise->bias_term);
|
|
pd.set(6, convolutiondepthwise->weight_data_size);
|
|
pd.set(7, convolutiondepthwise->group);
|
|
pd.set(8, convolutiondepthwise->int8_scale_term);
|
|
pd.set(9, convolutiondepthwise->activation_type);
|
|
pd.set(10, convolutiondepthwise->activation_params);
|
|
}
|
|
else if (layer->type == "InnerProduct")
|
|
{
|
|
ncnn::InnerProduct* innerproduct = (ncnn::InnerProduct*)layer;
|
|
|
|
pd.set(0, innerproduct->num_output);
|
|
pd.set(1, innerproduct->bias_term);
|
|
pd.set(2, innerproduct->weight_data_size);
|
|
pd.set(8, innerproduct->int8_scale_term);
|
|
pd.set(9, innerproduct->activation_type);
|
|
pd.set(10, innerproduct->activation_params);
|
|
}
|
|
else
|
|
{
|
|
fprintf(stderr, "unexpected layer type %s in get_layer_param\n", layer->type.c_str());
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int get_layer_weights(const ncnn::Layer* layer, std::vector<ncnn::Mat>& weights)
|
|
{
|
|
if (layer->type == "Convolution")
|
|
{
|
|
ncnn::Convolution* convolution = (ncnn::Convolution*)layer;
|
|
weights.push_back(convolution->weight_data);
|
|
if (convolution->bias_term)
|
|
weights.push_back(convolution->bias_data);
|
|
}
|
|
else if (layer->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolutiondepthwise = (ncnn::ConvolutionDepthWise*)layer;
|
|
weights.push_back(convolutiondepthwise->weight_data);
|
|
if (convolutiondepthwise->bias_term)
|
|
weights.push_back(convolutiondepthwise->bias_data);
|
|
}
|
|
else if (layer->type == "InnerProduct")
|
|
{
|
|
ncnn::InnerProduct* innerproduct = (ncnn::InnerProduct*)layer;
|
|
weights.push_back(innerproduct->weight_data);
|
|
if (innerproduct->bias_term)
|
|
weights.push_back(innerproduct->bias_data);
|
|
}
|
|
else
|
|
{
|
|
fprintf(stderr, "unexpected layer type %s in get_layer_weights\n", layer->type.c_str());
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int QuantNet::quantize_EQ()
|
|
{
|
|
// find the initial scale via KL
|
|
quantize_KL();
|
|
|
|
print_quant_info();
|
|
|
|
const int input_blob_count = (int)input_blobs.size();
|
|
const int conv_layer_count = (int)conv_layers.size();
|
|
const int conv_bottom_blob_count = (int)conv_bottom_blobs.size();
|
|
|
|
if (conv_layer_count == 0 || !use_calibration_dataset)
|
|
return 0;
|
|
|
|
std::vector<ncnn::UnlockedPoolAllocator> blob_allocators(quantize_num_threads);
|
|
std::vector<ncnn::UnlockedPoolAllocator> workspace_allocators(quantize_num_threads);
|
|
|
|
// max 50 images for EQ
|
|
const int file_count = std::min((int)listspaths[0].size(), 50);
|
|
|
|
const float scale_range_lower = 0.5f;
|
|
const float scale_range_upper = 2.0f;
|
|
const int search_steps = 100;
|
|
|
|
for (int i = 0; i < conv_layer_count; i++)
|
|
{
|
|
ncnn::Mat& weight_scale = weight_scales[i];
|
|
ncnn::Mat& bottom_blob_scale = bottom_blob_scales[i];
|
|
|
|
const ncnn::Layer* layer = layers[conv_layers[i]];
|
|
|
|
// search weight scale
|
|
for (int j = 0; j < weight_scale.w; j++)
|
|
{
|
|
const float scale = weight_scale[j];
|
|
const float scale_lower = scale * scale_range_lower;
|
|
const float scale_upper = scale * scale_range_upper;
|
|
const float scale_step = (scale_upper - scale_lower) / search_steps;
|
|
|
|
std::vector<double> avgsims(search_steps, 0.0);
|
|
|
|
#pragma omp parallel for num_threads(quantize_num_threads) schedule(static, 1)
|
|
for (int ii = 0; ii < file_count; ii++)
|
|
{
|
|
if (ii % 100 == 0)
|
|
{
|
|
fprintf(stderr, "search weight scale %.2f%% [ %d / %d ] for %d / %d of %d / %d\n", ii * 100.f / file_count, ii, file_count, j, weight_scale.w, i, conv_layer_count);
|
|
}
|
|
|
|
ncnn::Extractor ex = create_extractor();
|
|
ex.set_light_mode(true);
|
|
|
|
const int thread_num = ncnn::get_omp_thread_num();
|
|
ex.set_blob_allocator(&blob_allocators[thread_num]);
|
|
ex.set_workspace_allocator(&workspace_allocators[thread_num]);
|
|
|
|
for (int jj = 0; jj < input_blob_count; jj++)
|
|
{
|
|
ncnn::Mat in;
|
|
|
|
if (0 == file_type)
|
|
{
|
|
const int type_to_pixel = type_to_pixels[j];
|
|
const std::vector<float>& mean_vals = means[j];
|
|
const std::vector<float>& norm_vals = norms[j];
|
|
|
|
int pixel_convert_type = ncnn::Mat::PIXEL_BGR;
|
|
if (type_to_pixel != pixel_convert_type)
|
|
{
|
|
pixel_convert_type = pixel_convert_type | (type_to_pixel << ncnn::Mat::PIXEL_CONVERT_SHIFT);
|
|
}
|
|
in = read_and_resize_image(shapes[j], listspaths[j][i], pixel_convert_type);
|
|
in.substract_mean_normalize(mean_vals.data(), norm_vals.data());
|
|
}
|
|
else
|
|
{
|
|
in = read_npy(shapes[j], listspaths[j][i]);
|
|
}
|
|
|
|
ex.input(input_blobs[j], in);
|
|
}
|
|
|
|
ncnn::Mat in;
|
|
ex.extract(conv_bottom_blobs[i], in);
|
|
|
|
ncnn::Mat out;
|
|
ex.extract(conv_top_blobs[i], out);
|
|
|
|
ncnn::Layer* layer_int8 = ncnn::create_layer_cpu(layer->typeindex);
|
|
|
|
ncnn::ParamDict pd;
|
|
get_layer_param(layer, pd);
|
|
pd.set(8, 1); //int8_scale_term
|
|
layer_int8->load_param(pd);
|
|
|
|
std::vector<float> sims(search_steps);
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
ncnn::Mat new_weight_scale = weight_scale.clone();
|
|
new_weight_scale[j] = scale_lower + k * scale_step;
|
|
|
|
std::vector<ncnn::Mat> weights;
|
|
get_layer_weights(layer, weights);
|
|
weights.push_back(new_weight_scale);
|
|
weights.push_back(bottom_blob_scale);
|
|
layer_int8->load_model(ncnn::ModelBinFromMatArray(weights.data()));
|
|
|
|
ncnn::Option opt_int8;
|
|
opt_int8.use_packing_layout = false;
|
|
|
|
layer_int8->create_pipeline(opt_int8);
|
|
|
|
ncnn::Mat out_int8;
|
|
layer_int8->forward(in, out_int8, opt_int8);
|
|
|
|
layer_int8->destroy_pipeline(opt_int8);
|
|
|
|
sims[k] = cosine_similarity(out, out_int8);
|
|
}
|
|
|
|
delete layer_int8;
|
|
|
|
#pragma omp critical
|
|
{
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
avgsims[k] += sims[k];
|
|
}
|
|
}
|
|
}
|
|
|
|
double max_avgsim = 0.0;
|
|
float new_scale = scale;
|
|
|
|
// find the scale with min cosine distance
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
if (max_avgsim < avgsims[k])
|
|
{
|
|
max_avgsim = avgsims[k];
|
|
new_scale = scale_lower + k * scale_step;
|
|
}
|
|
}
|
|
|
|
fprintf(stderr, "%s w %d = %f -> %f\n", layer->name.c_str(), j, scale, new_scale);
|
|
weight_scale[j] = new_scale;
|
|
}
|
|
|
|
// search bottom blob scale
|
|
for (int j = 0; j < bottom_blob_scale.w; j++)
|
|
{
|
|
const float scale = bottom_blob_scale[j];
|
|
const float scale_lower = scale * scale_range_lower;
|
|
const float scale_upper = scale * scale_range_upper;
|
|
const float scale_step = (scale_upper - scale_lower) / search_steps;
|
|
|
|
std::vector<double> avgsims(search_steps, 0.0);
|
|
|
|
#pragma omp parallel for num_threads(quantize_num_threads) schedule(static, 1)
|
|
for (int ii = 0; ii < file_count; ii++)
|
|
{
|
|
if (ii % 100 == 0)
|
|
{
|
|
fprintf(stderr, "search bottom blob scale %.2f%% [ %d / %d ] for %d / %d of %d / %d\n", ii * 100.f / file_count, ii, file_count, j, bottom_blob_scale.w, i, conv_layer_count);
|
|
}
|
|
|
|
ncnn::Extractor ex = create_extractor();
|
|
ex.set_light_mode(true);
|
|
|
|
const int thread_num = ncnn::get_omp_thread_num();
|
|
ex.set_blob_allocator(&blob_allocators[thread_num]);
|
|
ex.set_workspace_allocator(&workspace_allocators[thread_num]);
|
|
|
|
for (int jj = 0; jj < input_blob_count; jj++)
|
|
{
|
|
ncnn::Mat in;
|
|
|
|
if (0 == file_type)
|
|
{
|
|
const int type_to_pixel = type_to_pixels[j];
|
|
const std::vector<float>& mean_vals = means[j];
|
|
const std::vector<float>& norm_vals = norms[j];
|
|
|
|
int pixel_convert_type = ncnn::Mat::PIXEL_BGR;
|
|
if (type_to_pixel != pixel_convert_type)
|
|
{
|
|
pixel_convert_type = pixel_convert_type | (type_to_pixel << ncnn::Mat::PIXEL_CONVERT_SHIFT);
|
|
}
|
|
in = read_and_resize_image(shapes[j], listspaths[j][i], pixel_convert_type);
|
|
in.substract_mean_normalize(mean_vals.data(), norm_vals.data());
|
|
}
|
|
else
|
|
{
|
|
in = read_npy(shapes[j], listspaths[j][i]);
|
|
}
|
|
|
|
ex.input(input_blobs[j], in);
|
|
}
|
|
|
|
ncnn::Mat in;
|
|
ex.extract(conv_bottom_blobs[i], in);
|
|
|
|
ncnn::Mat out;
|
|
ex.extract(conv_top_blobs[i], out);
|
|
|
|
ncnn::Layer* layer_int8 = ncnn::create_layer_cpu(layer->typeindex);
|
|
|
|
ncnn::ParamDict pd;
|
|
get_layer_param(layer, pd);
|
|
pd.set(8, 1); //int8_scale_term
|
|
layer_int8->load_param(pd);
|
|
|
|
std::vector<float> sims(search_steps);
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
ncnn::Mat new_bottom_blob_scale = bottom_blob_scale.clone();
|
|
new_bottom_blob_scale[j] = scale_lower + k * scale_step;
|
|
|
|
std::vector<ncnn::Mat> weights;
|
|
get_layer_weights(layer, weights);
|
|
weights.push_back(weight_scale);
|
|
weights.push_back(new_bottom_blob_scale);
|
|
layer_int8->load_model(ncnn::ModelBinFromMatArray(weights.data()));
|
|
|
|
ncnn::Option opt_int8;
|
|
opt_int8.use_packing_layout = false;
|
|
|
|
layer_int8->create_pipeline(opt_int8);
|
|
|
|
ncnn::Mat out_int8;
|
|
layer_int8->forward(in, out_int8, opt_int8);
|
|
|
|
layer_int8->destroy_pipeline(opt_int8);
|
|
|
|
sims[k] = cosine_similarity(out, out_int8);
|
|
}
|
|
|
|
delete layer_int8;
|
|
|
|
#pragma omp critical
|
|
{
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
avgsims[k] += sims[k];
|
|
}
|
|
}
|
|
}
|
|
|
|
double max_avgsim = 0.0;
|
|
float new_scale = scale;
|
|
|
|
// find the scale with min cosine distance
|
|
for (int k = 0; k < search_steps; k++)
|
|
{
|
|
if (max_avgsim < avgsims[k])
|
|
{
|
|
max_avgsim = avgsims[k];
|
|
new_scale = scale_lower + k * scale_step;
|
|
}
|
|
}
|
|
|
|
fprintf(stderr, "%s b %d = %f -> %f\n", layer->name.c_str(), j, scale, new_scale);
|
|
bottom_blob_scale[j] = new_scale;
|
|
}
|
|
|
|
// update quant info
|
|
QuantBlobStat& stat = quant_blob_stats[i];
|
|
stat.threshold = 127 / bottom_blob_scale[0];
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static std::vector<std::vector<std::string> > parse_comma_path_list(char* s)
|
|
{
|
|
std::vector<std::vector<std::string> > aps;
|
|
|
|
char* pch = strtok(s, ",");
|
|
while (pch != NULL)
|
|
{
|
|
FILE* fp = fopen(pch, "rb");
|
|
if (!fp)
|
|
{
|
|
fprintf(stderr, "fopen %s failed\n", pch);
|
|
break;
|
|
}
|
|
|
|
std::vector<std::string> paths;
|
|
|
|
// one filepath per line
|
|
char line[1024];
|
|
while (!feof(fp))
|
|
{
|
|
char* ss = fgets(line, 1024, fp);
|
|
if (!ss)
|
|
break;
|
|
|
|
char filepath[256];
|
|
int nscan = sscanf(line, "%255s", filepath);
|
|
if (nscan != 1)
|
|
continue;
|
|
|
|
paths.push_back(std::string(filepath));
|
|
}
|
|
|
|
fclose(fp);
|
|
|
|
aps.push_back(paths);
|
|
|
|
pch = strtok(NULL, ",");
|
|
}
|
|
|
|
return aps;
|
|
}
|
|
|
|
static float vstr_to_float(const char vstr[20])
|
|
{
|
|
double v = 0.0;
|
|
|
|
const char* p = vstr;
|
|
|
|
// sign
|
|
bool sign = *p != '-';
|
|
if (*p == '+' || *p == '-')
|
|
{
|
|
p++;
|
|
}
|
|
|
|
// digits before decimal point or exponent
|
|
uint64_t v1 = 0;
|
|
while (isdigit(*p))
|
|
{
|
|
v1 = v1 * 10 + (*p - '0');
|
|
p++;
|
|
}
|
|
|
|
v = (double)v1;
|
|
|
|
// digits after decimal point
|
|
if (*p == '.')
|
|
{
|
|
p++;
|
|
|
|
uint64_t pow10 = 1;
|
|
uint64_t v2 = 0;
|
|
|
|
while (isdigit(*p))
|
|
{
|
|
v2 = v2 * 10 + (*p - '0');
|
|
pow10 *= 10;
|
|
p++;
|
|
}
|
|
|
|
v += v2 / (double)pow10;
|
|
}
|
|
|
|
// exponent
|
|
if (*p == 'e' || *p == 'E')
|
|
{
|
|
p++;
|
|
|
|
// sign of exponent
|
|
bool fact = *p != '-';
|
|
if (*p == '+' || *p == '-')
|
|
{
|
|
p++;
|
|
}
|
|
|
|
// digits of exponent
|
|
uint64_t expon = 0;
|
|
while (isdigit(*p))
|
|
{
|
|
expon = expon * 10 + (*p - '0');
|
|
p++;
|
|
}
|
|
|
|
double scale = 1.0;
|
|
while (expon >= 8)
|
|
{
|
|
scale *= 1e8;
|
|
expon -= 8;
|
|
}
|
|
while (expon > 0)
|
|
{
|
|
scale *= 10.0;
|
|
expon -= 1;
|
|
}
|
|
|
|
v = fact ? v * scale : v / scale;
|
|
}
|
|
|
|
// fprintf(stderr, "v = %f\n", v);
|
|
return sign ? (float)v : (float)-v;
|
|
}
|
|
|
|
static std::vector<std::vector<float> > parse_comma_float_array_list(char* s)
|
|
{
|
|
std::vector<std::vector<float> > aaf;
|
|
|
|
char* pch = strtok(s, "[]");
|
|
while (pch != NULL)
|
|
{
|
|
// parse a,b,c
|
|
char vstr[20];
|
|
int nconsumed = 0;
|
|
int nscan = sscanf(pch, "%19[^,]%n", vstr, &nconsumed);
|
|
if (nscan == 1)
|
|
{
|
|
// ok we get array
|
|
pch += nconsumed;
|
|
|
|
std::vector<float> af;
|
|
float v = vstr_to_float(vstr);
|
|
af.push_back(v);
|
|
|
|
nscan = sscanf(pch, ",%19[^,]%n", vstr, &nconsumed);
|
|
while (nscan == 1)
|
|
{
|
|
pch += nconsumed;
|
|
|
|
float v = vstr_to_float(vstr);
|
|
af.push_back(v);
|
|
|
|
nscan = sscanf(pch, ",%19[^,]%n", vstr, &nconsumed);
|
|
}
|
|
|
|
// array end
|
|
aaf.push_back(af);
|
|
}
|
|
|
|
pch = strtok(NULL, "[]");
|
|
}
|
|
|
|
return aaf;
|
|
}
|
|
|
|
static std::vector<std::vector<int> > parse_comma_int_array_list(char* s)
|
|
{
|
|
std::vector<std::vector<int> > aai;
|
|
|
|
char* pch = strtok(s, "[]");
|
|
while (pch != NULL)
|
|
{
|
|
// parse a,b,c
|
|
int v;
|
|
int nconsumed = 0;
|
|
int nscan = sscanf(pch, "%d%n", &v, &nconsumed);
|
|
if (nscan == 1)
|
|
{
|
|
// ok we get array
|
|
pch += nconsumed;
|
|
|
|
std::vector<int> ai;
|
|
ai.push_back(v);
|
|
|
|
nscan = sscanf(pch, ",%d%n", &v, &nconsumed);
|
|
while (nscan == 1)
|
|
{
|
|
pch += nconsumed;
|
|
|
|
ai.push_back(v);
|
|
|
|
nscan = sscanf(pch, ",%d%n", &v, &nconsumed);
|
|
}
|
|
|
|
// array end
|
|
aai.push_back(ai);
|
|
}
|
|
|
|
pch = strtok(NULL, "[]");
|
|
}
|
|
|
|
return aai;
|
|
}
|
|
|
|
static std::vector<int> parse_comma_pixel_type_list(char* s)
|
|
{
|
|
std::vector<int> aps;
|
|
|
|
char* pch = strtok(s, ",");
|
|
while (pch != NULL)
|
|
{
|
|
// RAW/RGB/BGR/GRAY/RGBA/BGRA
|
|
if (strcmp(pch, "RAW") == 0)
|
|
aps.push_back(-233);
|
|
if (strcmp(pch, "RGB") == 0)
|
|
aps.push_back(ncnn::Mat::PIXEL_RGB);
|
|
if (strcmp(pch, "BGR") == 0)
|
|
aps.push_back(ncnn::Mat::PIXEL_BGR);
|
|
if (strcmp(pch, "GRAY") == 0)
|
|
aps.push_back(ncnn::Mat::PIXEL_GRAY);
|
|
if (strcmp(pch, "RGBA") == 0)
|
|
aps.push_back(ncnn::Mat::PIXEL_RGBA);
|
|
if (strcmp(pch, "BGRA") == 0)
|
|
aps.push_back(ncnn::Mat::PIXEL_BGRA);
|
|
|
|
pch = strtok(NULL, ",");
|
|
}
|
|
|
|
return aps;
|
|
}
|
|
|
|
static void print_float_array_list(const std::vector<std::vector<float> >& list)
|
|
{
|
|
for (size_t i = 0; i < list.size(); i++)
|
|
{
|
|
const std::vector<float>& array = list[i];
|
|
fprintf(stderr, "[");
|
|
for (size_t j = 0; j < array.size(); j++)
|
|
{
|
|
fprintf(stderr, "%f", array[j]);
|
|
if (j != array.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
fprintf(stderr, "]");
|
|
if (i != list.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
}
|
|
|
|
static void print_int_array_list(const std::vector<std::vector<int> >& list)
|
|
{
|
|
for (size_t i = 0; i < list.size(); i++)
|
|
{
|
|
const std::vector<int>& array = list[i];
|
|
fprintf(stderr, "[");
|
|
for (size_t j = 0; j < array.size(); j++)
|
|
{
|
|
fprintf(stderr, "%d", array[j]);
|
|
if (j != array.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
fprintf(stderr, "]");
|
|
if (i != list.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
}
|
|
|
|
static void print_pixel_type_list(const std::vector<int>& list)
|
|
{
|
|
for (size_t i = 0; i < list.size(); i++)
|
|
{
|
|
const int type = list[i];
|
|
if (type == -233)
|
|
fprintf(stderr, "RAW");
|
|
if (type == ncnn::Mat::PIXEL_RGB)
|
|
fprintf(stderr, "RGB");
|
|
if (type == ncnn::Mat::PIXEL_BGR)
|
|
fprintf(stderr, "BGR");
|
|
if (type == ncnn::Mat::PIXEL_GRAY)
|
|
fprintf(stderr, "GRAY");
|
|
if (type == ncnn::Mat::PIXEL_RGBA)
|
|
fprintf(stderr, "RGBA");
|
|
if (type == ncnn::Mat::PIXEL_BGRA)
|
|
fprintf(stderr, "BGRA");
|
|
if (i != list.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
}
|
|
|
|
static void show_usage()
|
|
{
|
|
fprintf(stderr, "Usage: ncnn2table [ncnnparam] [ncnnbin] [list,...] [ncnntable] [(key=value)...]\n");
|
|
fprintf(stderr, " ncnn2table [ncnnparam] [ncnnbin] [ncnntable] [(key=value)...]\n");
|
|
fprintf(stderr, " mean=[104.0,117.0,123.0],...\n");
|
|
fprintf(stderr, " norm=[1.0,1.0,1.0],...\n");
|
|
fprintf(stderr, " shape=[224,224,3],...[w,h,c] or [w,h] **[0,0] will not resize\n");
|
|
fprintf(stderr, " pixel=RAW/RGB/BGR/GRAY/RGBA/BGRA,...\n");
|
|
fprintf(stderr, " thread=8\n");
|
|
fprintf(stderr, " method=kl/aciq/eq/percentile\n");
|
|
fprintf(stderr, " percentile=[0.999,0.9995,0.9999,0.99995,0.99999,1]\n");
|
|
fprintf(stderr, " type=0/1, 0:image,1:npy\n");
|
|
fprintf(stderr, "Sample usage:\n");
|
|
fprintf(stderr, " ncnn2table squeezenet.param squeezenet.bin filelist.txt squeezenet.table mean=[104.0,117.0,123.0] norm=[1.0,1.0,1.0] shape=[227,227,3] pixel=BGR method=kl\n");
|
|
fprintf(stderr, " ncnn2table test.param test.bin filelist.txt squeezenet.table shape=[227,227,3] method=kl type=1\n");
|
|
fprintf(stderr, " ncnn2table rnn.param rnn.bin rnn.table method=kl\n");
|
|
}
|
|
|
|
int main(int argc, char** argv)
|
|
{
|
|
if (argc < 4)
|
|
{
|
|
show_usage();
|
|
return -1;
|
|
}
|
|
|
|
for (int i = 1; i < argc; i++)
|
|
{
|
|
if (argv[i][0] == '-')
|
|
{
|
|
show_usage();
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
const char* inparam = argv[1];
|
|
const char* inbin = argv[2];
|
|
|
|
ncnn::Option opt;
|
|
opt.num_threads = 1;
|
|
opt.lightmode = false;
|
|
opt.use_fp16_packed = false;
|
|
opt.use_fp16_storage = false;
|
|
opt.use_fp16_arithmetic = false;
|
|
|
|
QuantNet net;
|
|
net.opt = opt;
|
|
net.load_param(inparam);
|
|
net.load_model(inbin);
|
|
|
|
net.init();
|
|
|
|
const char* outtable = 0;
|
|
int kv_start = 0;
|
|
|
|
if (argc >= 5 && strchr(argv[4], '='))
|
|
{
|
|
outtable = argv[3];
|
|
kv_start = 4;
|
|
}
|
|
else if (argc >= 5)
|
|
{
|
|
net.listspaths = parse_comma_path_list(argv[3]);
|
|
net.use_calibration_dataset = true;
|
|
outtable = argv[4];
|
|
kv_start = 5;
|
|
}
|
|
else
|
|
{
|
|
outtable = argv[3];
|
|
kv_start = 4;
|
|
}
|
|
|
|
if (!net.conv_layers.empty() && !net.use_calibration_dataset)
|
|
{
|
|
fprintf(stderr, "warning: calibration dataset not provided, skip activation calibration and generate weight-only table\n");
|
|
}
|
|
|
|
std::string method = "kl";
|
|
std::vector<float> percentiles;
|
|
percentiles.push_back(0.999f);
|
|
percentiles.push_back(0.9995f);
|
|
percentiles.push_back(0.9999f);
|
|
percentiles.push_back(0.99995f);
|
|
percentiles.push_back(0.99999f);
|
|
percentiles.push_back(1.f);
|
|
net.file_type = 0;
|
|
|
|
for (int i = kv_start; i < argc; i++)
|
|
{
|
|
// key=value
|
|
char* kv = argv[i];
|
|
|
|
char* eqs = strchr(kv, '=');
|
|
if (eqs == NULL)
|
|
{
|
|
fprintf(stderr, "unrecognized arg %s\n", kv);
|
|
continue;
|
|
}
|
|
|
|
// split k v
|
|
eqs[0] = '\0';
|
|
const char* key = kv;
|
|
char* value = eqs + 1;
|
|
|
|
// load mean norm shape
|
|
if (memcmp(key, "mean", 4) == 0)
|
|
net.means = parse_comma_float_array_list(value);
|
|
if (memcmp(key, "norm", 4) == 0)
|
|
net.norms = parse_comma_float_array_list(value);
|
|
if (memcmp(key, "shape", 5) == 0)
|
|
net.shapes = parse_comma_int_array_list(value);
|
|
if (memcmp(key, "pixel", 5) == 0)
|
|
net.type_to_pixels = parse_comma_pixel_type_list(value);
|
|
if (memcmp(key, "thread", 6) == 0)
|
|
net.quantize_num_threads = atoi(value);
|
|
if (memcmp(key, "method", 6) == 0)
|
|
method = std::string(value);
|
|
if (memcmp(key, "percentile", 10) == 0)
|
|
{
|
|
percentiles.clear();
|
|
if (strchr(value, '['))
|
|
{
|
|
std::vector<std::vector<float> > percentile_list = parse_comma_float_array_list(value);
|
|
if (!percentile_list.empty())
|
|
percentiles = percentile_list[0];
|
|
}
|
|
else
|
|
{
|
|
percentiles.push_back(vstr_to_float(value));
|
|
}
|
|
}
|
|
if (memcmp(key, "type", 4) == 0)
|
|
net.file_type = atoi(value);
|
|
}
|
|
|
|
// sanity check
|
|
const size_t input_blob_count = net.input_blobs.size();
|
|
if (net.use_calibration_dataset && net.listspaths.size() != input_blob_count)
|
|
{
|
|
fprintf(stderr, "expect %d lists, but got %d\n", (int)input_blob_count, (int)net.listspaths.size());
|
|
return -1;
|
|
}
|
|
if (net.use_calibration_dataset && (0 == net.file_type) && (net.means.size() != input_blob_count))
|
|
{
|
|
fprintf(stderr, "expect %d means, but got %d\n", (int)input_blob_count, (int)net.means.size());
|
|
return -1;
|
|
}
|
|
if (net.use_calibration_dataset && (0 == net.file_type) && (net.norms.size() != input_blob_count))
|
|
{
|
|
fprintf(stderr, "expect %d norms, but got %d\n", (int)input_blob_count, (int)net.norms.size());
|
|
return -1;
|
|
}
|
|
if (net.use_calibration_dataset && net.shapes.size() != input_blob_count)
|
|
{
|
|
fprintf(stderr, "expect %d shapes, but got %d\n", (int)input_blob_count, (int)net.shapes.size());
|
|
return -1;
|
|
}
|
|
if (net.use_calibration_dataset && (0 == net.file_type) && (net.type_to_pixels.size() != input_blob_count))
|
|
{
|
|
fprintf(stderr, "expect %d pixels, but got %d\n", (int)input_blob_count, (int)net.type_to_pixels.size());
|
|
return -1;
|
|
}
|
|
if (net.quantize_num_threads < 0)
|
|
{
|
|
fprintf(stderr, "malformed thread %d\n", net.quantize_num_threads);
|
|
return -1;
|
|
}
|
|
if (percentiles.empty())
|
|
{
|
|
fprintf(stderr, "empty percentile candidates\n");
|
|
return -1;
|
|
}
|
|
for (int i = 0; i < (int)percentiles.size(); i++)
|
|
{
|
|
if (percentiles[i] <= 0.f || percentiles[i] > 1.f)
|
|
{
|
|
fprintf(stderr, "malformed percentile %f, expect (0, 1]\n", percentiles[i]);
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
// print quantnet config
|
|
{
|
|
fprintf(stderr, "mean = ");
|
|
print_float_array_list(net.means);
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "norm = ");
|
|
print_float_array_list(net.norms);
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "shape = ");
|
|
print_int_array_list(net.shapes);
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "pixel = ");
|
|
print_pixel_type_list(net.type_to_pixels);
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "thread = %d\n", net.quantize_num_threads);
|
|
fprintf(stderr, "method = %s\n", method.c_str());
|
|
if (method == "percentile")
|
|
{
|
|
fprintf(stderr, "percentile = [");
|
|
for (int i = 0; i < (int)percentiles.size(); i++)
|
|
{
|
|
fprintf(stderr, "%f", percentiles[i]);
|
|
if (i != (int)percentiles.size() - 1)
|
|
fprintf(stderr, ",");
|
|
}
|
|
fprintf(stderr, "]\n");
|
|
}
|
|
fprintf(stderr, "---------------------------------------\n");
|
|
}
|
|
|
|
if (method == "kl")
|
|
{
|
|
net.quantize_KL();
|
|
}
|
|
else if (method == "aciq")
|
|
{
|
|
net.quantize_ACIQ();
|
|
}
|
|
else if (method == "percentile")
|
|
{
|
|
net.quantize_percentile(percentiles);
|
|
}
|
|
else if (method == "eq")
|
|
{
|
|
net.quantize_EQ();
|
|
}
|
|
else
|
|
{
|
|
fprintf(stderr, "not implemented yet !\n");
|
|
fprintf(stderr, "unknown method %s, expect kl / aciq / eq / percentile\n", method.c_str());
|
|
return -1;
|
|
}
|
|
|
|
net.print_quant_info();
|
|
|
|
net.save_table(outtable);
|
|
|
|
return 0;
|
|
}
|