mirror of
https://github.com/Tencent/ncnn.git
synced 2026-10-17 04:41:47 +08:00
Layla
1158 lines
39 KiB
C++
1158 lines
39 KiB
C++
// Copyright 2019 BUG1989
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
#ifdef _MSC_VER
|
|
#define _CRT_SECURE_NO_DEPRECATE
|
|
#endif
|
|
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <map>
|
|
#include <set>
|
|
#include <vector>
|
|
|
|
// ncnn public header
|
|
#include "datareader.h"
|
|
#include "layer.h"
|
|
#include "layer_type.h"
|
|
#include "net.h"
|
|
|
|
// ncnn private header
|
|
#include "../modelwriter.h"
|
|
|
|
class DataReaderFromEmpty : public ncnn::DataReader
|
|
{
|
|
public:
|
|
virtual int scan(const char* format, void* p) const
|
|
{
|
|
return 0;
|
|
}
|
|
virtual size_t read(void* buf, size_t size) const
|
|
{
|
|
memset(buf, 0, size);
|
|
return size;
|
|
}
|
|
};
|
|
|
|
static bool read_int8scale_table(const char* filepath, std::map<std::string, ncnn::Mat>& blob_int8scale_table, std::map<std::string, ncnn::Mat>& weight_int8scale_table)
|
|
{
|
|
blob_int8scale_table.clear();
|
|
weight_int8scale_table.clear();
|
|
|
|
FILE* fp = fopen(filepath, "rb");
|
|
if (!fp)
|
|
{
|
|
fprintf(stderr, "Open %s failed.\n", filepath);
|
|
return false;
|
|
}
|
|
|
|
std::string key_str;
|
|
std::vector<float> scales;
|
|
|
|
std::vector<char> line(10240000);
|
|
char* pch = NULL;
|
|
size_t len = 0;
|
|
|
|
while (!feof(fp))
|
|
{
|
|
char* s = fgets(line.data(), (int)line.size(), fp);
|
|
if (!s)
|
|
break;
|
|
|
|
float scale = 1.f;
|
|
char key[256];
|
|
line[strcspn(line.data(), "\r\n")] = 0;
|
|
|
|
pch = strtok(line.data(), " ");
|
|
|
|
if (pch == NULL) break;
|
|
|
|
bool is_key = true;
|
|
while (pch != NULL)
|
|
{
|
|
if (is_key)
|
|
{
|
|
sscanf(pch, "%255s", key);
|
|
|
|
key_str = key;
|
|
is_key = false;
|
|
}
|
|
else
|
|
{
|
|
sscanf(pch, "%f", &scale);
|
|
|
|
scales.push_back(scale);
|
|
}
|
|
|
|
pch = strtok(NULL, " ");
|
|
}
|
|
|
|
// XYZ_param_N pattern
|
|
if (strstr(key_str.c_str(), "_param_"))
|
|
{
|
|
weight_int8scale_table[key_str] = ncnn::Mat((int)scales.size(), (void*)scales.data()).clone();
|
|
}
|
|
else
|
|
{
|
|
blob_int8scale_table[key_str] = ncnn::Mat((int)scales.size(), (void*)scales.data()).clone();
|
|
}
|
|
key_str.clear();
|
|
scales.clear();
|
|
}
|
|
|
|
fclose(fp);
|
|
|
|
return true;
|
|
}
|
|
|
|
class NetQuantize : public ModelWriter
|
|
{
|
|
public:
|
|
NetQuantize();
|
|
|
|
std::map<std::string, ncnn::Mat> blob_int8scale_table;
|
|
std::map<std::string, ncnn::Mat> weight_int8scale_table;
|
|
|
|
public:
|
|
int quantize_convolution();
|
|
int quantize_convolutiondepthwise();
|
|
int quantize_innerproduct();
|
|
|
|
int quantize_rnn();
|
|
int quantize_lstm();
|
|
int quantize_gru();
|
|
|
|
int quantize_embed();
|
|
int quantize_gemm();
|
|
int quantize_multiheadattention();
|
|
int quantize_sdpa();
|
|
|
|
int fuse_requantize();
|
|
|
|
int check_int8scale_table_requirement(const char* int8scale_table_path) const;
|
|
};
|
|
|
|
NetQuantize::NetQuantize()
|
|
: ModelWriter()
|
|
{
|
|
}
|
|
|
|
int NetQuantize::check_int8scale_table_requirement(const char* int8scale_table_path) const
|
|
{
|
|
if (int8scale_table_path)
|
|
return 0;
|
|
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
const std::string& type = layers[i]->type;
|
|
if (type != "Embed" && type != "MultiHeadAttention" && type != "RNN" && type != "LSTM" && type != "GRU")
|
|
continue;
|
|
|
|
fprintf(stderr, "%s (%s): calibration table is required for static weight quantization\n", layers[i]->name.c_str(), type.c_str());
|
|
fprintf(stderr, "run ncnn2table to generate weight scales and pass the table to ncnn2int8\n");
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_convolution()
|
|
{
|
|
const int layer_count = static_cast<int>(layers.size());
|
|
for (int i = 0; i < layer_count; i++)
|
|
{
|
|
// find convolution layer
|
|
if (layers[i]->type != "Convolution")
|
|
continue;
|
|
|
|
// find convolution layer
|
|
std::map<std::string, ncnn::Mat>::iterator iter_data = blob_int8scale_table.find(layers[i]->name);
|
|
if (iter_data == blob_int8scale_table.end())
|
|
continue;
|
|
|
|
char key[256];
|
|
snprintf(key, 256, "%s_param_0", layers[i]->name.c_str());
|
|
|
|
std::map<std::string, ncnn::Mat>::iterator iter = weight_int8scale_table.find(key);
|
|
if (iter == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// Convolution - quantize weight from fp32 to int8
|
|
ncnn::Convolution* convolution = (ncnn::Convolution*)layers[i];
|
|
|
|
ncnn::Mat bottom_blob_int8_scales = iter_data->second;
|
|
ncnn::Mat weight_data_int8_scales = iter->second;
|
|
|
|
fprintf(stderr, "quantize_convolution %s\n", convolution->name.c_str());
|
|
|
|
{
|
|
const int maxk = convolution->kernel_w * convolution->kernel_h;
|
|
const int num_input = convolution->weight_data_size / convolution->num_output / maxk;
|
|
|
|
ncnn::Mat weight_data_r2 = convolution->weight_data.reshape(maxk, num_input, convolution->num_output);
|
|
|
|
ncnn::Mat weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = convolution->weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_data_r2, weight_data_int8, weight_data_int8_scales, opt_q);
|
|
if (weight_data_int8.empty())
|
|
return -100;
|
|
|
|
convolution->weight_data = weight_data_int8.reshape(convolution->weight_data_size);
|
|
}
|
|
|
|
convolution->int8_scale_term = 2;
|
|
convolution->weight_data_int8_scales = weight_data_int8_scales;
|
|
convolution->bottom_blob_int8_scales = bottom_blob_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_convolutiondepthwise()
|
|
{
|
|
const int layer_count = static_cast<int>(layers.size());
|
|
for (int i = 0; i < layer_count; i++)
|
|
{
|
|
// find convolution layer
|
|
if (layers[i]->type != "ConvolutionDepthWise")
|
|
continue;
|
|
|
|
// find convolutiondepthwise layer
|
|
std::map<std::string, ncnn::Mat>::iterator iter_data = blob_int8scale_table.find(layers[i]->name);
|
|
if (iter_data == blob_int8scale_table.end())
|
|
continue;
|
|
|
|
char key[256];
|
|
snprintf(key, 256, "%s_param_0", layers[i]->name.c_str());
|
|
|
|
std::map<std::string, ncnn::Mat>::iterator iter = weight_int8scale_table.find(key);
|
|
if (iter == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// Convolution - quantize weight from fp32 to int8
|
|
ncnn::ConvolutionDepthWise* convdw = (ncnn::ConvolutionDepthWise*)layers[i];
|
|
|
|
ncnn::Mat bottom_blob_int8_scales = iter_data->second;
|
|
ncnn::Mat weight_data_int8_scales = iter->second;
|
|
|
|
fprintf(stderr, "quantize_convolutiondepthwise %s\n", convdw->name.c_str());
|
|
|
|
{
|
|
ncnn::Mat int8_weight_data(convdw->weight_data_size, (size_t)1u);
|
|
if (int8_weight_data.empty())
|
|
return -100;
|
|
|
|
const int weight_data_size_g = convdw->weight_data_size / convdw->group;
|
|
|
|
for (int g = 0; g < convdw->group; g++)
|
|
{
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = int8_weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
|
|
const ncnn::Mat weight_data_g = convdw->weight_data.range(weight_data_size_g * g, weight_data_size_g);
|
|
ncnn::Mat int8_weight_data_g = int8_weight_data.range(weight_data_size_g * g, weight_data_size_g);
|
|
const ncnn::Mat weight_data_int8_scales_g = weight_data_int8_scales.range(g, 1);
|
|
ncnn::quantize_to_int8(weight_data_g, int8_weight_data_g, weight_data_int8_scales_g, opt_q);
|
|
}
|
|
|
|
convdw->weight_data = int8_weight_data;
|
|
}
|
|
|
|
convdw->int8_scale_term = 1;
|
|
convdw->weight_data_int8_scales = weight_data_int8_scales;
|
|
convdw->bottom_blob_int8_scales = bottom_blob_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_innerproduct()
|
|
{
|
|
const int layer_count = static_cast<int>(layers.size());
|
|
for (int i = 0; i < layer_count; i++)
|
|
{
|
|
// find convolution layer
|
|
if (layers[i]->type != "InnerProduct")
|
|
continue;
|
|
|
|
// find InnerProduct layer
|
|
std::map<std::string, ncnn::Mat>::iterator iter_data = blob_int8scale_table.find(layers[i]->name);
|
|
if (iter_data == blob_int8scale_table.end())
|
|
continue;
|
|
|
|
char key[256];
|
|
snprintf(key, 256, "%s_param_0", layers[i]->name.c_str());
|
|
|
|
std::map<std::string, ncnn::Mat>::iterator iter = weight_int8scale_table.find(key);
|
|
if (iter == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// InnerProduct - quantize weight from fp32 to int8
|
|
ncnn::InnerProduct* fc = (ncnn::InnerProduct*)layers[i];
|
|
|
|
ncnn::Mat bottom_blob_int8_scales = iter_data->second;
|
|
ncnn::Mat weight_data_int8_scales = iter->second;
|
|
|
|
fprintf(stderr, "quantize_innerproduct %s\n", fc->name.c_str());
|
|
|
|
{
|
|
const int num_input = fc->weight_data_size / fc->num_output;
|
|
|
|
ncnn::Mat weight_data_r2 = fc->weight_data.reshape(num_input, fc->num_output);
|
|
|
|
ncnn::Mat weight_data_int8;
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_data_r2, weight_data_int8, weight_data_int8_scales, opt_q);
|
|
if (weight_data_int8.empty())
|
|
return -100;
|
|
|
|
fc->weight_data = weight_data_int8.reshape(fc->weight_data_size);
|
|
}
|
|
|
|
fc->int8_scale_term = 2;
|
|
fc->weight_data_int8_scales = weight_data_int8_scales;
|
|
fc->bottom_blob_int8_scales = bottom_blob_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_rnn()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "RNN")
|
|
continue;
|
|
|
|
char key_xc[256];
|
|
snprintf(key_xc, 256, "%s_param_0", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_xc = weight_int8scale_table.find(key_xc);
|
|
if (iter_xc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_hc[256];
|
|
snprintf(key_hc, 256, "%s_param_1", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_hc = weight_int8scale_table.find(key_hc);
|
|
if (iter_hc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// RNN - quantize weight from fp32 to int8
|
|
ncnn::RNN* rnn = (ncnn::RNN*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_rnn %s\n", rnn->name.c_str());
|
|
|
|
const int num_directions = rnn->direction == 2 ? 2 : 1;
|
|
const int size = rnn->weight_data_size / num_directions / rnn->num_output;
|
|
|
|
ncnn::Mat weight_xc_data_int8_scales = iter_xc->second;
|
|
ncnn::Mat weight_hc_data_int8_scales = iter_hc->second;
|
|
|
|
{
|
|
ncnn::Mat weight_xc_data_r2 = rnn->weight_xc_data.reshape(size, rnn->num_output * num_directions);
|
|
|
|
ncnn::Mat weight_xc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = rnn->weight_xc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_xc_data_r2, weight_xc_data_int8, weight_xc_data_int8_scales, opt_q);
|
|
if (weight_xc_data_int8.empty())
|
|
return -100;
|
|
|
|
rnn->weight_xc_data = weight_xc_data_int8.reshape(size * rnn->num_output * num_directions);
|
|
}
|
|
{
|
|
ncnn::Mat weight_hc_data_r2 = rnn->weight_hc_data.reshape(rnn->num_output, rnn->num_output * num_directions);
|
|
|
|
ncnn::Mat weight_hc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = rnn->weight_hc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_hc_data_r2, weight_hc_data_int8, weight_hc_data_int8_scales, opt_q);
|
|
if (weight_hc_data_int8.empty())
|
|
return -100;
|
|
|
|
rnn->weight_hc_data = weight_hc_data_int8.reshape(rnn->num_output * rnn->num_output * num_directions);
|
|
}
|
|
|
|
rnn->int8_scale_term = 2;
|
|
rnn->weight_xc_data_int8_scales = weight_xc_data_int8_scales;
|
|
rnn->weight_hc_data_int8_scales = weight_hc_data_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_lstm()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "LSTM")
|
|
continue;
|
|
|
|
char key_xc[256];
|
|
snprintf(key_xc, 256, "%s_param_0", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_xc = weight_int8scale_table.find(key_xc);
|
|
if (iter_xc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_hc[256];
|
|
snprintf(key_hc, 256, "%s_param_1", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_hc = weight_int8scale_table.find(key_hc);
|
|
if (iter_hc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// LSTM - quantize weight from fp32 to int8
|
|
ncnn::LSTM* lstm = (ncnn::LSTM*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_lstm %s\n", lstm->name.c_str());
|
|
|
|
const int num_directions = lstm->direction == 2 ? 2 : 1;
|
|
const int size = lstm->weight_data_size / num_directions / lstm->hidden_size / 4;
|
|
|
|
ncnn::Mat weight_xc_data_int8_scales = iter_xc->second;
|
|
ncnn::Mat weight_hc_data_int8_scales = iter_hc->second;
|
|
|
|
{
|
|
ncnn::Mat weight_xc_data_r2 = lstm->weight_xc_data.reshape(size, lstm->hidden_size * 4 * num_directions);
|
|
|
|
ncnn::Mat weight_xc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = lstm->weight_xc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_xc_data_r2, weight_xc_data_int8, weight_xc_data_int8_scales, opt_q);
|
|
if (weight_xc_data_int8.empty())
|
|
return -100;
|
|
|
|
lstm->weight_xc_data = weight_xc_data_int8.reshape(size * lstm->hidden_size * 4 * num_directions);
|
|
}
|
|
{
|
|
ncnn::Mat weight_hc_data_r2 = lstm->weight_hc_data.reshape(lstm->num_output, lstm->hidden_size * 4 * num_directions);
|
|
|
|
ncnn::Mat weight_hc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = lstm->weight_hc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_hc_data_r2, weight_hc_data_int8, weight_hc_data_int8_scales, opt_q);
|
|
if (weight_hc_data_int8.empty())
|
|
return -100;
|
|
|
|
lstm->weight_hc_data = weight_hc_data_int8.reshape(lstm->num_output * lstm->hidden_size * 4 * num_directions);
|
|
}
|
|
|
|
lstm->int8_scale_term = 2;
|
|
lstm->weight_xc_data_int8_scales = weight_xc_data_int8_scales;
|
|
lstm->weight_hc_data_int8_scales = weight_hc_data_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_gru()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "GRU")
|
|
continue;
|
|
|
|
char key_xc[256];
|
|
snprintf(key_xc, 256, "%s_param_0", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_xc = weight_int8scale_table.find(key_xc);
|
|
if (iter_xc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_hc[256];
|
|
snprintf(key_hc, 256, "%s_param_1", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_hc = weight_int8scale_table.find(key_hc);
|
|
if (iter_hc == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// GRU - quantize weight from fp32 to int8
|
|
ncnn::GRU* gru = (ncnn::GRU*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_gru %s\n", gru->name.c_str());
|
|
|
|
const int num_directions = gru->direction == 2 ? 2 : 1;
|
|
const int size = gru->weight_data_size / num_directions / gru->num_output / 3;
|
|
|
|
ncnn::Mat weight_xc_data_int8_scales = iter_xc->second;
|
|
ncnn::Mat weight_hc_data_int8_scales = iter_hc->second;
|
|
|
|
{
|
|
ncnn::Mat weight_xc_data_r2 = gru->weight_xc_data.reshape(size, gru->num_output * 3 * num_directions);
|
|
|
|
ncnn::Mat weight_xc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = gru->weight_xc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_xc_data_r2, weight_xc_data_int8, weight_xc_data_int8_scales, opt_q);
|
|
if (weight_xc_data_int8.empty())
|
|
return -100;
|
|
|
|
gru->weight_xc_data = weight_xc_data_int8.reshape(size * gru->num_output * 3 * num_directions);
|
|
}
|
|
{
|
|
ncnn::Mat weight_hc_data_r2 = gru->weight_hc_data.reshape(gru->num_output, gru->num_output * 3 * num_directions);
|
|
|
|
ncnn::Mat weight_hc_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = gru->weight_hc_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(weight_hc_data_r2, weight_hc_data_int8, weight_hc_data_int8_scales, opt_q);
|
|
if (weight_hc_data_int8.empty())
|
|
return -100;
|
|
|
|
gru->weight_hc_data = weight_hc_data_int8.reshape(gru->num_output * gru->num_output * 3 * num_directions);
|
|
}
|
|
|
|
gru->int8_scale_term = 2;
|
|
gru->weight_xc_data_int8_scales = weight_xc_data_int8_scales;
|
|
gru->weight_hc_data_int8_scales = weight_hc_data_int8_scales;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_embed()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "Embed")
|
|
continue;
|
|
|
|
char key[256];
|
|
snprintf(key, 256, "%s_param_0", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter = weight_int8scale_table.find(key);
|
|
if (iter == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// Embed - quantize weight from fp32 to int8
|
|
ncnn::Embed* embed = (ncnn::Embed*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_embed %s\n", embed->name.c_str());
|
|
|
|
const int num_output = embed->num_output;
|
|
const int input_dim = embed->input_dim;
|
|
|
|
ncnn::Mat weight_data_int8_scales = iter->second;
|
|
|
|
{
|
|
ncnn::Mat weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = embed->weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(embed->weight_data, weight_data_int8, weight_data_int8_scales, opt_q);
|
|
if (weight_data_int8.empty())
|
|
return -100;
|
|
|
|
embed->weight_data = weight_data_int8;
|
|
}
|
|
|
|
embed->int8_scale_term = 2;
|
|
embed->weight_data_int8_scale = weight_data_int8_scales[0];
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_gemm()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "Gemm")
|
|
continue;
|
|
|
|
// Gemm - quantize weight from fp32 to int8
|
|
ncnn::Gemm* gemm = (ncnn::Gemm*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_gemm %s\n", gemm->name.c_str());
|
|
|
|
ncnn::Mat A_data_int8_scales;
|
|
ncnn::Mat B_data_int8_scales;
|
|
|
|
char key_a[256];
|
|
char key_b[256];
|
|
snprintf(key_a, 256, "%s_param_0", layers[i]->name.c_str());
|
|
snprintf(key_b, 256, "%s_param_1", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_a = weight_int8scale_table.find(key_a);
|
|
std::map<std::string, ncnn::Mat>::iterator iter_b = weight_int8scale_table.find(key_b);
|
|
|
|
const bool has_weight_scales = (gemm->constantA && iter_a != weight_int8scale_table.end()) || (gemm->constantB && iter_b != weight_int8scale_table.end());
|
|
|
|
if (has_weight_scales && gemm->constantA)
|
|
{
|
|
if (iter_a == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "gemm %s missing scale %s, regenerate the table with ncnn2table\n", gemm->name.c_str(), key_a);
|
|
return -1;
|
|
}
|
|
|
|
A_data_int8_scales = iter_a->second;
|
|
if (A_data_int8_scales.w != gemm->constantM)
|
|
{
|
|
fprintf(stderr, "gemm %s param_0 scale size mismatch\n", gemm->name.c_str());
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
if (has_weight_scales && gemm->constantB)
|
|
{
|
|
if (iter_b == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "gemm %s missing scale %s, regenerate the table with ncnn2table\n", gemm->name.c_str(), key_b);
|
|
return -1;
|
|
}
|
|
|
|
B_data_int8_scales = iter_b->second;
|
|
if (B_data_int8_scales.w != 1)
|
|
{
|
|
fprintf(stderr, "gemm %s param_1 scale size mismatch\n", gemm->name.c_str());
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
if (gemm->constantA)
|
|
{
|
|
if (gemm->transA == 1)
|
|
{
|
|
// transpose for easier quantization
|
|
ncnn::Mat A_data_transposed(gemm->constantK * gemm->constantM);
|
|
for (int i = 0; i < gemm->constantM; i++)
|
|
{
|
|
float* ptr = (float*)A_data_transposed + i * gemm->constantK;
|
|
for (int j = 0; j < gemm->constantK; j++)
|
|
{
|
|
ptr[j] = gemm->A_data[j * gemm->constantM + i];
|
|
}
|
|
}
|
|
gemm->A_data = A_data_transposed;
|
|
gemm->transA = 0;
|
|
}
|
|
|
|
if (A_data_int8_scales.empty())
|
|
{
|
|
A_data_int8_scales.create(gemm->constantM);
|
|
for (int i = 0; i < gemm->constantM; i++)
|
|
{
|
|
float absmax = 0.f;
|
|
const float* ptr = (const float*)gemm->A_data + i * gemm->constantK;
|
|
for (int j = 0; j < gemm->constantK; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[j]));
|
|
}
|
|
|
|
A_data_int8_scales[i] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
}
|
|
|
|
gemm->A_data_int8_scales = A_data_int8_scales;
|
|
|
|
ncnn::Mat A_data = gemm->A_data.reshape(gemm->constantK, gemm->constantM);
|
|
ncnn::Mat A_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = A_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(A_data, A_data_int8, gemm->A_data_int8_scales, opt_q);
|
|
if (A_data_int8.empty())
|
|
return -100;
|
|
|
|
gemm->A_data = A_data_int8.reshape(gemm->constantK * gemm->constantM);
|
|
}
|
|
|
|
if (gemm->constantB)
|
|
{
|
|
if (gemm->transB == 0)
|
|
{
|
|
// transpose for easier quantization
|
|
ncnn::Mat B_data_transposed(gemm->constantK * gemm->constantN);
|
|
for (int i = 0; i < gemm->constantN; i++)
|
|
{
|
|
float* ptr = (float*)B_data_transposed + i * gemm->constantK;
|
|
for (int j = 0; j < gemm->constantK; j++)
|
|
{
|
|
ptr[j] = gemm->B_data[j * gemm->constantN + i];
|
|
}
|
|
}
|
|
gemm->B_data = B_data_transposed;
|
|
gemm->transB = 1;
|
|
}
|
|
|
|
if (B_data_int8_scales.empty())
|
|
{
|
|
const float* ptr = gemm->B_data;
|
|
float absmax = 0.f;
|
|
const int b_data_size = gemm->B_data.w * gemm->B_data.h;
|
|
for (int j = 0; j < b_data_size; j++)
|
|
{
|
|
absmax = std::max(absmax, (float)fabs(ptr[j]));
|
|
}
|
|
|
|
B_data_int8_scales.create(1);
|
|
B_data_int8_scales[0] = absmax == 0.f ? 1.f : 127 / absmax;
|
|
}
|
|
|
|
gemm->B_data_int8_scale = B_data_int8_scales[0];
|
|
|
|
ncnn::Mat B_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = gemm->B_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(gemm->B_data, B_data_int8, B_data_int8_scales, opt_q);
|
|
if (B_data_int8.empty())
|
|
return -100;
|
|
|
|
gemm->B_data = B_data_int8;
|
|
}
|
|
|
|
gemm->quantize_term = 2;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_multiheadattention()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "MultiHeadAttention")
|
|
continue;
|
|
|
|
char key_q[256];
|
|
snprintf(key_q, 256, "%s_param_0", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_q = weight_int8scale_table.find(key_q);
|
|
if (iter_q == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_k[256];
|
|
snprintf(key_k, 256, "%s_param_1", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_k = weight_int8scale_table.find(key_k);
|
|
if (iter_k == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_v[256];
|
|
snprintf(key_v, 256, "%s_param_2", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_v = weight_int8scale_table.find(key_v);
|
|
if (iter_v == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
char key_out[256];
|
|
snprintf(key_out, 256, "%s_param_3", layers[i]->name.c_str());
|
|
std::map<std::string, ncnn::Mat>::iterator iter_out = weight_int8scale_table.find(key_out);
|
|
if (iter_out == weight_int8scale_table.end())
|
|
{
|
|
fprintf(stderr, "this layer need to be quantized, but no scale param!\n");
|
|
return -1;
|
|
}
|
|
|
|
// MultiHeadAttention - quantize weight from fp32 to int8
|
|
ncnn::MultiHeadAttention* mha = (ncnn::MultiHeadAttention*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_multiheadattention %s\n", mha->name.c_str());
|
|
|
|
const int qdim = mha->weight_data_size / mha->embed_dim;
|
|
|
|
{
|
|
mha->q_weight_data_int8_scales = iter_q->second;
|
|
|
|
ncnn::Mat q_weight_data = mha->q_weight_data.reshape(qdim, mha->embed_dim);
|
|
ncnn::Mat q_weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = q_weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(q_weight_data, q_weight_data_int8, mha->q_weight_data_int8_scales, opt_q);
|
|
if (q_weight_data_int8.empty())
|
|
return -100;
|
|
|
|
mha->q_weight_data = q_weight_data_int8.reshape(qdim * mha->embed_dim);
|
|
}
|
|
|
|
{
|
|
mha->k_weight_data_int8_scales = iter_k->second;
|
|
|
|
ncnn::Mat k_weight_data = mha->k_weight_data.reshape(mha->kdim, mha->embed_dim);
|
|
ncnn::Mat k_weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = k_weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(k_weight_data, k_weight_data_int8, mha->k_weight_data_int8_scales, opt_q);
|
|
if (k_weight_data_int8.empty())
|
|
return -100;
|
|
|
|
mha->k_weight_data = k_weight_data_int8.reshape(mha->kdim * mha->embed_dim);
|
|
}
|
|
|
|
{
|
|
mha->v_weight_data_int8_scales = iter_v->second;
|
|
|
|
ncnn::Mat v_weight_data = mha->v_weight_data.reshape(mha->vdim, mha->embed_dim);
|
|
ncnn::Mat v_weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = v_weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(v_weight_data, v_weight_data_int8, mha->v_weight_data_int8_scales, opt_q);
|
|
if (v_weight_data_int8.empty())
|
|
return -100;
|
|
|
|
mha->v_weight_data = v_weight_data_int8.reshape(mha->vdim * mha->embed_dim);
|
|
}
|
|
|
|
{
|
|
ncnn::Mat out_weight_data_int8_scales = iter_out->second;
|
|
mha->out_weight_data_int8_scale = out_weight_data_int8_scales[0];
|
|
|
|
ncnn::Mat out_weight_data_int8;
|
|
|
|
ncnn::Option opt_q = opt;
|
|
opt_q.blob_allocator = mha->out_weight_data.allocator;
|
|
opt_q.use_packing_layout = false;
|
|
ncnn::quantize_to_int8(mha->out_weight_data, out_weight_data_int8, out_weight_data_int8_scales, opt_q);
|
|
if (out_weight_data_int8.empty())
|
|
return -100;
|
|
|
|
mha->out_weight_data = out_weight_data_int8;
|
|
}
|
|
|
|
mha->int8_scale_term = 2;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::quantize_sdpa()
|
|
{
|
|
for (size_t i = 0; i < layers.size(); i++)
|
|
{
|
|
if (layers[i]->type != "SDPA")
|
|
continue;
|
|
|
|
ncnn::SDPA* sdpa = (ncnn::SDPA*)layers[i];
|
|
|
|
fprintf(stderr, "quantize_sdpa %s\n", sdpa->name.c_str());
|
|
|
|
// SDPA uses dynamic activation quantization in forward_int8
|
|
|
|
sdpa->int8_scale_term = 2;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NetQuantize::fuse_requantize()
|
|
{
|
|
const size_t layer_count = layers.size();
|
|
for (size_t i = 0; i < layer_count; i++)
|
|
{
|
|
if (layers[i]->type != "Convolution" && layers[i]->type != "ConvolutionDepthWise")
|
|
continue;
|
|
|
|
// Convolution/ConvolutionDepthWise - Convolution/ConvolutionDepthWise
|
|
int top_blob_index = layers[i]->tops[0];
|
|
|
|
size_t j = i + 1;
|
|
for (; j < layer_count; j++)
|
|
{
|
|
if (layers[j]->type != "Convolution" && layers[j]->type != "ConvolutionDepthWise")
|
|
continue;
|
|
|
|
if (layers[j]->bottoms.size() != 1)
|
|
continue;
|
|
|
|
if (layers[j]->bottoms[0] == top_blob_index)
|
|
break;
|
|
}
|
|
|
|
if (j == layer_count)
|
|
continue;
|
|
|
|
// fuse requantize
|
|
fprintf(stderr, "fuse_requantize %s %s\n", layers[i]->name.c_str(), layers[j]->name.c_str());
|
|
|
|
if (layers[i]->type == "Convolution" && layers[j]->type == "Convolution")
|
|
{
|
|
ncnn::Convolution* convolution1 = (ncnn::Convolution*)layers[i];
|
|
ncnn::Convolution* convolution2 = (ncnn::Convolution*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "Convolution" && layers[j]->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::Convolution* convolution1 = (ncnn::Convolution*)layers[i];
|
|
ncnn::ConvolutionDepthWise* convolution2 = (ncnn::ConvolutionDepthWise*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "ConvolutionDepthWise" && layers[j]->type == "Convolution")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolution1 = (ncnn::ConvolutionDepthWise*)layers[i];
|
|
ncnn::Convolution* convolution2 = (ncnn::Convolution*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "ConvolutionDepthWise" && layers[j]->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolution1 = (ncnn::ConvolutionDepthWise*)layers[i];
|
|
ncnn::ConvolutionDepthWise* convolution2 = (ncnn::ConvolutionDepthWise*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < layer_count; i++)
|
|
{
|
|
if (layers[i]->type != "Convolution" && layers[i]->type != "ConvolutionDepthWise")
|
|
continue;
|
|
|
|
// Convolution/ConvolutionDepthWise - Split - Convolution/ConvolutionDepthWise
|
|
int top_blob_index = layers[i]->tops[0];
|
|
|
|
size_t j = i + 1;
|
|
for (; j < layer_count; j++)
|
|
{
|
|
if (layers[j]->type != "Split")
|
|
continue;
|
|
|
|
if (layers[j]->bottoms.size() != 1)
|
|
continue;
|
|
|
|
if (layers[j]->bottoms[0] == top_blob_index)
|
|
break;
|
|
}
|
|
|
|
if (j == layer_count)
|
|
continue;
|
|
|
|
ncnn::Split* split = (ncnn::Split*)layers[j];
|
|
|
|
bool all_conv = true;
|
|
for (size_t p = 0; p < split->tops.size(); p++)
|
|
{
|
|
int split_top_blob_index = split->tops[p];
|
|
|
|
size_t k = j + 1;
|
|
for (; k < layer_count; k++)
|
|
{
|
|
if (layers[k]->type != "Convolution" && layers[k]->type != "ConvolutionDepthWise")
|
|
continue;
|
|
|
|
if (layers[k]->bottoms.size() != 1)
|
|
continue;
|
|
|
|
if (layers[k]->bottoms[0] == split_top_blob_index)
|
|
break;
|
|
}
|
|
|
|
if (k == layer_count)
|
|
{
|
|
all_conv = false;
|
|
break;
|
|
}
|
|
|
|
if (layers[k]->type == "Convolution")
|
|
{
|
|
ncnn::Convolution* convolution = (ncnn::Convolution*)layers[k];
|
|
if (convolution->weight_data.elemsize != 1u)
|
|
{
|
|
all_conv = false;
|
|
break;
|
|
}
|
|
}
|
|
if (layers[k]->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolution = (ncnn::ConvolutionDepthWise*)layers[k];
|
|
if (convolution->weight_data.elemsize != 1u)
|
|
{
|
|
all_conv = false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!all_conv)
|
|
continue;
|
|
|
|
j = blobs[split->tops[0]].consumer;
|
|
|
|
// fuse requantize
|
|
fprintf(stderr, "fuse_requantize %s %s\n", layers[i]->name.c_str(), split->name.c_str());
|
|
|
|
if (layers[i]->type == "Convolution" && layers[j]->type == "Convolution")
|
|
{
|
|
ncnn::Convolution* convolution1 = (ncnn::Convolution*)layers[i];
|
|
ncnn::Convolution* convolution2 = (ncnn::Convolution*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "Convolution" && layers[j]->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::Convolution* convolution1 = (ncnn::Convolution*)layers[i];
|
|
ncnn::ConvolutionDepthWise* convolution2 = (ncnn::ConvolutionDepthWise*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "ConvolutionDepthWise" && layers[j]->type == "Convolution")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolution1 = (ncnn::ConvolutionDepthWise*)layers[i];
|
|
ncnn::Convolution* convolution2 = (ncnn::Convolution*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
if (layers[i]->type == "ConvolutionDepthWise" && layers[j]->type == "ConvolutionDepthWise")
|
|
{
|
|
ncnn::ConvolutionDepthWise* convolution1 = (ncnn::ConvolutionDepthWise*)layers[i];
|
|
ncnn::ConvolutionDepthWise* convolution2 = (ncnn::ConvolutionDepthWise*)layers[j];
|
|
|
|
if (convolution1->weight_data.elemsize != 1u || convolution2->weight_data.elemsize != 1u)
|
|
continue;
|
|
|
|
convolution1->int8_scale_term += 100;
|
|
convolution1->top_blob_int8_scales = convolution2->bottom_blob_int8_scales;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int main(int argc, char** argv)
|
|
{
|
|
if (argc != 5 && argc != 6)
|
|
{
|
|
fprintf(stderr, "usage: %s [inparam] [inbin] [outparam] [outbin] [calibration table]\n", argv[0]);
|
|
return -1;
|
|
}
|
|
|
|
const char* inparam = argv[1];
|
|
const char* inbin = argv[2];
|
|
const char* outparam = argv[3];
|
|
const char* outbin = argv[4];
|
|
const char* int8scale_table_path = argc == 6 ? argv[5] : NULL;
|
|
|
|
NetQuantize quantizer;
|
|
quantizer.storage_type = 1; // use fp16 where int8 not applied
|
|
|
|
// parse the calibration scale table
|
|
if (int8scale_table_path)
|
|
{
|
|
bool s2 = read_int8scale_table(int8scale_table_path, quantizer.blob_int8scale_table, quantizer.weight_int8scale_table);
|
|
if (!s2)
|
|
{
|
|
fprintf(stderr, "read_int8scale_table failed\n");
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
quantizer.load_param(inparam);
|
|
if (strcmp(inbin, "null") == 0)
|
|
{
|
|
DataReaderFromEmpty dr;
|
|
quantizer.load_model(dr);
|
|
quantizer.gen_random_weight = true;
|
|
}
|
|
else
|
|
quantizer.load_model(inbin);
|
|
|
|
if (quantizer.check_int8scale_table_requirement(int8scale_table_path) != 0)
|
|
return -1;
|
|
|
|
quantizer.quantize_convolution();
|
|
quantizer.quantize_convolutiondepthwise();
|
|
quantizer.quantize_innerproduct();
|
|
|
|
quantizer.quantize_rnn();
|
|
quantizer.quantize_lstm();
|
|
quantizer.quantize_gru();
|
|
quantizer.quantize_embed();
|
|
if (quantizer.quantize_gemm() != 0)
|
|
return -1;
|
|
quantizer.quantize_multiheadattention();
|
|
quantizer.quantize_sdpa();
|
|
|
|
quantizer.fuse_requantize();
|
|
|
|
quantizer.save(outparam, outbin);
|
|
|
|
return 0;
|
|
}
|