GithubStar/ncnn
1
0
Fork 0
mirror of https://github.com/Tencent/ncnn.git synced 2026-10-17 04:41:47 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
ncnn/tests/test_paramdict.cpp
T
nihui 5ea98309e3 layer param hardening (#7003) 
* harden layer parameter validation

Original patches and author identities:
https://github.com/Tencent/ncnn/pull/6912 (beilzx)
https://github.com/Tencent/ncnn/pull/6919 (FluxMind-O)
https://github.com/Tencent/ncnn/pull/6928 (lxcxjxhx)
https://github.com/Tencent/ncnn/pull/6948 (ktz03)
https://github.com/Tencent/ncnn/pull/6950 (ktz03)
https://github.com/Tencent/ncnn/pull/6971 (LI)
https://github.com/Tencent/ncnn/pull/6974 (LI)
https://github.com/Tencent/ncnn/pull/6977 (LI)

Co-authored-by: beilzx <2069360659@qq.com>
Co-authored-by: FluxMind-O <3283918317@qq.com>
Co-authored-by: lxcxjxhx <lxcxjxhx@users.noreply.github.com>
Co-authored-by: ktz03 <2484593937@qq.com>
Co-authored-by: LI <2484593937@qq.com>
2026-09-15 19:18:46 +08:00

1456 lines
45 KiB
C++
Raw Permalink Blame History

// Copyright 2025 Tencent
// SPDX-License-Identifier: BSD-3-Clause
#include <stdio.h>
#include <limits.h>
#include <float.h>
#include <string.h>
#include "datareader.h"
#include "testutil.h"
static int test_paramdict_0()
{
TestParamDict pdt;
pdt.load_param("0=100 1=1,-1,4,5,1,4 2=1.250000 -23303=5,0.1,0.2,-0.4,0.8,1.0 -23304=3,-1,10,-88");
// int
int typei = pdt.type(0);
if (typei != 2)
{
fprintf(stderr, "test_paramdict int type failed %d != 2\n", typei);
return -1;
}
int i = pdt.get(0, 0);
if (i != 100)
{
fprintf(stderr, "test_paramdict int value failed %d != 100\n", i);
return -1;
}
// int array
int typeai = pdt.type(1);
if (typeai != 5)
{
fprintf(stderr, "test_paramdict int array type failed %d != 5\n", typeai);
return -1;
}
ncnn::Mat ai = pdt.get(1, ncnn::Mat());
if (ai.w != 6)
{
fprintf(stderr, "test_paramdict int array size failed %d != 6\n", ai.w);
return -1;
}
const int* p = ai;
if (p[0] != 1 || p[1] != -1 || p[2] != 4 || p[3] != 5 || p[4] != 1 || p[5] != 4)
{
fprintf(stderr, "test_paramdict int array value failed %d %d %d %d %d %d\n", p[0], p[1], p[2], p[3], p[4], p[5]);
return -1;
}
// float
int typef = pdt.type(2);
if (typef != 3)
{
fprintf(stderr, "test_paramdict float type failed %d != 3\n", typef);
return -1;
}
float f = pdt.get(2, 0.f);
if (f != 1.25f)
{
fprintf(stderr, "test_paramdict float value failed %f != 1.25f\n", f);
return -1;
}
// float array
int typeaf = pdt.type(3);
if (typeaf != 6)
{
fprintf(stderr, "test_paramdict float array type failed %d != 6\n", typeaf);
return -1;
}
ncnn::Mat af = pdt.get(3, ncnn::Mat());
if (af.w != 5)
{
fprintf(stderr, "test_paramdict float array size failed %d != 5\n", af.w);
return -1;
}
if (af[0] != 0.1f || af[1] != 0.2f || af[2] != -0.4f || af[3] != 0.8f || af[4] != 1.0f)
{
fprintf(stderr, "test_paramdict float array value failed %f %f %f %f %f\n", af[0], af[1], af[2], af[3], af[4]);
return -1;
}
// int array
typeai = pdt.type(4);
if (typeai != 5)
{
fprintf(stderr, "test_paramdict int array type failed %d != 5\n", typeai);
return -1;
}
ai = pdt.get(4, ncnn::Mat());
if (ai.w != 3)
{
fprintf(stderr, "test_paramdict int array size failed %d != 3\n", ai.w);
return -1;
}
p = ai;
if (p[0] != -1 || p[1] != 10 || p[2] != -88)
{
fprintf(stderr, "test_paramdict int array value failed %d %d %d\n", p[0], p[1], p[2]);
return -1;
}
return 0;
}
static int test_paramdict_1()
{
TestParamDict pdt;
pdt.load_param("0=-1 1=4, 2=0.01 3=-1.45e-2,3.14");
// int
int typei = pdt.type(0);
if (typei != 2)
{
fprintf(stderr, "test_paramdict int type failed %d != 2\n", typei);
return -1;
}
int i = pdt.get(0, 0);
if (i != -1)
{
fprintf(stderr, "test_paramdict int value failed %d != -1\n", i);
return -1;
}
// int array
int typeai = pdt.type(1);
if (typeai != 5)
{
fprintf(stderr, "test_paramdict int array type failed %d != 5\n", typeai);
return -1;
}
ncnn::Mat ai = pdt.get(1, ncnn::Mat());
if (ai.w != 1)
{
fprintf(stderr, "test_paramdict int array size failed %d != 1\n", ai.w);
return -1;
}
const int* p = ai;
if (p[0] != 4)
{
fprintf(stderr, "test_paramdict int array value failed %d\n", p[0]);
return -1;
}
// float
int typef = pdt.type(2);
if (typef != 3)
{
fprintf(stderr, "test_paramdict float type failed %d != 3\n", typef);
return -1;
}
float f = pdt.get(2, 0.f);
if (f != 0.01f)
{
fprintf(stderr, "test_paramdict float value failed %f != 0.01f\n", f);
return -1;
}
// float array
int typeaf = pdt.type(3);
if (typeaf != 6)
{
fprintf(stderr, "test_paramdict float array type failed %d != 6\n", typeaf);
return -1;
}
ncnn::Mat af = pdt.get(3, ncnn::Mat());
if (af.w != 2)
{
fprintf(stderr, "test_paramdict float array size failed %d != 2\n", af.w);
return -1;
}
if (af[0] != -0.0145f || af[1] != 3.14f)
{
fprintf(stderr, "test_paramdict float array value failed %f %f\n", af[0], af[1]);
return -1;
}
return 0;
}
static int test_paramdict_2()
{
TestParamDict pdt;
pdt.load_param("0=bij,bjk->bik 1=This_is_a_very_long_long_string 3=\"1,2,3 and 6.667 zzz\" 2=\"X\" 6=\"qwqwqwq\"");
// string
int types = pdt.type(0);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
std::string s = pdt.get(0, "");
if (s != "bij,bjk->bik")
{
fprintf(stderr, "test_paramdict string text failed %s != bij,bjk->bik\n", s.c_str());
return -1;
}
// string
types = pdt.type(1);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(1, "");
if (s != "This_is_a_very_long_long_string")
{
fprintf(stderr, "test_paramdict string text failed %s != This_is_a_very_long_long_string\n", s.c_str());
return -1;
}
// string
types = pdt.type(2);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(2, "");
if (s != "X")
{
fprintf(stderr, "test_paramdict string text failed %s != X\n", s.c_str());
return -1;
}
// string
types = pdt.type(3);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(3, "");
if (s != "1,2,3 and 6.667 zzz")
{
fprintf(stderr, "test_paramdict string text failed %s != \"1,2,3 and 6.667 zzz\"\n", s.c_str());
return -1;
}
// string
types = pdt.type(6);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(6, "");
if (s != "qwqwqwq")
{
fprintf(stderr, "test_paramdict string text failed %s != \"qwqwqwq\"\n", s.c_str());
return -1;
}
return 0;
}
static int test_paramdict_3()
{
const unsigned char mem[] = {
0x00, 0x00, 0x00, 0x00,
0x64, 0x00, 0x00, 0x00,
0xfb, 0xa4, 0xff, 0xff,
0x06, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff,
0x04, 0x00, 0x00, 0x00,
0x05, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00,
0x04, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x00,
0x00, 0x00, 0xa0, 0x3f,
0xf9, 0xa4, 0xff, 0xff,
0x05, 0x00, 0x00, 0x00,
0xcd, 0xcc, 0xcc, 0x3d,
0xcd, 0xcc, 0x4c, 0x3e,
0xcd, 0xcc, 0xcc, 0xbe,
0xcd, 0xcc, 0x4c, 0x3f,
0x00, 0x00, 0x80, 0x3f,
0x17, 0xff, 0xff, 0xff
};
TestParamDict pdt;
pdt.load_param_bin(mem);
// int
int typei = pdt.type(0);
if (typei != 1)
{
fprintf(stderr, "test_paramdict int type failed %d != 1\n", typei);
return -1;
}
int i = pdt.get(0, 0);
if (i != 100)
{
fprintf(stderr, "test_paramdict int value failed %d != 100\n", i);
return -1;
}
// int array
int typeai = pdt.type(1);
if (typeai != 4)
{
fprintf(stderr, "test_paramdict int array type failed %d != 4\n", typeai);
return -1;
}
ncnn::Mat ai = pdt.get(1, ncnn::Mat());
if (ai.w != 6)
{
fprintf(stderr, "test_paramdict int array size failed %d != 6\n", ai.w);
return -1;
}
const int* p = ai;
if (p[0] != 1 || p[1] != -1 || p[2] != 4 || p[3] != 5 || p[4] != 1 || p[5] != 4)
{
fprintf(stderr, "test_paramdict int array value failed %d %d %d %d %d %d\n", p[0], p[1], p[2], p[3], p[4], p[5]);
return -1;
}
// float
int typef = pdt.type(2);
if (typef != 1)
{
fprintf(stderr, "test_paramdict float type failed %d != 1\n", typef);
return -1;
}
float f = pdt.get(2, 0.f);
if (f != 1.25f)
{
fprintf(stderr, "test_paramdict float value failed %f != 1.25f\n", f);
return -1;
}
// float array
int typeaf = pdt.type(3);
if (typeaf != 4)
{
fprintf(stderr, "test_paramdict float array type failed %d != 4\n", typeaf);
return -1;
}
ncnn::Mat af = pdt.get(3, ncnn::Mat());
if (af.w != 5)
{
fprintf(stderr, "test_paramdict float array size failed %d != 5\n", af.w);
return -1;
}
if (af[0] != 0.1f || af[1] != 0.2f || af[2] != -0.4f || af[3] != 0.8f || af[4] != 1.0f)
{
fprintf(stderr, "test_paramdict float array value failed %f %f %f %f %f\n", af[0], af[1], af[2], af[3], af[4]);
return -1;
}
return 0;
}
static int test_paramdict_4()
{
const unsigned char mem[] = {
0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff,
0xfb, 0xa4, 0xff, 0xff,
0x01, 0x00, 0x00, 0x00,
0x04, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x00,
0x0a, 0xd7, 0x23, 0x3c,
0xf9, 0xa4, 0xff, 0xff,
0x02, 0x00, 0x00, 0x00,
0x68, 0x91, 0x6d, 0xbc,
0xc3, 0xf5, 0x48, 0x40,
0x17, 0xff, 0xff, 0xff
};
TestParamDict pdt;
pdt.load_param_bin(mem);
// int
int typei = pdt.type(0);
if (typei != 1)
{
fprintf(stderr, "test_paramdict int type failed %d != 1\n", typei);
return -1;
}
int i = pdt.get(0, 0);
if (i != -1)
{
fprintf(stderr, "test_paramdict int value failed %d != -1\n", i);
return -1;
}
// int array
int typeai = pdt.type(1);
if (typeai != 4)
{
fprintf(stderr, "test_paramdict int array type failed %d != 4\n", typeai);
return -1;
}
ncnn::Mat ai = pdt.get(1, ncnn::Mat());
if (ai.w != 1)
{
fprintf(stderr, "test_paramdict int array size failed %d != 1\n", ai.w);
return -1;
}
const int* p = ai;
if (p[0] != 4)
{
fprintf(stderr, "test_paramdict int array value failed %d\n", p[0]);
return -1;
}
// float
int typef = pdt.type(2);
if (typef != 1)
{
fprintf(stderr, "test_paramdict float type failed %d != 1\n", typef);
return -1;
}
float f = pdt.get(2, 0.f);
if (f != 0.01f)
{
fprintf(stderr, "test_paramdict float value failed %f != 0.01f\n", f);
return -1;
}
// float array
int typeaf = pdt.type(3);
if (typeaf != 4)
{
fprintf(stderr, "test_paramdict float array type failed %d != 4\n", typeaf);
return -1;
}
ncnn::Mat af = pdt.get(3, ncnn::Mat());
if (af.w != 2)
{
fprintf(stderr, "test_paramdict float array size failed %d != 2\n", af.w);
return -1;
}
if (af[0] != -0.0145f || af[1] != 3.14f)
{
fprintf(stderr, "test_paramdict float array value failed %f %f\n", af[0], af[1]);
return -1;
}
return 0;
}
static int test_paramdict_5()
{
const unsigned char mem[] = {
0x98, 0xa4, 0xff, 0xff,
0x0c, 0x00, 0x00, 0x00,
0x62, 0x69, 0x6a, 0x2c,
0x62, 0x6a, 0x6b, 0x2d,
0x3e, 0x62, 0x69, 0x6b,
0x97, 0xa4, 0xff, 0xff,
0x1f, 0x00, 0x00, 0x00,
0x54, 0x68, 0x69, 0x73,
0x5f, 0x69, 0x73, 0x5f,
0x61, 0x5f, 0x76, 0x65,
0x72, 0x79, 0x5f, 0x6c,
0x6f, 0x6e, 0x67, 0x5f,
0x6c, 0x6f, 0x6e, 0x67,
0x5f, 0x73, 0x74, 0x72,
0x69, 0x6e, 0x67, 0x00,
0x96, 0xa4, 0xff, 0xff,
0x01, 0x00, 0x00, 0x00,
0x58, 0x00, 0x00, 0x00,
0x17, 0xff, 0xff, 0xff
};
TestParamDict pdt;
pdt.load_param_bin(mem);
// string
int types = pdt.type(0);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
std::string s = pdt.get(0, "");
if (s != "bij,bjk->bik")
{
fprintf(stderr, "test_paramdict string text failed %s != bij,bjk->bik\n", s.c_str());
return -1;
}
// string
types = pdt.type(1);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(1, "");
if (s != "This_is_a_very_long_long_string")
{
fprintf(stderr, "test_paramdict string text failed %s != This_is_a_very_long_long_string\n", s.c_str());
return -1;
}
// string
types = pdt.type(2);
if (types != 7)
{
fprintf(stderr, "test_paramdict string type failed %d != 7\n", types);
return -1;
}
s = pdt.get(2, "");
if (s != "X")
{
fprintf(stderr, "test_paramdict string text failed %s != X\n", s.c_str());
return -1;
}
return 0;
}
static int compare_paramdict(const ncnn::ParamDict& pd, const ncnn::ParamDict& pd0)
{
for (int id = 0; id < NCNN_MAX_PARAM_COUNT; id++)
{
const int type0 = pd0.type(id);
if (pd.type(id) != type0)
{
fprintf(stderr, "compare_paramdict type failed id=%d: %d != %d\n", id, pd.type(id), type0);
return -1;
}
if (type0 == 0)
{
continue;
}
else if (type0 == 2)
{
const int i0 = pd0.get(id, 0);
int i = pd.get(id, 0);
if (i != i0)
{
fprintf(stderr, "compare_paramdict int failed %d != %d\n", i, i0);
return -1;
}
}
else if (type0 == 3)
{
const float f0 = pd0.get(id, 0.f);
float f = pd.get(id, 0.f);
if (f != f0)
{
fprintf(stderr, "compare_paramdict float failed %f != %f\n", f, f0);
return -1;
}
}
else if (type0 == 4 || type0 == 5)
{
const ncnn::Mat ai0 = pd0.get(id, ncnn::Mat());
ncnn::Mat ai = pd.get(id, ncnn::Mat());
if (ai.w != ai0.w)
{
fprintf(stderr, "compare_paramdict int array size failed %d != %d\n", ai.w, ai0.w);
return -1;
}
for (int q = 0; q < ai0.w; q++)
{
int i0 = ((const int*)ai0)[q];
int i = ((const int*)ai)[q];
if (i != i0)
{
fprintf(stderr, "compare_paramdict int array element %d failed %d != %d\n", q, i, i0);
return -1;
}
}
}
else if (type0 == 6)
{
const ncnn::Mat af0 = pd0.get(id, ncnn::Mat());
ncnn::Mat af = pd.get(id, ncnn::Mat());
if (af.w != af0.w)
{
fprintf(stderr, "compare_paramdict float array size failed %d != %d\n", af.w, af0.w);
return -1;
}
for (int q = 0; q < af0.w; q++)
{
float f0 = af0[q];
float f = af[q];
if (f != f0)
{
fprintf(stderr, "compare_paramdict float array element %d failed %f != %f\n", q, f, f0);
return -1;
}
}
}
else if (type0 == 7)
{
const std::string s0 = pd0.get(id, "");
std::string s = pd.get(id, "");
if (s != s0)
{
fprintf(stderr, "compare_paramdict string failed %s != %s\n", s.c_str(), s0.c_str());
return -1;
}
}
else
{
fprintf(stderr, "unexpected paramdict type %d\n", type0);
return -1;
}
}
return 0;
}
static int test_paramdict_6()
{
const int i0 = 11;
const float f0 = -2.2f;
const std::string s0 = "qwqwqwq";
ncnn::Mat ai0(1);
{
int* p = ai0;
p[0] = 233;
}
ncnn::Mat af0(4);
{
float* p = af0;
p[0] = 2.33f;
p[1] = -0.2f;
p[2] = 0.f;
p[3] = 9494.f;
}
ncnn::ParamDict pd0;
pd0.set(1, i0);
pd0.set(2, ai0);
pd0.set(3, f0);
pd0.set(4, af0);
pd0.set(5, s0);
// copy
{
ncnn::ParamDict pd(pd0);
int ret = compare_paramdict(pd, pd0);
if (ret != 0)
{
fprintf(stderr, "paramdict copy failed\n");
return -1;
}
}
// assign
{
ncnn::ParamDict pd;
pd = pd0;
int ret = compare_paramdict(pd, pd0);
if (ret != 0)
{
fprintf(stderr, "paramdict assign failed\n");
return -1;
}
}
return 0;
}
static int test_paramdict_access()
{
ncnn::ParamDict pd;
ncnn::Mat array(1);
((int*)array)[0] = 17;
const std::string text = "text";
pd.set(0, 10);
pd.set(1, 2.5f);
pd.set(2, array);
pd.set(3, text);
pd.set(31, 31);
if (pd.get(0, 0) != 10 || pd.get(0, 7.f) != 10.f || pd.get(31, 0) != 31)
{
fprintf(stderr, "ParamDict valid id access failed\n");
return -1;
}
if (pd.get(1, 7) != 7
|| pd.get(2, 7) != 7 || pd.get(2, 7.f) != 7.f
|| pd.get(3, 7) != 7 || pd.get(3, 7.f) != 7.f
|| pd.get(0, array).data != array.data || pd.get(0, text) != text
|| pd.get(3, array).data != array.data || pd.get(2, text) != text)
{
fprintf(stderr, "ParamDict mismatched type access failed\n");
return -1;
}
// retagging and copying must not expose an earlier array or string value
pd.set(2, 22);
pd.set(3, 33);
ncnn::ParamDict copied(pd);
ncnn::ParamDict assigned;
assigned.set(2, array);
assigned.set(3, text);
assigned = pd;
const ncnn::ParamDict& alias = assigned;
assigned = alias;
ncnn::Mat fallback_array(1);
((int*)fallback_array)[0] = 99;
const std::string fallback_text = "fallback";
if (copied.get(2, fallback_array).data != fallback_array.data || copied.get(3, fallback_text) != fallback_text
|| assigned.get(2, fallback_array).data != fallback_array.data || assigned.get(3, fallback_text) != fallback_text
|| copied.get(2, 0) != 22 || assigned.get(3, 0) != 33)
{
fprintf(stderr, "ParamDict retagged copy access failed\n");
return -1;
}
pd.set(4, ncnn::Mat());
if (pd.type(4) != 4 || !pd.get(4, array).empty()
|| pd.type(5) != 0 || pd.get(5, array).data != array.data)
{
fprintf(stderr, "ParamDict empty array setter default failed\n");
return -1;
}
return 0;
}
static int test_paramdict_numeric_conversion()
{
const int integers[] = {0, 6, -6, 16777217, INT_MIN, INT_MAX};
const float expected[] = {0.f, 6.f, -6.f, 16777216.f, -2147483648.f, 2147483648.f};
for (size_t j = 0; j < sizeof(integers) / sizeof(integers[0]); j++)
for (int mode = 0; mode < 2; mode++)
{
TestParamDict pd;
if (mode == 0)
{
pd.set(0, integers[j]);
pd.set(1, 0.f);
}
else
{
char text[64];
snprintf(text, sizeof(text), "0=%d 1=0.0", integers[j]);
if (pd.load_param(text))
return -1;
}
// round to float storage precision before comparing an x87 return value
volatile float value = pd.get(0, 7.f);
if (value != expected[j] || pd.type(0) != 2 || pd.get(0, 7) != integers[j]
|| pd.type(1) != 3 || pd.get(1, 7.f) != 0.f || pd.get(1, 7) != 7
|| pd.get(2, 7.f) != 7.f)
{
fprintf(stderr, "ParamDict numeric conversion failed value=%d mode=%d\n", integers[j], mode);
return -1;
}
}
return 0;
}
static int check_text_result(const char* text, bool valid)
{
TestParamDict memory;
if ((memory.load_param(text) == 0) != valid)
{
fprintf(stderr, "ParamDict memory parse result failed: %s\n", text);
return -1;
}
return 0;
}
static std::string make_param_string(size_t len, char c)
{
std::string s;
s.resize(len);
for (size_t i = 0; i < len; i++)
s[i] = c;
return s;
}
static int test_paramdict_text_boundaries()
{
TestParamDict pd;
const char* text = "0=-2147483648\t1=2147483647 2=\"\" 3=\" \" 4=1.0,2.0,-3.0 5=7, -23306=2,1.0,2.0 -23307=0 8=0e9999999999 9=1e-9999999999 10=.5 11=4294967296.0";
if (pd.load_param(text))
return -1;
if (pd.get(0, 0) != INT_MIN || pd.get(1, 0) != INT_MAX
|| pd.type(2) != 7 || pd.get(2, std::string("default")) != "" || pd.get(3, std::string()) != " "
|| pd.type(7) != 4 || !pd.get(7, ncnn::Mat()).empty()
|| pd.get(8, 1.f) != 0.f || pd.get(9, 1.f) != 0.f || pd.get(10, 0.f) != 0.5f
|| pd.get(11, 0.f) != 4294967296.f)
{
fprintf(stderr, "ParamDict text boundary values failed\n");
return -1;
}
ncnn::Mat a = pd.get(4, ncnn::Mat());
ncnn::Mat b = pd.get(5, ncnn::Mat());
ncnn::Mat c = pd.get(6, ncnn::Mat());
if (a.w != 3 || a[0] != 1.f || a[1] != 2.f || a[2] != -3.f
|| b.w != 1 || ((const int*)b)[0] != 7 || c.w != 2 || c[0] != 1.f || c[1] != 2.f)
{
fprintf(stderr, "ParamDict text array values failed\n");
return -1;
}
if (!pd.get(7, a).empty() || pd.type(31) != 0 || pd.get(31, a).data != a.data)
{
fprintf(stderr, "ParamDict empty text array default failed\n");
return -1;
}
const int lengths[] = {1, 14, 15, 16, 240, 241, 254, 255};
for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++)
{
const std::string value = make_param_string(lengths[i], 'x');
for (int quoted = 0; quoted < 2; quoted++)
{
std::string input = quoted ? "0=\"" : "0=";
input += value;
input += quoted ? "\" 1=19" : " 1=19";
if (pd.load_param(input.c_str())
|| pd.get(0, std::string()) != value || pd.get(1, 0) != 19)
{
fprintf(stderr, "ParamDict string boundary failed len=%d quoted=%d\n", lengths[i], quoted);
return -1;
}
}
}
// accept signs, leading zeroes and trailing commas without changing element types
if (pd.load_param("0=+7 1=0007 -23302=1,+2 3=1,2, 4=+1.5 5=1.0,2.0,"))
return -1;
a = pd.get(2, ncnn::Mat());
b = pd.get(3, ncnn::Mat());
c = pd.get(5, ncnn::Mat());
if (pd.type(0) != 2 || pd.get(0, 0) != 7 || pd.type(1) != 2 || pd.get(1, 0) != 7
|| pd.type(2) != 5 || a.w != 1 || ((const int*)a)[0] != 2
|| pd.type(3) != 5 || b.w != 2 || ((const int*)b)[0] != 1 || ((const int*)b)[1] != 2
|| pd.type(4) != 3 || pd.get(4, 0.f) != 1.5f
|| pd.type(5) != 6 || c.w != 2 || c[0] != 1.f || c[1] != 2.f)
{
fprintf(stderr, "ParamDict numeric spelling failed\n");
return -1;
}
// lines may end at a read-buffer boundary or split a quoted/numeric token across it
const int line_lengths[] = {1022, 1023, 1024, 1035, 1048, 2046, 2047};
for (size_t j = 0; j < sizeof(line_lengths) / sizeof(line_lengths[0]); j++)
for (int leading = 0; leading < 2; leading++)
{
const char* params = "0=\"hello world\" 1=1.25,2.5 2=42";
const std::string padding = make_param_string(line_lengths[j] - strlen(params), ' ');
std::string input = leading ? padding : std::string(params);
input += leading ? std::string(params) : padding;
input += "\r\n+Probe next 1 1 in out\n";
const unsigned char* ptr = (const unsigned char*)input.c_str();
ncnn::DataReaderFromMemory dr(ptr);
if (pd.load_param(dr) || pd.get(0, std::string()) != "hello world" || pd.get(2, 0) != 42)
{
fprintf(stderr, "ParamDict long line failed len=%d leading=%d\n", line_lengths[j], leading);
return -1;
}
const ncnn::Mat values = pd.get(1, ncnn::Mat());
char header[64];
if (values.w != 2 || values[0] != 1.25f || values[1] != 2.5f
|| dr.scan(" %63[^\r\n]", header) != 1 || strcmp(header, "+Probe next 1 1 in out"))
{
fprintf(stderr, "ParamDict long line array or next header failed\n");
return -1;
}
}
// exercise stack storage, array growth and the final copy for both element types
const int array_lengths[] = {1, 15, 16, 17, 31, 32, 33, 257, 1024};
for (size_t j = 0; j < sizeof(array_lengths) / sizeof(array_lengths[0]); j++)
for (int floating = 0; floating < 2; floating++)
{
std::string input = "0=";
for (int i = 0; i < array_lengths[j]; i++)
{
char value[32];
snprintf(value, sizeof(value), "%s%d%s", i ? "," : "", i, floating ? ".0" : "");
input += value;
}
if (array_lengths[j] == 1)
input += ",";
input += " 1=42";
if (pd.load_param(input.c_str()) || pd.get(1, 0) != 42)
{
fprintf(stderr, "ParamDict array growth parse failed len=%d floating=%d\n", array_lengths[j], floating);
return -1;
}
const ncnn::Mat values = pd.get(0, ncnn::Mat());
if (values.w != array_lengths[j])
{
fprintf(stderr, "ParamDict array growth length failed %d != %d\n", values.w, array_lengths[j]);
return -1;
}
for (int i = 0; i < values.w; i++)
if (floating ? values[i] != (float)i : ((const int*)values)[i] != i)
{
fprintf(stderr, "ParamDict array growth value failed len=%d index=%d floating=%d\n", values.w, i, floating);
return -1;
}
}
return 0;
}
static int check_float_boundary(const char* text, float expected, int count, bool valid)
{
TestParamDict pd;
const int ret = pd.load_param(text);
if ((ret == 0) != valid)
{
fprintf(stderr, "ParamDict float boundary parse failed: %s\n", text);
return -1;
}
if (!valid)
return 0;
bool matches;
if (count == 0)
{
matches = pd.type(0) == 3 && pd.get(0, 0.f) == expected;
}
else
{
const ncnn::Mat values = pd.get(0, ncnn::Mat());
matches = pd.type(0) == 6 && values.w == count && values[0] == expected;
if (matches && count == 2)
matches = values[1] == 1.f;
}
if (!matches)
{
fprintf(stderr, "ParamDict float boundary value failed: %s\n", text);
return -1;
}
return 0;
}
static int test_paramdict_float_boundaries()
{
// old and modern array elements retain the full numeric token limit
for (int len = 127; len <= 128; len++)
{
std::string text = "-23300=1,0.";
text += make_param_string(len - 2, '0');
text += " 1=42\nReLU next 1 1 in out\n";
if (check_float_boundary(text.c_str(), 0.f, 1, len == 127))
return -1;
text = "0=0.0,1.";
text += make_param_string(len - 2, '0');
text += " 1=42\nReLU next 1 1 in out\n";
if (check_float_boundary(text.c_str(), 0.f, 2, len == 127))
return -1;
}
const struct
{
const char* text;
float expected;
} cases[] = {
// exact float midpoints and their decimal neighbors in the old token-length range
{"32768.017578124", 32768.015625f},
{"32768.017578125", 32768.015625f},
{"32768.017578126", 32768.01953125f},
{"32768.021484374", 32768.01953125f},
{"32768.021484375", 32768.0234375f},
{"32768.021484376", 32768.0234375f},
{"3.40282347e+38", FLT_MAX},
{"3.4028235e38", FLT_MAX},
{"3.40282356e38", FLT_MAX},
{"3.402823e38", 3.402823e38f},
{"3.402823567797336e38", FLT_MAX},
{"340282356779733661637539395458142568447.0", FLT_MAX},
{"0.000340282356779733661637539395458142568447e42", FLT_MAX},
{"1e-46", 0.f},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++)
for (int negative = 0; negative < 2; negative++)
{
char text[128];
const char* sign = negative ? "-" : "";
const float value = negative ? -cases[i].expected : cases[i].expected;
snprintf(text, sizeof(text), "0=%s%s", sign, cases[i].text);
if (check_float_boundary(text, value, 0, true))
return -1;
snprintf(text, sizeof(text), "0=%s%s,1.0", sign, cases[i].text);
if (check_float_boundary(text, value, 2, true))
return -1;
snprintf(text, sizeof(text), "-23300=1,%s%s", sign, cases[i].text);
if (check_float_boundary(text, value, 1, true))
return -1;
}
// nine significant digits must round trip the largest finite float
const float values[] = {FLT_MAX, -FLT_MAX};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++)
{
char text[64];
snprintf(text, sizeof(text), "0=%.9g", values[i]);
if (check_float_boundary(text, values[i], 0, true))
return -1;
}
return 0;
}
// short-read coverage requires a reader that knows the input size
class BoundedParamReader : public ncnn::DataReader
{
public:
BoundedParamReader(const unsigned char* data, size_t size)
: ptr(data), remaining(size)
{
}
virtual size_t read(void* buf, size_t size) const
{
const size_t n = std::min(size, remaining);
if (n != 0)
{
memcpy(buf, ptr, n);
ptr += n;
remaining -= n;
}
return n;
}
private:
mutable const unsigned char* ptr;
mutable size_t remaining;
};
static void append_param_word(std::vector<unsigned char>& bytes, int value)
{
const unsigned int v = (unsigned int)value;
for (int i = 0; i < 4; i++)
bytes.push_back((v >> (i * 8)) & 255);
}
static int check_reloaded_params(const TestParamDict& pd, int present_id, int value)
{
for (int id = 0; id < NCNN_MAX_PARAM_COUNT; id++)
{
if (id == present_id)
{
if (pd.get(id, 0) != value)
{
fprintf(stderr, "ParamDict reloaded value failed id=%d\n", id);
return -1;
}
}
else if (pd.type(id) != 0)
{
fprintf(stderr, "ParamDict stale parameter after reload id=%d type=%d\n", id, pd.type(id));
return -1;
}
}
return 0;
}
static int check_paramdict_reload(const ncnn::DataReader& dr, bool binary)
{
TestParamDict pd;
int ret = binary ? pd.load_param_bin(dr) : pd.load_param(dr);
const ncnn::Mat values = pd.get(2, ncnn::Mat());
if (ret != 0 || pd.get(0, 0) != 42 || pd.get(1, 0.f) != 1.5f
|| values.w != 2 || ((const int*)values)[0] != 3 || ((const int*)values)[1] != 4
|| pd.get(3, std::string()) != "hello" || pd.get(31, 0) != 31)
{
fprintf(stderr, "ParamDict initial reload state failed\n");
return -1;
}
// read each following header without consuming its parameters on the same line
char header[64] = {0};
if (!binary && (dr.scan(" %20c", header) != 1 || strcmp(header, "ReLU next 1 1 in out")))
{
fprintf(stderr, "ParamDict next layer header failed\n");
return -1;
}
ret = binary ? pd.load_param_bin(dr) : pd.load_param(dr);
if (ret != 0 || check_reloaded_params(pd, 4, 7))
{
fprintf(stderr, "ParamDict second load failed\n");
return -1;
}
if (!binary && (dr.scan(" %23c", header) != 1 || strcmp(header, "Clip last 1 1 out final")))
{
fprintf(stderr, "ParamDict last layer header failed\n");
return -1;
}
ret = binary ? pd.load_param_bin(dr) : pd.load_param(dr);
if (ret != 0 || check_reloaded_params(pd, -1, 0))
{
fprintf(stderr, "ParamDict empty load failed\n");
return -1;
}
if (((const int*)values)[0] != 3 || ((const int*)values)[1] != 4)
{
fprintf(stderr, "ParamDict array reference after reload failed\n");
return -1;
}
return 0;
}
static int test_paramdict_reload_after_failure()
{
for (int binary = 0; binary < 2; binary++)
{
TestParamDict pd;
pd.set(9, 99);
// the first scalar is valid, but the second value is invalid or missing
std::vector<unsigned char> data;
append_param_word(data, 0);
append_param_word(data, 1);
append_param_word(data, 1);
BoundedParamReader truncated(&data[0], data.size());
const int ret = binary ? pd.load_param_bin(truncated) : pd.load_param("0=1 1=1e+");
if (ret == 0 || pd.get(0, 0) != 1 || pd.type(1) != 0 || pd.type(9) != 0)
{
fprintf(stderr, "ParamDict failed load state failed binary=%d\n", binary);
return -1;
}
// reloading must discard the partial result of the failed load
data.clear();
append_param_word(data, 4);
append_param_word(data, 7);
append_param_word(data, -233);
BoundedParamReader valid(&data[0], data.size());
const int reload_ret = binary ? pd.load_param_bin(valid) : pd.load_param("4=7");
if (reload_ret != 0 || check_reloaded_params(pd, 4, 7))
{
fprintf(stderr, "ParamDict reload after failure failed binary=%d\n", binary);
return -1;
}
}
return 0;
}
static int test_paramdict_reload()
{
// the next layer name is independent of parameter syntax, even with no parameters
const char* headers[] = {".Custom next 1 1 in out", "/Custom next 1 1 in out", ",Custom next 1 1 in out", "+Probe next 1 1 in out", "-Probe next 1 1 in out", "123Probe next 1 1 in out"};
for (size_t i = 0; i < sizeof(headers) / sizeof(headers[0]); i++)
for (int empty = 0; empty < 2; empty++)
{
char input[128];
snprintf(input, sizeof(input), "%s\n%s\n", empty ? "" : "+0=42 -23301=2,3,4", headers[i]);
const unsigned char* ptr = (const unsigned char*)input;
ncnn::DataReaderFromMemory dr(ptr);
TestParamDict pd;
if (pd.load_param(dr))
return -1;
if (!empty)
{
const ncnn::Mat values = pd.get(1, ncnn::Mat());
if (pd.get(0, 0) != 42 || values.w != 2 || ((const int*)values)[0] != 3 || ((const int*)values)[1] != 4)
{
fprintf(stderr, "ParamDict signed parameter id failed\n");
return -1;
}
}
char header[64];
if (dr.scan(" %63[^\r\n]", header) != 1 || strcmp(header, headers[i]))
{
fprintf(stderr, "ParamDict custom layer header changed: %s\n", headers[i]);
return -1;
}
}
const char* text = "0=42 1=1.5 2=3,4 3=hello 31=31\r\nReLU next 1 1 in out 4=7\r\nClip last 1 1 out final\r\n";
const char* old_array_text = "0=42 1=1.5 -23302=2,3,4 3=hello 31=31\r\nReLU next 1 1 in out 4=7\r\nClip last 1 1 out final\r\n";
// three consecutive parameter blocks, with no parameters in the last block
const int words[] = {0, 42, 1, 0x3fc00000, -23302, 2, 3, 4, -23403, 5, 0x6c6c6568, 0x6f, 31, 31, -233, 4, 7, -233, -233};
std::vector<unsigned char> data;
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++)
append_param_word(data, words[i]);
for (int mode = 0; mode < 3; mode++)
{
const bool binary = mode == 2;
const char* input = mode == 0 ? text : old_array_text;
const unsigned char* ptr = binary ? &data[0] : (const unsigned char*)input;
ncnn::DataReaderFromMemory dr(ptr);
const int ret = check_paramdict_reload(dr, binary);
if (ret != 0)
{
fprintf(stderr, "ParamDict reload failed mode=%d\n", mode);
return -1;
}
}
return 0;
}
static int test_paramdict_binary_bounds()
{
// untyped binary scalars/arrays remain readable as both int and float
std::vector<unsigned char> data;
append_param_word(data, 0);
append_param_word(data, 0x3f800000);
append_param_word(data, -23301);
append_param_word(data, 1);
append_param_word(data, 0x3f800000);
append_param_word(data, -23302);
append_param_word(data, 0);
append_param_word(data, -23403);
append_param_word(data, 0);
append_param_word(data, -23404);
append_param_word(data, 1);
append_param_word(data, 0x64636261); // only 'a' belongs to the string; padding is nonzero
append_param_word(data, -23405);
append_param_word(data, 3);
append_param_word(data, 0x7f620061); // an embedded NUL terminates the text string
append_param_word(data, -23406);
append_param_word(data, 255);
for (int i = 0; i < 256; i++)
data.push_back('q');
append_param_word(data, -23407);
append_param_word(data, 4);
append_param_word(data, 0x63626100); // a leading NUL produces an empty string
append_param_word(data, -23408);
append_param_word(data, 4);
append_param_word(data, 0x322c0031); // the expression suffix after NUL must not become visible
append_param_word(data, 9);
append_param_word(data, 99);
append_param_word(data, -233);
TestParamDict pd;
BoundedParamReader reader(&data[0], data.size());
if (pd.load_param_bin(reader) || pd.get(0, 0) != 0x3f800000 || pd.get(0, 0.f) != 1.f
|| pd.get(1, ncnn::Mat()).w != 1 || pd.get(1, ncnn::Mat())[0] != 1.f
|| pd.type(2) != 4 || !pd.get(2, ncnn::Mat()).empty()
|| pd.type(3) != 7 || pd.get(3, std::string("default")) != ""
|| pd.get(4, std::string()) != "a" || pd.get(5, std::string()) != "a"
|| pd.get(6, std::string()) != make_param_string(255, 'q')
|| pd.type(7) != 7 || pd.get(7, std::string("default")) != ""
|| pd.get(8, std::string()) != "1" || pd.get(9, 0) != 99)
{
fprintf(stderr, "ParamDict binary boundary values failed\n");
return -1;
}
const ncnn::Mat fallback = pd.get(1, ncnn::Mat());
if (!pd.get(2, fallback).empty() || pd.type(31) != 0 || pd.get(31, fallback).data != fallback.data)
{
fprintf(stderr, "ParamDict empty binary array default failed\n");
return -1;
}
// every truncation, including missing EOP and short scalar/array/string data, fails
for (size_t size = 0; size < data.size(); size++)
{
BoundedParamReader truncated(&data[0], size);
TestParamDict partial;
if (partial.load_param_bin(truncated) == 0)
{
fprintf(stderr, "ParamDict truncated binary accepted at %zu\n", size);
return -1;
}
}
return 0;
}
#if NCNN_VALIDATION
static int test_paramdict_invalid_text()
{
const char* malformed[] = {
"-23304=-4", "-23304=-1", "-23304=-2147483648", "-23304=2147483648",
"-2147483648=0", "2147483648=0", "999999999999999999999=0", "32=0", "-1=0", "-23332=0",
"0", "0 1", "0=", "+=1", "0=2147483648", "0=-2147483649", "0=1junk", "0=1.0junk",
"0=.", "0=-", "0=--1", "0=1e", "0=1e+", "0=1.2.3", "0=1e4294967295", "0=1e39",
"0=\"", "0=\"unterminated", "0=\"abc\n", "0=\"abc\"suffix", "0=\"abc\"1=2",
"-23300=1", "-23300=2,1", "-23300=1,1,2", "-23300=0,1", "-23300=1x,2",
"-23300=1 1", "-23300=1,,1", "-23300=1, 1",
"0=1,,2", "0=1,2x", "0=1.0,2e", "-23300=1,1e+"
};
for (size_t i = 0; i < sizeof(malformed) / sizeof(malformed[0]); i++)
if (check_text_result(malformed[i], false))
return -1;
std::string long_number = "0=0.";
long_number += make_param_string(128, '0');
long_number += "1";
if (check_text_result(long_number.c_str(), false))
return -1;
std::string long_string = "0=";
long_string += make_param_string(256, 'a');
std::string long_quoted = "0=\"";
long_quoted += make_param_string(256, 'a');
long_quoted += "\"";
if (check_text_result(long_string.c_str(), false) || check_text_result(long_quoted.c_str(), false))
return -1;
return 0;
}
static int test_paramdict_invalid_access()
{
ncnn::ParamDict pd;
ncnn::Mat array(1);
((int*)array)[0] = 17;
const std::string text = "text";
pd.set(0, 10);
pd.set(1, 2.5f);
pd.set(2, array);
pd.set(3, text);
pd.set(31, 31);
const int invalid_ids[] = {INT_MIN, -1, NCNN_MAX_PARAM_COUNT, INT_MAX};
for (size_t j = 0; j < sizeof(invalid_ids) / sizeof(invalid_ids[0]); j++)
{
const int id = invalid_ids[j];
pd.set(id, 1);
pd.set(id, 1.f);
pd.set(id, array);
pd.set(id, text);
if (pd.type(id) != 0 || pd.get(id, 7) != 7 || pd.get(id, 7.f) != 7.f
|| pd.get(id, array).data != array.data || pd.get(id, text) != text)
{
fprintf(stderr, "ParamDict invalid id access failed %d\n", id);
return -1;
}
}
if (pd.get(0, 0) != 10 || pd.get(0, 7.f) != 10.f || pd.get(31, 0) != 31)
{
fprintf(stderr, "ParamDict valid id access failed\n");
return -1;
}
return 0;
}
static int test_paramdict_invalid_float()
{
const char* cases[] = {
"3.40282357e38",
"1e39",
"3.402823567797337e38",
"340282356779733661637539395458142568448.0",
"340282356779733661637539395458142568449.0",
"000340282356779733661637539395458142568448e0",
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++)
for (int negative = 0; negative < 2; negative++)
{
char text[128];
const char* sign = negative ? "-" : "";
snprintf(text, sizeof(text), "0=%s%s", sign, cases[i]);
if (check_float_boundary(text, 0.f, 0, false))
return -1;
snprintf(text, sizeof(text), "0=%s%s,1.0", sign, cases[i]);
if (check_float_boundary(text, 0.f, 2, false))
return -1;
snprintf(text, sizeof(text), "-23300=1,%s%s", sign, cases[i]);
if (check_float_boundary(text, 0.f, 1, false))
return -1;
}
return 0;
}
static int test_paramdict_invalid_binary()
{
const int ids[] = {-23304, -23404};
const int lengths[] = {-1, -4, -8, INT_MIN};
for (size_t i = 0; i < sizeof(ids) / sizeof(ids[0]); i++)
for (size_t j = 0; j < sizeof(lengths) / sizeof(lengths[0]); j++)
{
std::vector<unsigned char> data;
append_param_word(data, ids[i]);
append_param_word(data, lengths[j]);
append_param_word(data, -233);
BoundedParamReader reader(&data[0], data.size());
TestParamDict pd;
if (pd.load_param_bin(reader) == 0)
{
fprintf(stderr, "ParamDict negative binary length accepted id=%d len=%d\n", ids[i], lengths[j]);
return -1;
}
}
const int invalid_ids[] = {INT_MIN, -1, 32, INT_MAX, -23332, -23432};
for (size_t i = 0; i < sizeof(invalid_ids) / sizeof(invalid_ids[0]); i++)
{
std::vector<unsigned char> data;
append_param_word(data, invalid_ids[i]);
append_param_word(data, -233);
BoundedParamReader reader(&data[0], data.size());
TestParamDict pd;
if (pd.load_param_bin(reader) == 0)
{
fprintf(stderr, "ParamDict invalid binary id accepted %d\n", invalid_ids[i]);
return -1;
}
}
const int invalid_string_lengths[] = {256, INT_MAX};
for (size_t i = 0; i < sizeof(invalid_string_lengths) / sizeof(invalid_string_lengths[0]); i++)
{
std::vector<unsigned char> data;
append_param_word(data, -23400);
append_param_word(data, invalid_string_lengths[i]);
BoundedParamReader reader(&data[0], data.size());
TestParamDict pd;
if (pd.load_param_bin(reader) == 0)
{
fprintf(stderr, "ParamDict oversized binary string accepted len=%d\n", invalid_string_lengths[i]);
return -1;
}
}
if (sizeof(size_t) == 4)
{
std::vector<unsigned char> data;
append_param_word(data, -23300);
append_param_word(data, 0x40000000); // byte count wraps on 32-bit targets
BoundedParamReader reader(&data[0], data.size());
TestParamDict pd;
if (pd.load_param_bin(reader) == 0 || check_text_result("-23300=1073741824", false))
{
fprintf(stderr, "ParamDict overflowing array byte count accepted\n");
return -1;
}
}
return 0;
}
#endif // NCNN_VALIDATION
int main()
{
return 0
|| test_paramdict_0()
|| test_paramdict_1()
|| test_paramdict_2()
|| test_paramdict_3()
|| test_paramdict_4()
|| test_paramdict_5()
|| test_paramdict_6()
|| test_paramdict_access()
|| test_paramdict_numeric_conversion()
|| test_paramdict_text_boundaries()
|| test_paramdict_float_boundaries()
|| test_paramdict_reload()
|| test_paramdict_reload_after_failure()
|| test_paramdict_binary_bounds()
#if NCNN_VALIDATION
|| test_paramdict_invalid_access()
|| test_paramdict_invalid_text()
|| test_paramdict_invalid_float()
|| test_paramdict_invalid_binary()
#endif // NCNN_VALIDATION
;
}
Reference in New Issue View Git Blame Copy Permalink