GithubStar/neuron
1
0
Fork 0
mirror of https://github.com/emqx/neuron.git synced 2026-10-27 12:28:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add idhash & mem_alloc to utils

Browse Source
This commit is contained in:
alvin1221
2021-07-28 16:29:22 +08:00
committed by Parker Liu
parent 481db8d1fc
commit 02c36bccf4
6 changed files with 453 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+29 -33
View File
@@ -1,61 +1,57 @@
# Neuron project cmake file
cmake_minimum_required (VERSION 3.12)
project (neuron)
cmake_minimum_required(VERSION 3.12)
project(neuron)
ENABLE_LANGUAGE(C)
enable_language(C)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
add_subdirectory(extern/vector)
add_subdirectory(extern/libcsptr)
configure_file(extern/vector/vector.h ${CMAKE_SOURCE_DIR}/include/vector.h COPYONLY)
configure_file(extern/vector/vector.h ${CMAKE_SOURCE_DIR}/include/vector.h
COPYONLY)
# Call this from your own project's makefile.
find_package(nng CONFIG REQUIRED)
find_package(Threads)
# define neuron executalbe program
set(NEURON_SOURCES src/main.c
src/core/message.c
src/core/databuf.c
src/core/neu_manager.c
src/core/plugin_manager.c
src/adapter/adapter.c)
set(NEURON_SOURCES
src/main.c src/core/message.c src/core/databuf.c src/core/neu_manager.c
src/core/plugin_manager.c src/adapter/adapter.c)
set(NEURON_INCLUDE_DIRECTORIES include src deps)
add_executable(neuron)
target_sources(neuron
PRIVATE ${NEURON_SOURCES})
target_include_directories(neuron
PRIVATE ${NEURON_INCLUDE_DIRECTORIES})
if (APPLE)
target_link_libraries(neuron nng::nng neuron-base)
target_sources(neuron PRIVATE ${NEURON_SOURCES})
target_include_directories(neuron PRIVATE ${NEURON_INCLUDE_DIRECTORIES})
if(APPLE)
target_link_libraries(neuron nng::nng neuron-base)
else()
target_link_libraries(neuron dl nng neuron-base)
target_link_libraries(neuron dl nng neuron-base)
endif()
target_link_libraries(neuron ${CMAKE_THREAD_LIBS_INIT} )
target_link_libraries(neuron ${CMAKE_THREAD_LIBS_INIT})
# define neuron base library
set(NEURON_BASE_SOURCES deps/cJSON.c
src/types.c
src/utils/atomic_data.c
src/utils/panic.c
src/utils/log.c)
set(NEURON_BASE_SOURCES
deps/cJSON.c
src/types.c
src/utils/atomic_data.c
src/utils/mem_alloc.c
src/utils/idhash.c
src/utils/panic.c
src/utils/log.c)
set(NEURON_BASE_INCLUDE_DIRECTORIES include deps)
add_library(neuron-base SHARED)
target_sources(neuron-base
PRIVATE ${NEURON_BASE_SOURCES})
target_sources(neuron-base PRIVATE ${NEURON_BASE_SOURCES})
target_include_directories(neuron-base
PRIVATE ${NEURON_BASE_INCLUDE_DIRECTORIES})
target_link_libraries(neuron-base vector-static ${CMAKE_THREAD_LIBS_INIT} )
PRIVATE ${NEURON_BASE_INCLUDE_DIRECTORIES})
target_link_libraries(neuron-base vector-static ${CMAKE_THREAD_LIBS_INIT})
# define for plugins
set(SAMPLE_PLUGIN_SOURCES plugins/plugin_sample.c)
add_library(sample-plugin SHARED)
target_sources(sample-plugin
PRIVATE ${SAMPLE_PLUGIN_SOURCES})
target_include_directories(sample-plugin
PRIVATE include)
target_sources(sample-plugin PRIVATE ${SAMPLE_PLUGIN_SOURCES})
target_include_directories(sample-plugin PRIVATE include)
target_link_libraries(sample-plugin neuron-base)
target_link_libraries(sample-plugin ${CMAKE_THREAD_LIBS_INIT} )
target_link_libraries(sample-plugin ${CMAKE_THREAD_LIBS_INIT})
include/neu_panic.h
+8
View File
@@ -26,4 +26,12 @@
void neu_show_backtrace(void);
void neu_panic(const char *fmt, ...);
#ifndef NDEBUG
#define NEU_ASSERT(x) \
if (!(x)) \
neu_panic("%s: %d: assert err: %s", __FILE__, __LINE__, #x)
#else
#define NEU_ASSERT(x) (0)
#endif
#endif
src/utils/idhash.c
+282
View File
@@ -0,0 +1,282 @@
/**
* NEURON IIoT System for Industry 4.0
* Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**/
#include "idhash.h"
#include "mem_alloc.h"
struct neu_id_entry {
uint32_t key;
uint32_t skips;
void * val;
};
void neu_id_map_init(neu_id_map *m, uint32_t lo, uint32_t hi, bool randomize)
{
if (lo == 0) {
lo = 1;
}
if (hi == 0) {
hi = 0xffffffffu;
}
NEU_ASSERT(lo != 0);
NEU_ASSERT(hi > lo);
m->id_entries = NULL;
m->id_count = 0;
m->id_load = 0;
m->id_cap = 0;
m->id_max_load = 0;
m->id_min_load = 0; // never shrink below this
m->id_min_val = lo;
m->id_max_val = hi;
m->id_dyn_val = lo;
}
void neu_id_map_fini(neu_id_map *m)
{
if (m->id_entries != NULL) {
NEU_FREE_STRUCTS(m->id_entries, m->id_cap);
m->id_entries = NULL;
m->id_cap = m->id_count = 0;
m->id_load = m->id_min_load = m->id_max_load = 0;
}
}
// Inspired by Python dict implementation. This probe will visit every
// cell. We always hash consecutively assigned IDs. This requires that
// the capacity is always a power of two.
#define ID_NEXT(m, j) ((((j) *5) + 1) & (m->id_cap - 1))
#define ID_INDEX(m, j) ((j) & (m->id_cap - 1))
static size_t id_find(neu_id_map *m, uint32_t id)
{
size_t index;
size_t start;
if (m->id_count == 0) {
return ((size_t) -1);
}
index = ID_INDEX(m, id);
start = index;
for (;;) {
// The value of ihe_key is only valid if ihe_val is not NULL.
if ((m->id_entries[index].key == id) &&
(m->id_entries[index].val != NULL)) {
return (index);
}
if (m->id_entries[index].skips == 0) {
return ((size_t) -1);
}
index = ID_NEXT(m, index);
if (index == start) {
break;
}
}
return ((size_t) -1);
}
void *neu_id_get(neu_id_map *m, uint32_t id)
{
size_t index;
if ((index = id_find(m, id)) == (size_t) -1) {
return (NULL);
}
return (m->id_entries[index].val);
}
static int id_resize(neu_id_map *m)
{
size_t new_cap;
size_t old_cap;
neu_id_entry *new_entries;
neu_id_entry *old_entries;
uint32_t i;
if ((m->id_load < m->id_max_load) && (m->id_load >= m->id_min_load)) {
// No resize needed.
return (0);
}
old_cap = m->id_cap;
new_cap = 8;
while (new_cap < (m->id_count * 2)) {
new_cap *= 2;
}
if (new_cap == old_cap) {
// Same size.
return (0);
}
old_entries = m->id_entries;
new_entries = NEU_ALLOC_STRUCTS(new_entries, new_cap);
if (new_entries == NULL) {
return 2;
}
m->id_entries = new_entries;
m->id_cap = new_cap;
m->id_load = 0;
if (new_cap > 8) {
m->id_min_load = new_cap / 8;
m->id_max_load = new_cap * 2 / 3;
} else {
m->id_min_load = 0;
m->id_max_load = 5;
}
for (i = 0; i < old_cap; i++) {
size_t index;
if (old_entries[i].val == NULL) {
continue;
}
index = old_entries[i].key & (new_cap - 1);
for (;;) {
// Increment the load unconditionally. It counts
// once for every item stored, plus once for each
// hashing operation we use to store the item (i.e.
// one for the item, plus once for each rehash.)
m->id_load++;
if (new_entries[index].val == NULL) {
// As we are hitting this entry for the first
// time, it won't have any skips.
NEU_ASSERT(new_entries[index].skips == 0);
new_entries[index].val = old_entries[i].val;
new_entries[index].key = old_entries[i].key;
break;
}
new_entries[index].skips++;
index = ID_NEXT(m, index);
}
}
if (old_cap != 0) {
NEU_FREE_STRUCTS(old_entries, old_cap);
}
return (0);
}
int neu_id_remove(neu_id_map *m, uint32_t id)
{
size_t index;
size_t probe;
if ((index = id_find(m, id)) == (size_t) -1) {
return 2;
}
// Now we have found the index where the object exists. We are going
// to restart the search, until the index matches, to decrement the
// skips counter.
probe = ID_INDEX(m, id);
for (;;) {
neu_id_entry *entry;
// The load was increased once each hashing operation we used
// to place the the item. Decrement it accordingly.
m->id_load--;
entry = &m->id_entries[probe];
if (probe == index) {
entry->val = NULL;
entry->key = 0; // invalid key
break;
}
NEU_ASSERT(entry->skips > 0);
entry->skips--;
probe = ID_NEXT(m, probe);
}
m->id_count--;
// Shrink -- but it's ok if we can't.
(void) id_resize(m);
return (0);
}
int neu_id_set(neu_id_map *m, uint32_t id, void *val)
{
size_t index;
neu_id_entry *ent;
// Try to resize -- if we don't need to, this will be a no-op.
if (id_resize(m) != 0) {
return 2;
}
// If it already exists, just overwrite the old value.
if ((index = id_find(m, id)) != (size_t) -1) {
ent = &m->id_entries[index];
ent->val = val;
return (0);
}
index = ID_INDEX(m, id);
for (;;) {
ent = &m->id_entries[index];
// Increment the load count. We do this each time time we
// rehash. This may over-count items that collide on the
// same rehashing, but this should just cause a table to
// grow sooner, which is probably a good thing.
m->id_load++;
if (ent->val == NULL) {
m->id_count++;
ent->key = id;
ent->val = val;
return (0);
}
// Record the skip count. This being non-zero informs
// that a rehash will be necessary. Without this we
// would need to scan the entire hash for the match.
ent->skips++;
index = ID_NEXT(m, index);
}
}
int neu_id_alloc(neu_id_map *m, uint32_t *idp, void *val)
{
uint32_t id;
int rv;
NEU_ASSERT(val != NULL);
// range is inclusive, so > to get +1 effect.
if (m->id_count > (m->id_max_val - m->id_min_val)) {
// Really more like ENOSPC.. the table is filled to max.
return 2;
}
for (;;) {
id = m->id_dyn_val;
m->id_dyn_val++;
if (m->id_dyn_val > m->id_max_val) {
m->id_dyn_val = m->id_min_val;
}
if (id_find(m, id) == (size_t) -1) {
break;
}
}
rv = neu_id_set(m, id, val);
if (rv == 0) {
*idp = id;
}
return (rv);
}
src/utils/idhash.h
+61
View File
@@ -0,0 +1,61 @@
/**
* NEURON IIoT System for Industry 4.0
* Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**/
#ifndef __NEU_IDHASH_H__
#define __NEU_IDHASH_H__
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
// We find that we often want to have a list of things listed by a
// numeric ID, which is generally monotonically increasing. This is
// most often a pipe ID. To help keep collections of these things
// indexed by their ID (which might start from a very large value),
// we offer a hash table. The hash table uses open addressing, but
// we use a better probe (taken from Python) to avoid hitting the same
// positions. Our hash algorithm is just the low order bits, and we
// use table sizes that are powers of two. Note that hash items
// must be non-NULL. The table is protected by an internal lock.
typedef struct neu_id_map neu_id_map;
typedef struct neu_id_entry neu_id_entry;
// NB: These details are entirely private to the hash implementation.
// They are provided here to facilitate inlining in structures.
struct neu_id_map {
size_t id_cap;
size_t id_count;
size_t id_load;
size_t id_min_load; // considers placeholders
size_t id_max_load;
uint32_t id_min_val;
uint32_t id_max_val;
uint32_t id_dyn_val;
neu_id_entry *id_entries;
};
extern void neu_id_map_init(neu_id_map *, uint32_t, uint32_t, bool);
extern void neu_id_map_fini(neu_id_map *);
extern void *neu_id_get(neu_id_map *, uint32_t);
extern int neu_id_set(neu_id_map *, uint32_t, void *);
extern int neu_id_alloc(neu_id_map *, uint32_t *, void *);
extern int neu_id_remove(neu_id_map *, uint32_t);
#endif // __NEU_IDHASH_H__
src/utils/mem_alloc.c
+36
View File
@@ -0,0 +1,36 @@
/**
* NEURON IIoT System for Industry 4.0
* Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**/
#include "mem_alloc.h"
void *neu_alloc(size_t sz)
{
return (sz > 0 ? malloc(sz) : NULL);
}
void *neu_zalloc(size_t sz)
{
return (sz > 0 ? calloc(1, sz) : NULL);
}
void neu_free(void *b, size_t z)
{
NEU_ARG_UNUSED(z);
free(b);
}
src/utils/mem_alloc.h
+37
View File
@@ -0,0 +1,37 @@
/**
* NEURON IIoT System for Industry 4.0
* Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**/
#ifndef __MEM_ALLOC_H__
#define __MEM_ALLOC_H__
#include "neu_panic.h"
#include <stdlib.h>
#define NEU_ARG_UNUSED(x) ((void) x)
void *neu_alloc(size_t sz);
void *neu_zalloc(size_t sz);
void neu_free(void *b, size_t z);
#define NNI_ALLOC_STRUCT(s) neu_zalloc(sizeof(*s))
#define NNI_FREE_STRUCT(s) neu_free((s), sizeof(*s))
#define NEU_ALLOC_STRUCTS(s, n) neu_zalloc(sizeof(*s) * n)
#define NEU_FREE_STRUCTS(s, n) neu_free(s, sizeof(*s) * n)
#endif