From 02c36bccf4666fe8a794445adc5340ef46a000fe Mon Sep 17 00:00:00 2001 From: alvin1221 Date: Wed, 28 Jul 2021 11:52:52 +0800 Subject: [PATCH] Add idhash & mem_alloc to utils --- CMakeLists.txt | 62 +++++----- include/neu_panic.h | 8 ++ src/utils/idhash.c | 282 ++++++++++++++++++++++++++++++++++++++++++ src/utils/idhash.h | 61 +++++++++ src/utils/mem_alloc.c | 36 ++++++ src/utils/mem_alloc.h | 37 ++++++ 6 files changed, 453 insertions(+), 33 deletions(-) create mode 100644 src/utils/idhash.c create mode 100644 src/utils/idhash.h create mode 100644 src/utils/mem_alloc.c create mode 100644 src/utils/mem_alloc.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc1f5c7..aa387a93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,61 +1,57 @@ # Neuron project cmake file -cmake_minimum_required (VERSION 3.12) -project (neuron) +cmake_minimum_required(VERSION 3.12) +project(neuron) -ENABLE_LANGUAGE(C) +enable_language(C) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) add_subdirectory(extern/vector) add_subdirectory(extern/libcsptr) -configure_file(extern/vector/vector.h ${CMAKE_SOURCE_DIR}/include/vector.h COPYONLY) +configure_file(extern/vector/vector.h ${CMAKE_SOURCE_DIR}/include/vector.h + COPYONLY) # Call this from your own project's makefile. find_package(nng CONFIG REQUIRED) find_package(Threads) # define neuron executalbe program -set(NEURON_SOURCES src/main.c - src/core/message.c - src/core/databuf.c - src/core/neu_manager.c - src/core/plugin_manager.c - src/adapter/adapter.c) +set(NEURON_SOURCES + src/main.c src/core/message.c src/core/databuf.c src/core/neu_manager.c + src/core/plugin_manager.c src/adapter/adapter.c) set(NEURON_INCLUDE_DIRECTORIES include src deps) add_executable(neuron) -target_sources(neuron - PRIVATE ${NEURON_SOURCES}) -target_include_directories(neuron - PRIVATE ${NEURON_INCLUDE_DIRECTORIES}) -if (APPLE) - target_link_libraries(neuron nng::nng neuron-base) +target_sources(neuron PRIVATE ${NEURON_SOURCES}) +target_include_directories(neuron PRIVATE ${NEURON_INCLUDE_DIRECTORIES}) +if(APPLE) + target_link_libraries(neuron nng::nng neuron-base) else() - target_link_libraries(neuron dl nng neuron-base) + target_link_libraries(neuron dl nng neuron-base) endif() -target_link_libraries(neuron ${CMAKE_THREAD_LIBS_INIT} ) +target_link_libraries(neuron ${CMAKE_THREAD_LIBS_INIT}) # define neuron base library -set(NEURON_BASE_SOURCES deps/cJSON.c - src/types.c - src/utils/atomic_data.c - src/utils/panic.c - src/utils/log.c) +set(NEURON_BASE_SOURCES + deps/cJSON.c + src/types.c + src/utils/atomic_data.c + src/utils/mem_alloc.c + src/utils/idhash.c + src/utils/panic.c + src/utils/log.c) + set(NEURON_BASE_INCLUDE_DIRECTORIES include deps) add_library(neuron-base SHARED) -target_sources(neuron-base - PRIVATE ${NEURON_BASE_SOURCES}) +target_sources(neuron-base PRIVATE ${NEURON_BASE_SOURCES}) target_include_directories(neuron-base - PRIVATE ${NEURON_BASE_INCLUDE_DIRECTORIES}) -target_link_libraries(neuron-base vector-static ${CMAKE_THREAD_LIBS_INIT} ) + PRIVATE ${NEURON_BASE_INCLUDE_DIRECTORIES}) +target_link_libraries(neuron-base vector-static ${CMAKE_THREAD_LIBS_INIT}) # define for plugins set(SAMPLE_PLUGIN_SOURCES plugins/plugin_sample.c) add_library(sample-plugin SHARED) -target_sources(sample-plugin - PRIVATE ${SAMPLE_PLUGIN_SOURCES}) -target_include_directories(sample-plugin - PRIVATE include) +target_sources(sample-plugin PRIVATE ${SAMPLE_PLUGIN_SOURCES}) +target_include_directories(sample-plugin PRIVATE include) target_link_libraries(sample-plugin neuron-base) -target_link_libraries(sample-plugin ${CMAKE_THREAD_LIBS_INIT} ) - +target_link_libraries(sample-plugin ${CMAKE_THREAD_LIBS_INIT}) diff --git a/include/neu_panic.h b/include/neu_panic.h index 22b3b777..5d44743b 100644 --- a/include/neu_panic.h +++ b/include/neu_panic.h @@ -26,4 +26,12 @@ void neu_show_backtrace(void); void neu_panic(const char *fmt, ...); +#ifndef NDEBUG +#define NEU_ASSERT(x) \ + if (!(x)) \ + neu_panic("%s: %d: assert err: %s", __FILE__, __LINE__, #x) +#else +#define NEU_ASSERT(x) (0) +#endif + #endif diff --git a/src/utils/idhash.c b/src/utils/idhash.c new file mode 100644 index 00000000..19e887cf --- /dev/null +++ b/src/utils/idhash.c @@ -0,0 +1,282 @@ +/** + * NEURON IIoT System for Industry 4.0 + * Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + **/ + +#include "idhash.h" +#include "mem_alloc.h" + +struct neu_id_entry { + uint32_t key; + uint32_t skips; + void * val; +}; + +void neu_id_map_init(neu_id_map *m, uint32_t lo, uint32_t hi, bool randomize) +{ + if (lo == 0) { + lo = 1; + } + if (hi == 0) { + hi = 0xffffffffu; + } + NEU_ASSERT(lo != 0); + NEU_ASSERT(hi > lo); + m->id_entries = NULL; + m->id_count = 0; + m->id_load = 0; + m->id_cap = 0; + m->id_max_load = 0; + m->id_min_load = 0; // never shrink below this + m->id_min_val = lo; + m->id_max_val = hi; + m->id_dyn_val = lo; +} + +void neu_id_map_fini(neu_id_map *m) +{ + if (m->id_entries != NULL) { + NEU_FREE_STRUCTS(m->id_entries, m->id_cap); + m->id_entries = NULL; + m->id_cap = m->id_count = 0; + m->id_load = m->id_min_load = m->id_max_load = 0; + } +} + +// Inspired by Python dict implementation. This probe will visit every +// cell. We always hash consecutively assigned IDs. This requires that +// the capacity is always a power of two. +#define ID_NEXT(m, j) ((((j) *5) + 1) & (m->id_cap - 1)) +#define ID_INDEX(m, j) ((j) & (m->id_cap - 1)) + +static size_t id_find(neu_id_map *m, uint32_t id) +{ + size_t index; + size_t start; + if (m->id_count == 0) { + return ((size_t) -1); + } + + index = ID_INDEX(m, id); + start = index; + for (;;) { + // The value of ihe_key is only valid if ihe_val is not NULL. + if ((m->id_entries[index].key == id) && + (m->id_entries[index].val != NULL)) { + return (index); + } + if (m->id_entries[index].skips == 0) { + return ((size_t) -1); + } + index = ID_NEXT(m, index); + + if (index == start) { + break; + } + } + + return ((size_t) -1); +} + +void *neu_id_get(neu_id_map *m, uint32_t id) +{ + size_t index; + if ((index = id_find(m, id)) == (size_t) -1) { + return (NULL); + } + return (m->id_entries[index].val); +} + +static int id_resize(neu_id_map *m) +{ + size_t new_cap; + size_t old_cap; + neu_id_entry *new_entries; + neu_id_entry *old_entries; + uint32_t i; + + if ((m->id_load < m->id_max_load) && (m->id_load >= m->id_min_load)) { + // No resize needed. + return (0); + } + + old_cap = m->id_cap; + new_cap = 8; + while (new_cap < (m->id_count * 2)) { + new_cap *= 2; + } + if (new_cap == old_cap) { + // Same size. + return (0); + } + + old_entries = m->id_entries; + new_entries = NEU_ALLOC_STRUCTS(new_entries, new_cap); + if (new_entries == NULL) { + return 2; + } + + m->id_entries = new_entries; + m->id_cap = new_cap; + m->id_load = 0; + if (new_cap > 8) { + m->id_min_load = new_cap / 8; + m->id_max_load = new_cap * 2 / 3; + } else { + m->id_min_load = 0; + m->id_max_load = 5; + } + for (i = 0; i < old_cap; i++) { + size_t index; + if (old_entries[i].val == NULL) { + continue; + } + index = old_entries[i].key & (new_cap - 1); + for (;;) { + // Increment the load unconditionally. It counts + // once for every item stored, plus once for each + // hashing operation we use to store the item (i.e. + // one for the item, plus once for each rehash.) + m->id_load++; + if (new_entries[index].val == NULL) { + // As we are hitting this entry for the first + // time, it won't have any skips. + NEU_ASSERT(new_entries[index].skips == 0); + new_entries[index].val = old_entries[i].val; + new_entries[index].key = old_entries[i].key; + break; + } + new_entries[index].skips++; + index = ID_NEXT(m, index); + } + } + if (old_cap != 0) { + NEU_FREE_STRUCTS(old_entries, old_cap); + } + return (0); +} + +int neu_id_remove(neu_id_map *m, uint32_t id) +{ + size_t index; + size_t probe; + + if ((index = id_find(m, id)) == (size_t) -1) { + return 2; + } + + // Now we have found the index where the object exists. We are going + // to restart the search, until the index matches, to decrement the + // skips counter. + probe = ID_INDEX(m, id); + + for (;;) { + neu_id_entry *entry; + + // The load was increased once each hashing operation we used + // to place the the item. Decrement it accordingly. + m->id_load--; + entry = &m->id_entries[probe]; + if (probe == index) { + entry->val = NULL; + entry->key = 0; // invalid key + break; + } + NEU_ASSERT(entry->skips > 0); + entry->skips--; + probe = ID_NEXT(m, probe); + } + + m->id_count--; + + // Shrink -- but it's ok if we can't. + (void) id_resize(m); + + return (0); +} + +int neu_id_set(neu_id_map *m, uint32_t id, void *val) +{ + size_t index; + neu_id_entry *ent; + + // Try to resize -- if we don't need to, this will be a no-op. + if (id_resize(m) != 0) { + return 2; + } + + // If it already exists, just overwrite the old value. + if ((index = id_find(m, id)) != (size_t) -1) { + ent = &m->id_entries[index]; + ent->val = val; + return (0); + } + + index = ID_INDEX(m, id); + for (;;) { + ent = &m->id_entries[index]; + + // Increment the load count. We do this each time time we + // rehash. This may over-count items that collide on the + // same rehashing, but this should just cause a table to + // grow sooner, which is probably a good thing. + m->id_load++; + if (ent->val == NULL) { + m->id_count++; + ent->key = id; + ent->val = val; + return (0); + } + // Record the skip count. This being non-zero informs + // that a rehash will be necessary. Without this we + // would need to scan the entire hash for the match. + ent->skips++; + index = ID_NEXT(m, index); + } +} + +int neu_id_alloc(neu_id_map *m, uint32_t *idp, void *val) +{ + uint32_t id; + int rv; + + NEU_ASSERT(val != NULL); + + // range is inclusive, so > to get +1 effect. + if (m->id_count > (m->id_max_val - m->id_min_val)) { + // Really more like ENOSPC.. the table is filled to max. + return 2; + } + + for (;;) { + id = m->id_dyn_val; + m->id_dyn_val++; + if (m->id_dyn_val > m->id_max_val) { + m->id_dyn_val = m->id_min_val; + } + + if (id_find(m, id) == (size_t) -1) { + break; + } + } + + rv = neu_id_set(m, id, val); + if (rv == 0) { + *idp = id; + } + return (rv); +} diff --git a/src/utils/idhash.h b/src/utils/idhash.h new file mode 100644 index 00000000..91e5237a --- /dev/null +++ b/src/utils/idhash.h @@ -0,0 +1,61 @@ +/** + * NEURON IIoT System for Industry 4.0 + * Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + **/ + +#ifndef __NEU_IDHASH_H__ +#define __NEU_IDHASH_H__ + +#include +#include +#include + +// We find that we often want to have a list of things listed by a +// numeric ID, which is generally monotonically increasing. This is +// most often a pipe ID. To help keep collections of these things +// indexed by their ID (which might start from a very large value), +// we offer a hash table. The hash table uses open addressing, but +// we use a better probe (taken from Python) to avoid hitting the same +// positions. Our hash algorithm is just the low order bits, and we +// use table sizes that are powers of two. Note that hash items +// must be non-NULL. The table is protected by an internal lock. + +typedef struct neu_id_map neu_id_map; +typedef struct neu_id_entry neu_id_entry; + +// NB: These details are entirely private to the hash implementation. +// They are provided here to facilitate inlining in structures. +struct neu_id_map { + size_t id_cap; + size_t id_count; + size_t id_load; + size_t id_min_load; // considers placeholders + size_t id_max_load; + uint32_t id_min_val; + uint32_t id_max_val; + uint32_t id_dyn_val; + neu_id_entry *id_entries; +}; + +extern void neu_id_map_init(neu_id_map *, uint32_t, uint32_t, bool); +extern void neu_id_map_fini(neu_id_map *); +extern void *neu_id_get(neu_id_map *, uint32_t); +extern int neu_id_set(neu_id_map *, uint32_t, void *); +extern int neu_id_alloc(neu_id_map *, uint32_t *, void *); +extern int neu_id_remove(neu_id_map *, uint32_t); + +#endif // __NEU_IDHASH_H__ diff --git a/src/utils/mem_alloc.c b/src/utils/mem_alloc.c new file mode 100644 index 00000000..814b7e27 --- /dev/null +++ b/src/utils/mem_alloc.c @@ -0,0 +1,36 @@ +/** + * NEURON IIoT System for Industry 4.0 + * Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + **/ + +#include "mem_alloc.h" + +void *neu_alloc(size_t sz) +{ + return (sz > 0 ? malloc(sz) : NULL); +} + +void *neu_zalloc(size_t sz) +{ + return (sz > 0 ? calloc(1, sz) : NULL); +} + +void neu_free(void *b, size_t z) +{ + NEU_ARG_UNUSED(z); + free(b); +} diff --git a/src/utils/mem_alloc.h b/src/utils/mem_alloc.h new file mode 100644 index 00000000..9aefa3e2 --- /dev/null +++ b/src/utils/mem_alloc.h @@ -0,0 +1,37 @@ +/** + * NEURON IIoT System for Industry 4.0 + * Copyright (C) 2020-2021 EMQ Technologies Co., Ltd All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + **/ + +#ifndef __MEM_ALLOC_H__ +#define __MEM_ALLOC_H__ + +#include "neu_panic.h" +#include + +#define NEU_ARG_UNUSED(x) ((void) x) + +void *neu_alloc(size_t sz); +void *neu_zalloc(size_t sz); +void neu_free(void *b, size_t z); + +#define NNI_ALLOC_STRUCT(s) neu_zalloc(sizeof(*s)) +#define NNI_FREE_STRUCT(s) neu_free((s), sizeof(*s)) +#define NEU_ALLOC_STRUCTS(s, n) neu_zalloc(sizeof(*s) * n) +#define NEU_FREE_STRUCTS(s, n) neu_free(s, sizeof(*s) * n) + +#endif \ No newline at end of file