From 8a224f4f156bdb478d5ee8b061d314fa346ccd73 Mon Sep 17 00:00:00 2001 From: eeff <33566201+eeff@users.noreply.github.com> Date: Mon, 11 Jul 2022 15:27:09 +0800 Subject: [PATCH 1/5] fix(persistence): migrate.py command line interface --- migrate.py | 65 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/migrate.py b/migrate.py index 051e57d1..ee2acca6 100755 --- a/migrate.py +++ b/migrate.py @@ -1,10 +1,29 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python +# +# NEURON IIoT System for Industry 4.0 +# Copyright (C) 2020-2022 EMQ Technologies Co., Ltd All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +from __future__ import print_function import argparse import os import re import sqlite3 -import operator import json @@ -20,16 +39,33 @@ def parse_args(): """Parse command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="count", help="verbosity") - parser.add_argument("-d", "--data_dir", type=str, help="data directory") + parser.add_argument( + "-d", + "--data_dir", + type=str, + default="persistence/", + help="data directory", + ) subparsers = parser.add_subparsers( dest="subcommand", help="supported subcommands" ) - show_parser = subparsers.add_parser("show") + import_parser = subparsers.add_parser( + "import", help="import persistence json data" + ) - import_parser = subparsers.add_parser("import") + show_parser = subparsers.add_parser( + "show", help="show database schema version" + ) - up_parser = subparsers.add_parser("up") + up_parser = subparsers.add_parser( + "up", help="bring database schema to latest version" + ) + up_parser.add_argument( + "--import_once", + action="store_true", + help="import persistence json data if database is empty", + ) return parser.parse_args() @@ -40,7 +76,7 @@ class Migration: database_path = os.path.join(database_dir, database_name) conn = sqlite3.connect(database_path) except Exception as e: - log("Connecting database `{}` fail, {}".format(database_path, e)) + print("Connecting database `{}` fail, {}".format(database_path, e)) raise e self.conn_ = conn @@ -123,7 +159,7 @@ class Migration: "description": match.group("description"), } - def should_apply(self, migration_filename: str, target_version=None): + def should_apply(self, migration_filename, target_version=None): """Check whether to apply the migration file.""" head = self.head() @@ -149,7 +185,7 @@ class Migration: return True - def apply(self): + def apply(self, import_once): """Migrate to the latest state.""" head = self.head() if head["dirty"]: @@ -190,6 +226,15 @@ class Migration: ) self.conn_.commit() + if import_once and self.head()["version"] and self.check_no_data(): + self.import_json() + + def check_no_data(self): + """Check the database contains no data.""" + self.cursor_.execute("SELECT COUNT(*) FROM nodes") + node_count = self.cursor_.fetchone()[0] + return node_count == 0 + def import_json(self): node_group_names = self.find_json_importable_node_and_group_names() node_names = self.find_json_importable_node_names() @@ -395,4 +440,4 @@ if __name__ == "__main__": elif args.subcommand == "show": migrations.print_head() elif args.subcommand == "up": - migrations.apply() + migrations.apply(args.import_once) From 1b4c25e992e4a9d4e822eeb8e010e518f1d4780e Mon Sep 17 00:00:00 2001 From: eeff <33566201+eeff@users.noreply.github.com> Date: Mon, 11 Jul 2022 15:31:50 +0800 Subject: [PATCH 2/5] build: add persistence data migration tool to neuron sdk --- package-sdk.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package-sdk.sh b/package-sdk.sh index d35fce05..1eb8860f 100755 --- a/package-sdk.sh +++ b/package-sdk.sh @@ -43,6 +43,9 @@ cp build/neuron ${package_name} cp build/libneuron-base.so ${package_name}/lib cp zlog/src/libzlog.so.1.2 ${package_name}/lib +cp -r persistence ${package_name}/ +cp migrate.py ${package_name}/ + cp zlog.conf ${package_name}/config/ cp neuron.key ${package_name}/config/ cp neuron.pem ${package_name}/config/ From 4e1654ba1753d686a00d99ff6cdb00d745ad7c84 Mon Sep 17 00:00:00 2001 From: eeff <33566201+eeff@users.noreply.github.com> Date: Mon, 11 Jul 2022 17:40:16 +0800 Subject: [PATCH 3/5] build: update install.sh to install sqlite3 --- Install-dependencies.md | 11 ++++++++++- install.sh | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Install-dependencies.md b/Install-dependencies.md index 49d76117..0f12df1f 100644 --- a/Install-dependencies.md +++ b/Install-dependencies.md @@ -45,4 +45,13 @@ $ cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_ $ git clone -b release-1.11.0 https://github.com/google/googletest.git $ cd googletest && mkdir build && cd build $ cmake .. && make && sudo make install -``` \ No newline at end of file +``` + +[sqlite](https://github.com/sqlite/sqlite) +```shell +$ curl -o sqlite3.tar.gz https://www.sqlite.org/2022/sqlite-autoconf-3390000.tar.gz +$ mkdir sqlite3 +$ tar xzf sqlite3.tar.gz --strip-components=1 -C sqlite3 +$ cd sqlite3 +$ ./configure && make && sudo make install +``` diff --git a/install.sh b/install.sh index 1e4cb566..31596d11 100755 --- a/install.sh +++ b/install.sh @@ -76,6 +76,18 @@ function build_zlog() { fi } +function build_sqlite3() { + curl https://www.sqlite.org/2022/sqlite-autoconf-3390000.tar.gz \ + --output sqlite3.tar.gz + mkdir -p sqlite3 + tar xzf sqlite3.tar.gz --strip-components=1 -C sqlite3 + cd sqlite3 + ./configure --prefix=${install_dir} \ + && make CC=${compiler_prefix}-gcc -j4 \ + && sudo make install + cd ../ +} + build_zlog compile_source madler/zlib.git v1.2.11 compile_source akheron/jansson.git v2.14 "-DJANSSON_BUILD_DOCS=OFF -DJANSSON_EXAMPLES=OFF" @@ -83,6 +95,7 @@ build_openssl compile_source neugates/nng.git HEAD "-DBUILD_SHARED_LIBS=OFF -DNNG_TESTS=OFF" compile_source benmcollins/libjwt.git v1.13.1 "-DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF" compile_source neugates/MQTT-C.git HEAD "-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF" +build_sqlite3 if [ $compiler_prefix == "x86_64-linux-gnu" ]; then compile_source google/googletest.git release-1.11.0 From e429259ff55600bd59a0ea0dafc71cf8c8af0dbc Mon Sep 17 00:00:00 2001 From: eeff <33566201+eeff@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:49:57 +0800 Subject: [PATCH 4/5] fix(persistence): fix sql schema --- persistence/0000_2.1.0_initial.sql | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/persistence/0000_2.1.0_initial.sql b/persistence/0000_2.1.0_initial.sql index 97f601b1..35393b5d 100644 --- a/persistence/0000_2.1.0_initial.sql +++ b/persistence/0000_2.1.0_initial.sql @@ -17,28 +17,29 @@ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. **/ --- Add plugins table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS plugins (library TEXT PRIMARY KEY check(length(library) <= 32)); --- Add adapters table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS nodes ( name TEXT PRIMARY KEY check(length(name) <= 32), - type integer(1) NOT NULL check(TYPE IN (1, 2)), + type integer(1) NOT NULL check(type IN (1, 2)), state integer(1) NOT NULL check(state BETWEEN 0 AND 4), plugin_name TEXT NOT NULL check(length(plugin_name) <= 32) ); --- Add settings table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS settings ( node_name TEXT NOT NULL, setting TEXT NOT NULL, + UNIQUE (node_name), FOREIGN KEY (node_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE ); --- Add groups table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS groups ( driver_name TEXT NOT NULL, name NOT NULL check(length(name) <= 32), @@ -48,7 +49,7 @@ CREATE TABLE ); --- Add tags table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS tags ( driver_name TEXT NOT NULL, group_name TEXT NOT NULL, @@ -58,12 +59,11 @@ CREATE TABLE type INTEGER NOT NULL check(type BETWEEN 0 AND 15), description NOT NULL check(length(description) <= 128), UNIQUE (driver_name, group_name, name), - FOREIGN KEY (driver_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE, - FOREIGN KEY (group_name) REFERENCES groups (name) ON UPDATE CASCADE ON DELETE CASCADE + FOREIGN KEY (driver_name, group_name) REFERENCES groups (driver_name, name) ON UPDATE CASCADE ON DELETE CASCADE ); --- Add subscriptions table --- -CREATE TABLE +CREATE TABLE IF NOT EXISTS subscriptions ( app_name TEXT NOT NULL, driver_name TEXT NOT NULL, @@ -71,6 +71,5 @@ CREATE TABLE CHECK (app_name != driver_name), UNIQUE (app_name, driver_name, group_name), FOREIGN KEY (app_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE, - FOREIGN KEY (driver_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE, - FOREIGN KEY (group_name) REFERENCES groups (name) ON UPDATE CASCADE ON DELETE CASCADE + FOREIGN KEY (driver_name, group_name) REFERENCES groups (driver_name, name) ON UPDATE CASCADE ON DELETE CASCADE ); From 840cb3fd554793a6a3b9564db4e619c8408316b6 Mon Sep 17 00:00:00 2001 From: eeff <33566201+eeff@users.noreply.github.com> Date: Thu, 14 Jul 2022 17:49:58 +0800 Subject: [PATCH 5/5] build: copy persistence directory to CMake binary dir --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dd49034..915a4581 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,7 @@ file(COPY ${CMAKE_SOURCE_DIR}/dev.conf DESTINATION ${CMAKE_BINARY_DIR}/config) file(COPY ${CMAKE_SOURCE_DIR}/.gitkeep DESTINATION ${CMAKE_BINARY_DIR}/logs) file(COPY ${CMAKE_SOURCE_DIR}/neuron.key DESTINATION ${CMAKE_BINARY_DIR}/config) file(COPY ${CMAKE_SOURCE_DIR}/neuron.pem DESTINATION ${CMAKE_BINARY_DIR}/config) +file(COPY ${CMAKE_SOURCE_DIR}/persistence DESTINATION ${CMAKE_BINARY_DIR}/) file(COPY ${CMAKE_SOURCE_DIR}/default_plugins.json DESTINATION ${CMAKE_BINARY_DIR}/persistence) file(RENAME ${CMAKE_BINARY_DIR}/persistence/default_plugins.json ${CMAKE_BINARY_DIR}/persistence/plugins.json)