GithubStar/neuron
1
0
Fork 0
mirror of https://github.com/emqx/neuron.git synced 2026-10-28 22:59:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #632 from eeff/eeff/dev

Browse Source 
Add migrate.py to sdk
This commit is contained in:
fengzeroz
2022-07-15 15:16:29 +08:00
committed by GitHub
parent 4cc5357834 840cb3fd55
commit 0a7a491356
6 changed files with 92 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1
View File
@@ -129,6 +129,7 @@ file(COPY ${CMAKE_SOURCE_DIR}/dev.conf DESTINATION ${CMAKE_BINARY_DIR}/config)
file(COPY ${CMAKE_SOURCE_DIR}/.gitkeep DESTINATION ${CMAKE_BINARY_DIR}/logs)
file(COPY ${CMAKE_SOURCE_DIR}/neuron.key DESTINATION ${CMAKE_BINARY_DIR}/config)
file(COPY ${CMAKE_SOURCE_DIR}/neuron.pem DESTINATION ${CMAKE_BINARY_DIR}/config)
file(COPY ${CMAKE_SOURCE_DIR}/persistence DESTINATION ${CMAKE_BINARY_DIR}/)
file(COPY ${CMAKE_SOURCE_DIR}/default_plugins.json DESTINATION ${CMAKE_BINARY_DIR}/persistence)
file(RENAME ${CMAKE_BINARY_DIR}/persistence/default_plugins.json ${CMAKE_BINARY_DIR}/persistence/plugins.json)
Install-dependencies.md
+10 -1
View File
@@ -45,4 +45,13 @@ $ cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_
$ git clone -b release-1.11.0 https://github.com/google/googletest.git
$ cd googletest && mkdir build && cd build
$ cmake .. && make && sudo make install
```
```
[sqlite](https://github.com/sqlite/sqlite)
```shell
$ curl -o sqlite3.tar.gz https://www.sqlite.org/2022/sqlite-autoconf-3390000.tar.gz
$ mkdir sqlite3
$ tar xzf sqlite3.tar.gz --strip-components=1 -C sqlite3
$ cd sqlite3
$ ./configure && make && sudo make install
```
install.sh
+13
View File
@@ -76,6 +76,18 @@ function build_zlog() {
fi
}
function build_sqlite3() {
curl https://www.sqlite.org/2022/sqlite-autoconf-3390000.tar.gz \
--output sqlite3.tar.gz
mkdir -p sqlite3
tar xzf sqlite3.tar.gz --strip-components=1 -C sqlite3
cd sqlite3
./configure --prefix=${install_dir} \
&& make CC=${compiler_prefix}-gcc -j4 \
&& sudo make install
cd ../
}
build_zlog
compile_source madler/zlib.git v1.2.11
compile_source akheron/jansson.git v2.14 "-DJANSSON_BUILD_DOCS=OFF -DJANSSON_EXAMPLES=OFF"
@@ -83,6 +95,7 @@ build_openssl
compile_source neugates/nng.git HEAD "-DBUILD_SHARED_LIBS=OFF -DNNG_TESTS=OFF"
compile_source benmcollins/libjwt.git v1.13.1 "-DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF"
compile_source neugates/MQTT-C.git HEAD "-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF"
build_sqlite3
if [ $compiler_prefix == "x86_64-linux-gnu" ]; then
compile_source google/googletest.git release-1.11.0
migrate.py
+55 -10
View File
@@ -1,10 +1,29 @@
#!/usr/bin/env python3
#!/usr/bin/env python
#
# NEURON IIoT System for Industry 4.0
# Copyright (C) 2020-2022 EMQ Technologies Co., Ltd All rights reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
from __future__ import print_function
import argparse
import os
import re
import sqlite3
import operator
import json
@@ -20,16 +39,33 @@ def parse_args():
"""Parse command line arguments."""
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", action="count", help="verbosity")
parser.add_argument("-d", "--data_dir", type=str, help="data directory")
parser.add_argument(
"-d",
"--data_dir",
type=str,
default="persistence/",
help="data directory",
)
subparsers = parser.add_subparsers(
dest="subcommand", help="supported subcommands"
)
show_parser = subparsers.add_parser("show")
import_parser = subparsers.add_parser(
"import", help="import persistence json data"
)
import_parser = subparsers.add_parser("import")
show_parser = subparsers.add_parser(
"show", help="show database schema version"
)
up_parser = subparsers.add_parser("up")
up_parser = subparsers.add_parser(
"up", help="bring database schema to latest version"
)
up_parser.add_argument(
"--import_once",
action="store_true",
help="import persistence json data if database is empty",
)
return parser.parse_args()
@@ -40,7 +76,7 @@ class Migration:
database_path = os.path.join(database_dir, database_name)
conn = sqlite3.connect(database_path)
except Exception as e:
log("Connecting database `{}` fail, {}".format(database_path, e))
print("Connecting database `{}` fail, {}".format(database_path, e))
raise e
self.conn_ = conn
@@ -123,7 +159,7 @@ class Migration:
"description": match.group("description"),
}
def should_apply(self, migration_filename: str, target_version=None):
def should_apply(self, migration_filename, target_version=None):
"""Check whether to apply the migration file."""
head = self.head()
@@ -149,7 +185,7 @@ class Migration:
return True
def apply(self):
def apply(self, import_once):
"""Migrate to the latest state."""
head = self.head()
if head["dirty"]:
@@ -190,6 +226,15 @@ class Migration:
)
self.conn_.commit()
if import_once and self.head()["version"] and self.check_no_data():
self.import_json()
def check_no_data(self):
"""Check the database contains no data."""
self.cursor_.execute("SELECT COUNT(*) FROM nodes")
node_count = self.cursor_.fetchone()[0]
return node_count == 0
def import_json(self):
node_group_names = self.find_json_importable_node_and_group_names()
node_names = self.find_json_importable_node_names()
@@ -395,4 +440,4 @@ if __name__ == "__main__":
elif args.subcommand == "show":
migrations.print_head()
elif args.subcommand == "up":
migrations.apply()
migrations.apply(args.import_once)
package-sdk.sh
+3
View File
@@ -43,6 +43,9 @@ cp build/neuron ${package_name}
cp build/libneuron-base.so ${package_name}/lib
cp zlog/src/libzlog.so.1.2 ${package_name}/lib
cp -r persistence ${package_name}/
cp migrate.py ${package_name}/
cp zlog.conf ${package_name}/config/
cp neuron.key ${package_name}/config/
cp neuron.pem ${package_name}/config/
persistence/0000_2.1.0_initial.sql
+10 -11
View File
@@ -17,28 +17,29 @@
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**/
--- Add plugins table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
plugins (library TEXT PRIMARY KEY check(length(library) <= 32));
--- Add adapters table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
nodes (
name TEXT PRIMARY KEY check(length(name) <= 32),
type integer(1) NOT NULL check(TYPE IN (1, 2)),
type integer(1) NOT NULL check(type IN (1, 2)),
state integer(1) NOT NULL check(state BETWEEN 0 AND 4),
plugin_name TEXT NOT NULL check(length(plugin_name) <= 32)
);
--- Add settings table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
settings (
node_name TEXT NOT NULL,
setting TEXT NOT NULL,
UNIQUE (node_name),
FOREIGN KEY (node_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE
);
--- Add groups table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
groups (
driver_name TEXT NOT NULL,
name NOT NULL check(length(name) <= 32),
@@ -48,7 +49,7 @@ CREATE TABLE
);
--- Add tags table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
tags (
driver_name TEXT NOT NULL,
group_name TEXT NOT NULL,
@@ -58,12 +59,11 @@ CREATE TABLE
type INTEGER NOT NULL check(type BETWEEN 0 AND 15),
description NOT NULL check(length(description) <= 128),
UNIQUE (driver_name, group_name, name),
FOREIGN KEY (driver_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE,
FOREIGN KEY (group_name) REFERENCES groups (name) ON UPDATE CASCADE ON DELETE CASCADE
FOREIGN KEY (driver_name, group_name) REFERENCES groups (driver_name, name) ON UPDATE CASCADE ON DELETE CASCADE
);
--- Add subscriptions table ---
CREATE TABLE
CREATE TABLE IF NOT EXISTS
subscriptions (
app_name TEXT NOT NULL,
driver_name TEXT NOT NULL,
@@ -71,6 +71,5 @@ CREATE TABLE
CHECK (app_name != driver_name),
UNIQUE (app_name, driver_name, group_name),
FOREIGN KEY (app_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE,
FOREIGN KEY (driver_name) REFERENCES nodes (name) ON UPDATE CASCADE ON DELETE CASCADE,
FOREIGN KEY (group_name) REFERENCES groups (name) ON UPDATE CASCADE ON DELETE CASCADE
FOREIGN KEY (driver_name, group_name) REFERENCES groups (driver_name, name) ON UPDATE CASCADE ON DELETE CASCADE
);