From 365795300d8814c2794cf05817fb09ac03471b8a Mon Sep 17 00:00:00 2001 From: Milan Toman Date: Wed, 21 Aug 2019 18:07:47 +0200 Subject: [PATCH] production push, including pressure --- python/davis_etl.py | 172 ++++++++++++++++++++++++++--- python/sysstats.py | 256 +++++++++++++++++++++++--------------------- 2 files changed, 296 insertions(+), 132 deletions(-) mode change 100755 => 100644 python/sysstats.py diff --git a/python/davis_etl.py b/python/davis_etl.py index 5578180..c1a4b29 100755 --- a/python/davis_etl.py +++ b/python/davis_etl.py @@ -50,6 +50,19 @@ field | tag +# + TABLE vantage_vue_iss + ---------------------- + t_stamp | voltage | type | lqi | rssi | batt_low + ------------------------------------------------------------------ + INT | INT | VARCHAR(20) | TINYINT | TINYINT | BOOL + + CREATE TABLE raspi( + t_stamp INT, + count INT, + type VARCHAR(20), + nic VARCHAR(20), + host VARCHAR(50)); -------------------------------------------------------------------------------- Import libraries @@ -84,10 +97,13 @@ sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib") ''' _VERSION = 2.0 _NAME = u"Vantage Vue Decoding shite" +_LOG_DIR = _SCRIPT_PATH + '/log/' +_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0]) +_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log' +_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg' # finite loop implementation, tout for 43200 cycles tout = 0 -_SERIAL_CONSOLE = '/dev/ttyUSBdavis' _ABS_ZERO = 273.15 _HEIGHT = 455 temp = {} @@ -105,11 +121,75 @@ influx_weather_write = [] influx_host = 'localhost' influx_port = 8086 influx_user = 'pi' -influx_pwd = 'freedavis' +influx_pwd = 'Ventil6996' weather_db = 'weather_v2' status_db = 'status' +''' +-------------------------------------------------------------------------------- + Set up logging - disabled, need to enable this iin future +-------------------------------------------------------------------------------- +''' + +''' +-------------------------------------------------------------------------------- + Setup arguments and Options - not edited, sample shite +-------------------------------------------------------------------------------- +''' +desc = u'''\ +DESCRIPTION: + Vantage Vue wireless data transfer decoder, V2 + consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/ + for wtf is going on + ''' +epi = u'''\ + ERROR CODES: + ? + + EXAMPLES: + ? + + ''' +formatter = argparse.RawDescriptionHelpFormatter +arg_parser = argparse.ArgumentParser(description = desc, + formatter_class = formatter, + epilog = textwrap.dedent(epi)) + +arg_parser.add_argument('-d', '--details', + help = 'help', + action='store_true') +arg_parser.add_argument('-v', '--verbose', + help = 'help', + action='store_true') +arg_parser.add_argument('-p', '--section', + dest = 'section', + default = ['last', 'count', 'diff'], + choices = ['last', 'count', 'diff'], + nargs = '+', + type = str, + help = 'help') +arg_parser.add_argument('-s', '--snapusage', + help = 'help', + action='store_true') + +args = arg_parser.parse_args() +if args.details: + _details = True +else: + _details = False +if args.verbose: + _more_details = True +else: + _more_details = False +if args.snapusage: + _SNAP_USAGE = True +else: + _SNAP_USAGE = False +try: + _sections = args.sections +except: + _sections = ['last', 'count', 'diff'] ''' -------------------------------------------------------------------------------- Generic, standalone functions @@ -124,6 +204,32 @@ influx_status_client = influxdb.client.InfluxDBClient( influx_host, influx_port, influx_user, influx_pwd, status_db ) +def create_connection(db_file): + """ create a database connection to the SQLite database + specified by db_file + :param db_file: database file + :return: Connection object or None + """ + try: + conn = sqlite3.connect(db_file) + return conn + except Error as e: + print(e) + return None + +def create_project(conn, project): + """ + Create a new project into the projects table + :param conn: + :param project: + :return: project id + """ + sql = ''' INSERT INTO wind(name,begin_date,end_date) + VALUES(?,?,?) ''' + cur = conn.cursor() + cur.execute(sql, project) + return cur.lastrowid + ''' -------------------------------------------------------------------------------- Classes @@ -136,6 +242,13 @@ class davisDecoder(object): self.height = _HEIGHT self.temp_dict = {} + def load_external_data(self): + # Data external to the ISS + self.pressure = float(davis_data['P']) + self.inside_temp = round((float(davis_data['Ti'])\ + + float(davis_data['Thtu'])) / 2, 2) + self.inside_hum = davis_data['Hhtu'] + def zero_fill(self, data): binary_data = format(int(data), '08b') msb = binary_data[0:4] @@ -172,6 +285,12 @@ class davisDecoder(object): def decode_humidity(self, hum): pass + def adjust_pressure(self, temp): + sh = 0.0065 * self.height + base = 1 - (sh) / (temp + sh + _ABS_ZERO) + result = round(self.pressure * pow(base, -5.257), 2) + return result + def supercap_decode(self, byte2, byte3): cap = (byte2 << 2) + (byte3 >> 6) result = float(cap / 100.00) @@ -189,15 +308,12 @@ class davisDecoder(object): def rainrate_decode(self, byte2, byte3): # if byte3(b2 here) is 0xFF, or 255, there is not rain print("b2:{} b3:{} = result:{}".format(byte2, byte3, byte2 + (byte3 >> 4 << 8))) - # No Rain rainstate = 0 if byte2 == 255: rainstate = 0 rainrate = 0 - # Light Rain rainstate = 1 elif byte2 == 254: rainstate = 1 rainrate = 0.1 - # Heavy rain rainstate = 2 else: rainstate = 2 if byte3 > 4: @@ -254,6 +370,14 @@ class DBwriter(object): "measurement": "wind", "fields": { "value": base_value_dict['direction'] }, "tags": { "type": "direction" } + }, + { + "measurement": "temphumi", + "fields": { + "temperature": base_value_dict['temperature'], + "humidity": base_value_dict['humidity'] + }, + "tags": { "type": "internal" } }] return base_connector @@ -285,7 +409,7 @@ if '__main__': davis_decoder = davisDecoder() davis_writer = DBwriter() try: - with serial.Serial(_SERIAL_CONSOLE, 9600) as davis: + with serial.Serial('/dev/ttyUSBdavis', 9600) as davis: # Now, let it run a couple times, end and restart via systemd while tout < 400: line = davis.readline() @@ -311,10 +435,14 @@ if '__main__': raw_winddir = davis_data['b1'] wind = davis_decoder.decode_wind({"windspeed": raw_windspeed, "winddir": raw_winddir}) - print(wind) + # Get data external to the ISS, from local PCB / internal + # sensors and create base values for influx writing + davis_decoder.load_external_data() influx_weather_write = davis_writer.base_construct( { "speed": float(wind['speed']), - "direction": float(wind['direction'])} + "direction": float(wind['direction']), + "temperature": float(davis_decoder.inside_temp), + "humidity": float(davis_decoder.inside_hum)} ) # Wind gusts calculation @@ -333,6 +461,19 @@ if '__main__': raw_temp = (davis_data['b2'] << 8) + davis_data['b3'] temp_dict = davis_decoder.decode_temp(raw_temp) temp = float(temp_dict['celsius']) + pressure_adjusted = davis_decoder.adjust_pressure(temp) + influx_weather_write = davis_writer.construct( + influx_weather_write, + "temphumi", + {"pressure": float(pressure_adjusted)}, + {"type" : "adjusted"} + ) + influx_weather_write = davis_writer.construct( + influx_weather_write, + "temphumi", + {"pressure": float(davis_decoder.pressure)}, + {"type" : "raw"} + ) influx_weather_write = davis_writer.construct( influx_weather_write, @@ -362,7 +503,7 @@ if '__main__': influx_status_write, "iss", {"voltage": float(supercap)}, - {"type": "capcaitor"} + {"type": "capacitor"} ) # 0x7 -> SolarPanel Voltage @@ -420,15 +561,22 @@ if '__main__': temp, wind, humidity) - + out2 = "Padj: {}, Praw {}, Tins: {}, Humins: {}".format( + pressure_adjusted, + davis_decoder.pressure, + davis_decoder.inside_temp, + davis_decoder.inside_hum) out3 = "RainState: {}, Rrate {}, Rain Total: {}, Cap:{}, Volt: {}".format( rainstate, rainrate, rain, supercap, solarvolt) - - print("\n{} \n{} \n{}\n".format(out_id, out1, out3)) + out4 = {'RAW': davis_data} + with open('/home/pi/davis.rawdata', 'a+') as fh: + now = datetime.datetime.strftime(datetime.datetime.now(), "%s") + fh.write(now + ";" + str(out4) + "\n") + #print("\n{} \n{} \n{} \n{}\n{}\n".format(out_id, out1, out2, out3, out4)) # Write the whole blob into Influx DB influx_weather_client.write_points(influx_weather_write) diff --git a/python/sysstats.py b/python/sysstats.py old mode 100755 new mode 100644 index 1c98315..0f75f89 --- a/python/sysstats.py +++ b/python/sysstats.py @@ -1,35 +1,46 @@ #!/usr/bin/python -u ''' --------------------------------------------------------------------------------- +------------------------------------------------------------------------ Type: Python 3.x script Author: Milan Toman (milan.v.toman@gmail.com) Description: System stats + Version: 2.0 (SQLite) - TODO: CPU util. - - - influxDB SCHEMA: + SQLite SCHEMA: DB status - ISS measure + TABLE raspi ---------------- - voltage | solar or capacitor | state / lqi / | battery or future_shit | - ---------------------------------------------------------------- - field tag field tag + t_stamp | usage | host | type + ----------------------------------------------- + INT | INT | VARCHAR(50) | VARCHAR(20) - RasPI + CREATE TABLE raspi( + t_stamp INT, + usage INT, + host VARCHAR(50), + type VARCHAR(20)); + + TABLE network ---------------- - usage | disk, mem, cpu, eth, wifi % - ------------------------------------ - field | tag + t_stamp | count | type | nic | host + ------------------------------------------------------------ + INT | INT | VARCHAR(20) | VARCHAR(20) | VARCHAR(50) + + CREATE TABLE network( + t_stamp INT, + count INT, + type VARCHAR(20), + nic VARCHAR(20), + host VARCHAR(50)); --------------------------------------------------------------------------------- +------------------------------------------------------------------------ Import libraries --------------------------------------------------------------------------------- +------------------------------------------------------------------------ ''' # mandatory import requests @@ -41,107 +52,39 @@ import argparse import time import datetime import simplejson as json -import influxdb +import sqlite3 import psutil +import socket # optionally, future modules, locally available, I hate dependencies from pprint import pprint _SCRIPT_PATH = os.path.dirname(sys.argv[0]) -sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib") +#sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib") #print(_SCRIPT_PATH + "/lib") -#import ventilLogger - +global _hostname +_hostname = socket.gethostname() ''' --------------------------------------------------------------------------------- +------------------------------------------------------------------------ + SQLite def +------------------------------------------------------------------------ +''' +_SQLiteDB = '/var/lib/plutonium/status.db' + +''' +------------------------------------------------------------------------ Define variables --------------------------------------------------------------------------------- +------------------------------------------------------------------------ ''' _VERSION = 2.0 -_NAME = u"Vantage Vue Decoding shite" -_LOG_DIR = _SCRIPT_PATH + '/log/' -_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0]) -_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log' -_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg' - -influx_status_write = [] - -influx_host = 'localhost' -influx_port = 8086 -influx_user = 'pi' -influx_pwd = 'freedavis' -status_db = 'status' - -''' --------------------------------------------------------------------------------- - Set up logging - disabled, need to enable this iin future --------------------------------------------------------------------------------- -''' +_NAME = u"System stats collector" ''' --------------------------------------------------------------------------------- - Setup arguments and Options - not edited, sample shite --------------------------------------------------------------------------------- -''' -desc = u'''\ -DESCRIPTION: - Vantage Vue wireless data transfer decoder, V2 - consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/ - for wtf is going on - ''' -epi = u'''\ - ERROR CODES: - ? - - EXAMPLES: - ? - - ''' -formatter = argparse.RawDescriptionHelpFormatter -arg_parser = argparse.ArgumentParser(description = desc, - formatter_class = formatter, - epilog = textwrap.dedent(epi)) - -arg_parser.add_argument('-d', '--details', - help = 'help', - action='store_true') -arg_parser.add_argument('-v', '--verbose', - help = 'help', - action='store_true') -arg_parser.add_argument('-p', '--section', - dest = 'section', - default = ['last', 'count', 'diff'], - choices = ['last', 'count', 'diff'], - nargs = '+', - type = str, - help = 'help') -arg_parser.add_argument('-s', '--snapusage', - help = 'help', - action='store_true') - -args = arg_parser.parse_args() -if args.details: - _details = True -else: - _details = False -if args.verbose: - _more_details = True -else: - _more_details = False -if args.snapusage: - _SNAP_USAGE = True -else: - _SNAP_USAGE = False -try: - _sections = args.sections -except: - _sections = ['last', 'count', 'diff'] -''' --------------------------------------------------------------------------------- +------------------------------------------------------------------------ Generic, standalone functions --------------------------------------------------------------------------------- +------------------------------------------------------------------------ ''' # Obvious shit, set up the client class @@ -149,20 +92,35 @@ influx_status_client = influxdb.client.InfluxDBClient( influx_host, influx_port, influx_user, influx_pwd, status_db ) + + +def create_project(conn, project): + """ + Create a new project into the projects table + :param conn: + :param project: + :return: project id + """ + sql = ''' INSERT INTO wind(name,begin_date,end_date) + VALUES(?,?,?) ''' + cur = conn.cursor() + cur.execute(sql, project) + return cur.lastrowid + ''' --------------------------------------------------------------------------------- +------------------------------------------------------------------------ Classes --------------------------------------------------------------------------------- +------------------------------------------------------------------------ ''' class DBwriter(object): def __init__(self): - __name__ = "Database writer class, Influx" + __name__ = "Database writer class, SQLite" - def construct(self, connector, measurement, fields, tags): + def construct(self, DB, query): """ Takes values in a writes them to influxdb - requires: list(connector): connector with all ticks to be written + requires: list(connector): connector with ticks to be written at once str(measurement): the measurement ID to be written dict(fields): fields to be written in one tick @@ -170,51 +128,109 @@ class DBwriter(object): returns: list(result_connector) """ - result_connector = connector - result_connector.append({"measurement": measurement, - "fields": fields, - "tags": tags} - ) + conn = sqlite3.connect(db_file) + c = conn.cursor() + c.execute(query) + conn.commit() + conn.close() return result_connector +class Stats(object): + def __init__(self): + __name__ = "Database writer class, SQLite" + + + ''' --------------------------------------------------------------------------------- +------------------------------------------------------------------------ Main --------------------------------------------------------------------------------- +------------------------------------------------------------------------ ''' if '__main__': - davis_writer = DBwriter() + status_writer = DBwriter() while True: - averaged_cpu = 0 + averaged_cpu = psutil.cpu_percent() + mem_consumption = psutil.virtual_memory()[2] + disk_usage = psutil.disk_usage('/')[3] + print(psutil) # CPU stats - for timeout in range(1,5): + for timeout in range(1,15): if timeout == 1: - averaged_cpu = psutil.cpu_percent() + averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2 mem_consumption = psutil.virtual_memory()[2] disk_usage = psutil.disk_usage('/')[3] + if_counters = psutil.net_io_counters(pernic=True) + interfaces = if_counters.keys() + + wlan_counters = if_counters["wlan0"] + wlan_sent = wlan_counters[0] + wlan_recv = wlan_counters[1] + wlan_error_in = wlan_counters[4] + wlan_error_out = wlan_counters[5] + wlan_drop_in = wlan_counters[6] + wlan_drop_out = wlan_counters[7] + eth_counters = if_counters["eth0"] + eth_sent = eth_counters[0] + eth_recv = eth_counters[1] + eth_error_in = eth_counters[4] + eth_error_out = eth_counters[5] + eth_drop_in = eth_counters[6] + eth_drop_out = eth_counters[7] else: averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2 time.sleep(1) # Write the whole blob into Influx DB - influx_status_write = davis_writer.construct( + influx_status_write = status_writer.construct( influx_status_write, "RasPI", {"usage": float(averaged_cpu)}, {"type": "cpu"} ) - influx_status_write = davis_writer.construct( + influx_status_write = status_writer.construct( influx_status_write, "RasPI", {"usage": float(mem_consumption)}, {"type": "mem"} ) - influx_status_write = davis_writer.construct( + influx_status_write = status_writer.construct( influx_status_write, "RasPI", {"usage": float(disk_usage)}, {"type": "disk"} ) - print(influx_status_write) + influx_status_write = status_writer.construct( + influx_status_write, + "net", + { + "b_out": float(wlan_sent), + "b_in": float(wlan_recv), + "e_out": float(wlan_error_out), + "e_in": float(wlan_error_in), + "drop_out": float(wlan_drop_out), + "drop_in": float(wlan_drop_in), + }, + { + "type": "wlan0", + "host": _hostname + } + ) + influx_status_write = status_writer.construct( + influx_status_write, + "net", + { + "b_out": float(eth_sent), + "b_in": float(eth_recv), + "e_out": float(eth_error_out), + "e_in": float(eth_error_in), + "drop_out": float(eth_drop_out), + "drop_in": float(eth_drop_in), + }, + { + "type": "eth0", + "host": _hostname + } + ) + print("Writing values: {}".format(influx_status_write)) influx_status_client.write_points(influx_status_write) influx_status_write = [] averaged_cpu = 0