ventil/freedavis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

production push, including pressure

Browse Source
This commit is contained in:
Milan Toman
2019-08-21 18:07:47 +02:00
parent f3bd135ce0
commit 365795300d
2 changed files with 296 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
python/davis_etl.py
View File

@@ -50,6 +50,19 @@
field | tag
#
TABLE vantage_vue_iss
----------------------
t_stamp | voltage | type | lqi | rssi | batt_low
------------------------------------------------------------------
INT | INT | VARCHAR(20) | TINYINT | TINYINT | BOOL
CREATE TABLE raspi(
t_stamp INT,
count INT,
type VARCHAR(20),
nic VARCHAR(20),
host VARCHAR(50));
--------------------------------------------------------------------------------
Import libraries
@@ -84,10 +97,13 @@ sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
'''
_VERSION = 2.0
_NAME = u"Vantage Vue Decoding shite"
_LOG_DIR = _SCRIPT_PATH + '/log/'
_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0])
_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log'
_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg'
# finite loop implementation, tout for 43200 cycles
tout = 0
_SERIAL_CONSOLE = '/dev/ttyUSBdavis'
_ABS_ZERO = 273.15
_HEIGHT = 455
temp = {}
@@ -105,11 +121,75 @@ influx_weather_write = []
influx_host = 'localhost'
influx_port = 8086
influx_user = 'pi'
influx_pwd = 'freedavis'
influx_pwd = 'Ventil6996'
weather_db = 'weather_v2'
status_db = 'status'
'''
--------------------------------------------------------------------------------
Set up logging - disabled, need to enable this iin future
--------------------------------------------------------------------------------
'''
'''
--------------------------------------------------------------------------------
Setup arguments and Options - not edited, sample shite
--------------------------------------------------------------------------------
'''
desc = u'''\
DESCRIPTION:
Vantage Vue wireless data transfer decoder, V2
consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
for wtf is going on
'''
epi = u'''\
ERROR CODES:
?
EXAMPLES:
?
'''
formatter = argparse.RawDescriptionHelpFormatter
arg_parser = argparse.ArgumentParser(description = desc,
formatter_class = formatter,
epilog = textwrap.dedent(epi))
arg_parser.add_argument('-d', '--details',
help = 'help',
action='store_true')
arg_parser.add_argument('-v', '--verbose',
help = 'help',
action='store_true')
arg_parser.add_argument('-p', '--section',
dest = 'section',
default = ['last', 'count', 'diff'],
choices = ['last', 'count', 'diff'],
nargs = '+',
type = str,
help = 'help')
arg_parser.add_argument('-s', '--snapusage',
help = 'help',
action='store_true')
args = arg_parser.parse_args()
if args.details:
_details = True
else:
_details = False
if args.verbose:
_more_details = True
else:
_more_details = False
if args.snapusage:
_SNAP_USAGE = True
else:
_SNAP_USAGE = False
try:
_sections = args.sections
except:
_sections = ['last', 'count', 'diff']
'''
--------------------------------------------------------------------------------
Generic, standalone functions
@@ -124,6 +204,32 @@ influx_status_client = influxdb.client.InfluxDBClient(
influx_host, influx_port, influx_user, influx_pwd, status_db
)
def create_connection(db_file):
""" create a database connection to the SQLite database
specified by db_file
:param db_file: database file
:return: Connection object or None
"""
try:
conn = sqlite3.connect(db_file)
return conn
except Error as e:
print(e)
return None
def create_project(conn, project):
"""
Create a new project into the projects table
:param conn:
:param project:
:return: project id
"""
sql = ''' INSERT INTO wind(name,begin_date,end_date)
VALUES(?,?,?) '''
cur = conn.cursor()
cur.execute(sql, project)
return cur.lastrowid
'''
--------------------------------------------------------------------------------
Classes
@@ -136,6 +242,13 @@ class davisDecoder(object):
self.height = _HEIGHT
self.temp_dict = {}
def load_external_data(self):
# Data external to the ISS
self.pressure = float(davis_data['P'])
self.inside_temp = round((float(davis_data['Ti'])\
+ float(davis_data['Thtu'])) / 2, 2)
self.inside_hum = davis_data['Hhtu']
def zero_fill(self, data):
binary_data = format(int(data), '08b')
msb = binary_data[0:4]
@@ -172,6 +285,12 @@ class davisDecoder(object):
def decode_humidity(self, hum):
pass
def adjust_pressure(self, temp):
sh = 0.0065 * self.height
base = 1 - (sh) / (temp + sh + _ABS_ZERO)
result = round(self.pressure * pow(base, -5.257), 2)
return result
def supercap_decode(self, byte2, byte3):
cap = (byte2 << 2) + (byte3 >> 6)
result = float(cap / 100.00)
@@ -189,15 +308,12 @@ class davisDecoder(object):
def rainrate_decode(self, byte2, byte3):
# if byte3(b2 here) is 0xFF, or 255, there is not rain
print("b2:{} b3:{} = result:{}".format(byte2, byte3, byte2 + (byte3 >> 4 << 8)))
# No Rain rainstate = 0
if byte2 == 255:
rainstate = 0
rainrate = 0
# Light Rain rainstate = 1
elif byte2 == 254:
rainstate = 1
rainrate = 0.1
# Heavy rain rainstate = 2
else:
rainstate = 2
if byte3 > 4:
@@ -254,6 +370,14 @@ class DBwriter(object):
"measurement": "wind",
"fields": { "value": base_value_dict['direction'] },
"tags": { "type": "direction" }
},
{
"measurement": "temphumi",
"fields": {
"temperature": base_value_dict['temperature'],
"humidity": base_value_dict['humidity']
},
"tags": { "type": "internal" }
}]
return base_connector
@@ -285,7 +409,7 @@ if '__main__':
davis_decoder = davisDecoder()
davis_writer = DBwriter()
try:
with serial.Serial(_SERIAL_CONSOLE, 9600) as davis:
with serial.Serial('/dev/ttyUSBdavis', 9600) as davis:
# Now, let it run a couple times, end and restart via systemd
while tout < 400:
line = davis.readline()
@@ -311,10 +435,14 @@ if '__main__':
raw_winddir = davis_data['b1']
wind = davis_decoder.decode_wind({"windspeed": raw_windspeed,
"winddir": raw_winddir})
print(wind)
# Get data external to the ISS, from local PCB / internal
# sensors and create base values for influx writing
davis_decoder.load_external_data()
influx_weather_write = davis_writer.base_construct(
{ "speed": float(wind['speed']),
"direction": float(wind['direction'])}
"direction": float(wind['direction']),
"temperature": float(davis_decoder.inside_temp),
"humidity": float(davis_decoder.inside_hum)}
)
# Wind gusts calculation
@@ -333,6 +461,19 @@ if '__main__':
raw_temp = (davis_data['b2'] << 8) + davis_data['b3']
temp_dict = davis_decoder.decode_temp(raw_temp)
temp = float(temp_dict['celsius'])
pressure_adjusted = davis_decoder.adjust_pressure(temp)
influx_weather_write = davis_writer.construct(
influx_weather_write,
"temphumi",
{"pressure": float(pressure_adjusted)},
{"type" : "adjusted"}
)
influx_weather_write = davis_writer.construct(
influx_weather_write,
"temphumi",
{"pressure": float(davis_decoder.pressure)},
{"type" : "raw"}
)
influx_weather_write = davis_writer.construct(
influx_weather_write,
@@ -362,7 +503,7 @@ if '__main__':
influx_status_write,
"iss",
{"voltage": float(supercap)},
{"type": "capcaitor"}
{"type": "capacitor"}
)
# 0x7 -> SolarPanel Voltage
@@ -420,15 +561,22 @@ if '__main__':
temp,
wind,
humidity)
out2 = "Padj: {}, Praw {}, Tins: {}, Humins: {}".format(
pressure_adjusted,
davis_decoder.pressure,
davis_decoder.inside_temp,
davis_decoder.inside_hum)
out3 = "RainState: {}, Rrate {}, Rain Total: {}, Cap:{}, Volt: {}".format(
rainstate,
rainrate,
rain,
supercap,
solarvolt)
print("\n{} \n{} \n{}\n".format(out_id, out1, out3))
out4 = {'RAW': davis_data}
with open('/home/pi/davis.rawdata', 'a+') as fh:
now = datetime.datetime.strftime(datetime.datetime.now(), "%s")
fh.write(now + ";" + str(out4) + "\n")
#print("\n{} \n{} \n{} \n{}\n{}\n".format(out_id, out1, out2, out3, out4))
# Write the whole blob into Influx DB
influx_weather_client.write_points(influx_weather_write)

256
python/sysstats.py Executable file → Normal file
View File

@@ -1,35 +1,46 @@
#!/usr/bin/python -u
'''
--------------------------------------------------------------------------------
------------------------------------------------------------------------
Type: Python 3.x script
Author: Milan Toman (milan.v.toman@gmail.com)
Description: System stats
Version: 2.0 (SQLite)
TODO: CPU util.
influxDB SCHEMA:
SQLite SCHEMA:
DB status
ISS measure
TABLE raspi
----------------
voltage | solar or capacitor | state / lqi / | battery or future_shit |
----------------------------------------------------------------
field tag field tag
t_stamp | usage | host | type
-----------------------------------------------
INT | INT | VARCHAR(50) | VARCHAR(20)
RasPI
CREATE TABLE raspi(
t_stamp INT,
usage INT,
host VARCHAR(50),
type VARCHAR(20));
TABLE network
----------------
usage | disk, mem, cpu, eth, wifi %
------------------------------------
field | tag
t_stamp | count | type | nic | host
------------------------------------------------------------
INT | INT | VARCHAR(20) | VARCHAR(20) | VARCHAR(50)
CREATE TABLE network(
t_stamp INT,
count INT,
type VARCHAR(20),
nic VARCHAR(20),
host VARCHAR(50));
--------------------------------------------------------------------------------
------------------------------------------------------------------------
Import libraries
--------------------------------------------------------------------------------
------------------------------------------------------------------------
'''
# mandatory
import requests
@@ -41,107 +52,39 @@ import argparse
import time
import datetime
import simplejson as json
import influxdb
import sqlite3
import psutil
import socket
# optionally, future modules, locally available, I hate dependencies
from pprint import pprint
_SCRIPT_PATH = os.path.dirname(sys.argv[0])
sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
#sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
#print(_SCRIPT_PATH + "/lib")
#import ventilLogger
global _hostname
_hostname = socket.gethostname()
'''
--------------------------------------------------------------------------------
------------------------------------------------------------------------
SQLite def
------------------------------------------------------------------------
'''
_SQLiteDB = '/var/lib/plutonium/status.db'
'''
------------------------------------------------------------------------
Define variables
--------------------------------------------------------------------------------
------------------------------------------------------------------------
'''
_VERSION = 2.0
_NAME = u"Vantage Vue Decoding shite"
_LOG_DIR = _SCRIPT_PATH + '/log/'
_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0])
_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log'
_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg'
influx_status_write = []
influx_host = 'localhost'
influx_port = 8086
influx_user = 'pi'
influx_pwd = 'freedavis'
status_db = 'status'
'''
--------------------------------------------------------------------------------
Set up logging - disabled, need to enable this iin future
--------------------------------------------------------------------------------
'''
_NAME = u"System stats collector"
'''
--------------------------------------------------------------------------------
Setup arguments and Options - not edited, sample shite
--------------------------------------------------------------------------------
'''
desc = u'''\
DESCRIPTION:
Vantage Vue wireless data transfer decoder, V2
consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
for wtf is going on
'''
epi = u'''\
ERROR CODES:
?
EXAMPLES:
?
'''
formatter = argparse.RawDescriptionHelpFormatter
arg_parser = argparse.ArgumentParser(description = desc,
formatter_class = formatter,
epilog = textwrap.dedent(epi))
arg_parser.add_argument('-d', '--details',
help = 'help',
action='store_true')
arg_parser.add_argument('-v', '--verbose',
help = 'help',
action='store_true')
arg_parser.add_argument('-p', '--section',
dest = 'section',
default = ['last', 'count', 'diff'],
choices = ['last', 'count', 'diff'],
nargs = '+',
type = str,
help = 'help')
arg_parser.add_argument('-s', '--snapusage',
help = 'help',
action='store_true')
args = arg_parser.parse_args()
if args.details:
_details = True
else:
_details = False
if args.verbose:
_more_details = True
else:
_more_details = False
if args.snapusage:
_SNAP_USAGE = True
else:
_SNAP_USAGE = False
try:
_sections = args.sections
except:
_sections = ['last', 'count', 'diff']
'''
--------------------------------------------------------------------------------
------------------------------------------------------------------------
Generic, standalone functions
--------------------------------------------------------------------------------
------------------------------------------------------------------------
'''
# Obvious shit, set up the client class
@@ -149,20 +92,35 @@ influx_status_client = influxdb.client.InfluxDBClient(
influx_host, influx_port, influx_user, influx_pwd, status_db
)
def create_project(conn, project):
"""
Create a new project into the projects table
:param conn:
:param project:
:return: project id
"""
sql = ''' INSERT INTO wind(name,begin_date,end_date)
VALUES(?,?,?) '''
cur = conn.cursor()
cur.execute(sql, project)
return cur.lastrowid
'''
--------------------------------------------------------------------------------
------------------------------------------------------------------------
Classes
--------------------------------------------------------------------------------
------------------------------------------------------------------------
'''
class DBwriter(object):
def __init__(self):
__name__ = "Database writer class, Influx"
__name__ = "Database writer class, SQLite"
def construct(self, connector, measurement, fields, tags):
def construct(self, DB, query):
""" Takes values in a writes them to influxdb
requires: list(connector): connector with all ticks to be written
requires: list(connector): connector with ticks to be written
at once
str(measurement): the measurement ID to be written
dict(fields): fields to be written in one tick
@@ -170,51 +128,109 @@ class DBwriter(object):
returns: list(result_connector)
"""
result_connector = connector
result_connector.append({"measurement": measurement,
"fields": fields,
"tags": tags}
)
conn = sqlite3.connect(db_file)
c = conn.cursor()
c.execute(query)
conn.commit()
conn.close()
return result_connector
class Stats(object):
def __init__(self):
__name__ = "Database writer class, SQLite"
'''
--------------------------------------------------------------------------------
------------------------------------------------------------------------
Main
--------------------------------------------------------------------------------
------------------------------------------------------------------------
'''
if '__main__':
davis_writer = DBwriter()
status_writer = DBwriter()
while True:
averaged_cpu = 0
averaged_cpu = psutil.cpu_percent()
mem_consumption = psutil.virtual_memory()[2]
disk_usage = psutil.disk_usage('/')[3]
print(psutil)
# CPU stats
for timeout in range(1,5):
for timeout in range(1,15):
if timeout == 1:
averaged_cpu = psutil.cpu_percent()
averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
mem_consumption = psutil.virtual_memory()[2]
disk_usage = psutil.disk_usage('/')[3]
if_counters = psutil.net_io_counters(pernic=True)
interfaces = if_counters.keys()
wlan_counters = if_counters["wlan0"]
wlan_sent = wlan_counters[0]
wlan_recv = wlan_counters[1]
wlan_error_in = wlan_counters[4]
wlan_error_out = wlan_counters[5]
wlan_drop_in = wlan_counters[6]
wlan_drop_out = wlan_counters[7]
eth_counters = if_counters["eth0"]
eth_sent = eth_counters[0]
eth_recv = eth_counters[1]
eth_error_in = eth_counters[4]
eth_error_out = eth_counters[5]
eth_drop_in = eth_counters[6]
eth_drop_out = eth_counters[7]
else:
averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
time.sleep(1)
# Write the whole blob into Influx DB
influx_status_write = davis_writer.construct(
influx_status_write = status_writer.construct(
influx_status_write,
"RasPI",
{"usage": float(averaged_cpu)},
{"type": "cpu"}
)
influx_status_write = davis_writer.construct(
influx_status_write = status_writer.construct(
influx_status_write,
"RasPI",
{"usage": float(mem_consumption)},
{"type": "mem"}
)
influx_status_write = davis_writer.construct(
influx_status_write = status_writer.construct(
influx_status_write,
"RasPI",
{"usage": float(disk_usage)},
{"type": "disk"}
)
print(influx_status_write)
influx_status_write = status_writer.construct(
influx_status_write,
"net",
{
"b_out": float(wlan_sent),
"b_in": float(wlan_recv),
"e_out": float(wlan_error_out),
"e_in": float(wlan_error_in),
"drop_out": float(wlan_drop_out),
"drop_in": float(wlan_drop_in),
},
{
"type": "wlan0",
"host": _hostname
}
)
influx_status_write = status_writer.construct(
influx_status_write,
"net",
{
"b_out": float(eth_sent),
"b_in": float(eth_recv),
"e_out": float(eth_error_out),
"e_in": float(eth_error_in),
"drop_out": float(eth_drop_out),
"drop_in": float(eth_drop_in),
},
{
"type": "eth0",
"host": _hostname
}
)
print("Writing values: {}".format(influx_status_write))
influx_status_client.write_points(influx_status_write)
influx_status_write = []
averaged_cpu = 0