production push, including pressure

python/davis_etl.py

@@ -50,6 +50,19 @@
       field | tag
 
 
+# 
+	TABLE vantage_vue_iss
+    ----------------------
+      t_stamp | voltage | type         | lqi      | rssi    | batt_low  
+      ------------------------------------------------------------------
+      INT     | INT     | VARCHAR(20)  | TINYINT  | TINYINT | BOOL    
+
+	CREATE TABLE raspi(
+		t_stamp INT, 
+		count INT, 
+		type VARCHAR(20), 
+		nic VARCHAR(20), 
+		host VARCHAR(50));
 
 --------------------------------------------------------------------------------
             Import libraries
@@ -84,10 +97,13 @@ sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
 '''
 _VERSION = 2.0
 _NAME = u"Vantage Vue Decoding shite"
+_LOG_DIR = _SCRIPT_PATH + '/log/'
+_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0])
+_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log'
+_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg'
 # finite loop implementation, tout for 43200 cycles
 tout = 0
 
-_SERIAL_CONSOLE = '/dev/ttyUSBdavis'
 _ABS_ZERO = 273.15
 _HEIGHT = 455
 temp = {}
@@ -105,11 +121,75 @@ influx_weather_write = []
 influx_host = 'localhost'
 influx_port = 8086
 influx_user = 'pi'
-influx_pwd = 'freedavis'
+influx_pwd = 'Ventil6996'
 weather_db = 'weather_v2'
 status_db = 'status'
 
+'''
+--------------------------------------------------------------------------------
+            Set up logging - disabled, need to enable this iin future
+--------------------------------------------------------------------------------
+'''
 
+
+'''
+--------------------------------------------------------------------------------
+            Setup arguments and Options - not edited, sample shite
+--------------------------------------------------------------------------------
+'''
+desc = u'''\
+DESCRIPTION:
+    Vantage Vue wireless data transfer decoder, V2
+    consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
+    for wtf is going on
+    '''
+epi = u'''\
+    ERROR CODES:
+        ?
+
+    EXAMPLES:
+        ?
+
+    '''
+formatter = argparse.RawDescriptionHelpFormatter
+arg_parser = argparse.ArgumentParser(description = desc,
+                                     formatter_class = formatter,
+                                     epilog = textwrap.dedent(epi))
+
+arg_parser.add_argument('-d', '--details',
+                        help = 'help',
+                        action='store_true')
+arg_parser.add_argument('-v', '--verbose',
+                        help = 'help',
+                        action='store_true')
+arg_parser.add_argument('-p', '--section',
+                        dest = 'section',
+                        default = ['last', 'count', 'diff'],
+                        choices = ['last', 'count', 'diff'],
+                        nargs = '+',
+                        type = str,
+                        help = 'help')
+arg_parser.add_argument('-s', '--snapusage',
+                        help = 'help',
+                        action='store_true')
+
+args = arg_parser.parse_args()
+if args.details:
+    _details = True
+else:
+    _details = False
+if args.verbose:
+    _more_details = True
+else:
+    _more_details = False
+if args.snapusage:
+    _SNAP_USAGE = True
+else:
+    _SNAP_USAGE = False
+try:
+    _sections = args.sections
+except:
+    _sections = ['last', 'count', 'diff']
 '''
 --------------------------------------------------------------------------------
             Generic, standalone functions
@@ -124,6 +204,32 @@ influx_status_client = influxdb.client.InfluxDBClient(
     influx_host, influx_port, influx_user, influx_pwd, status_db
 )
 
+def create_connection(db_file):
+    """ create a database connection to the SQLite database
+        specified by db_file
+    :param db_file: database file
+    :return: Connection object or None
+    """
+    try:
+        conn = sqlite3.connect(db_file)
+        return conn
+    except Error as e:
+        print(e)
+    return None
+    
+def create_project(conn, project):
+    """
+    Create a new project into the projects table
+    :param conn:
+    :param project:
+    :return: project id
+    """
+    sql = ''' INSERT INTO wind(name,begin_date,end_date)
+              VALUES(?,?,?) '''
+    cur = conn.cursor()
+    cur.execute(sql, project)
+    return cur.lastrowid
+
 '''
 --------------------------------------------------------------------------------
             Classes
@@ -136,6 +242,13 @@ class davisDecoder(object):
         self.height = _HEIGHT
         self.temp_dict = {}
 
+    def load_external_data(self):
+        # Data external to the ISS
+        self.pressure = float(davis_data['P'])
+        self.inside_temp = round((float(davis_data['Ti'])\
+                         + float(davis_data['Thtu'])) / 2, 2)
+        self.inside_hum = davis_data['Hhtu']
+
     def zero_fill(self, data):
         binary_data = format(int(data), '08b')
         msb = binary_data[0:4]
@@ -172,6 +285,12 @@ class davisDecoder(object):
     def decode_humidity(self, hum):
         pass
 
+    def adjust_pressure(self, temp):
+        sh = 0.0065 * self.height
+        base = 1 - (sh) / (temp + sh + _ABS_ZERO)
+        result = round(self.pressure * pow(base, -5.257), 2)
+        return result
+
     def supercap_decode(self, byte2, byte3):
         cap = (byte2 << 2) + (byte3 >> 6)
         result = float(cap / 100.00)
@@ -189,15 +308,12 @@ class davisDecoder(object):
     def rainrate_decode(self, byte2, byte3):
         # if byte3(b2 here) is 0xFF, or 255, there is not rain
         print("b2:{} b3:{} = result:{}".format(byte2, byte3, byte2 + (byte3 >> 4 << 8)))
-        # No Rain rainstate = 0
         if byte2 == 255:
             rainstate = 0
             rainrate = 0
-        # Light Rain rainstate = 1
         elif byte2 == 254:
             rainstate = 1
             rainrate = 0.1
-        # Heavy rain rainstate = 2
         else:
             rainstate = 2
             if byte3 > 4:
@@ -254,6 +370,14 @@ class DBwriter(object):
                 "measurement": "wind",
                 "fields": { "value": base_value_dict['direction'] },
                 "tags": { "type": "direction" }
+            },
+            {
+                "measurement": "temphumi",
+                "fields": {
+                    "temperature": base_value_dict['temperature'],
+                    "humidity": base_value_dict['humidity']
+                },
+                "tags": { "type": "internal" }
             }]
         return base_connector
 
@@ -285,7 +409,7 @@ if '__main__':
     davis_decoder = davisDecoder()
     davis_writer = DBwriter()
     try:
-        with serial.Serial(_SERIAL_CONSOLE, 9600) as davis:
+        with serial.Serial('/dev/ttyUSBdavis', 9600) as davis:
             # Now, let it run a couple times, end and restart via systemd
             while tout < 400:
                 line = davis.readline()
@@ -311,10 +435,14 @@ if '__main__':
                     raw_winddir = davis_data['b1']
                     wind = davis_decoder.decode_wind({"windspeed": raw_windspeed,
                                                     "winddir": raw_winddir})
-                    print(wind)
+                    # Get data external to the ISS, from local PCB / internal
+                    # sensors and create base values for influx writing
+                    davis_decoder.load_external_data()
                     influx_weather_write = davis_writer.base_construct(
                         { "speed": float(wind['speed']),
-                        "direction": float(wind['direction'])}
+                        "direction": float(wind['direction']),
+                        "temperature": float(davis_decoder.inside_temp),
+                        "humidity": float(davis_decoder.inside_hum)}
                     )
 
                     # Wind gusts calculation
@@ -333,6 +461,19 @@ if '__main__':
                         raw_temp = (davis_data['b2'] << 8) + davis_data['b3']
                         temp_dict = davis_decoder.decode_temp(raw_temp)
                         temp = float(temp_dict['celsius'])
+                        pressure_adjusted = davis_decoder.adjust_pressure(temp)
+                        influx_weather_write = davis_writer.construct(
+                            influx_weather_write,
+                            "temphumi",
+                            {"pressure": float(pressure_adjusted)},
+                            {"type" : "adjusted"}
+                        )
+                        influx_weather_write = davis_writer.construct(
+                            influx_weather_write,
+                            "temphumi",
+                            {"pressure": float(davis_decoder.pressure)},
+                            {"type" : "raw"}
+                        )
 
                         influx_weather_write = davis_writer.construct(
                             influx_weather_write,
@@ -362,7 +503,7 @@ if '__main__':
                             influx_status_write,
                             "iss",
                             {"voltage": float(supercap)},
-                            {"type": "capcaitor"}
+                            {"type": "capacitor"}
                         )
 
                     # 0x7 -> SolarPanel Voltage
@@ -420,15 +561,22 @@ if '__main__':
                                 temp,
                                 wind,
                                 humidity)
-
+                    out2 = "Padj: {}, Praw {}, Tins: {}, Humins: {}".format(
+                                pressure_adjusted,
+                                davis_decoder.pressure,
+                                davis_decoder.inside_temp,
+                                davis_decoder.inside_hum)
                     out3 = "RainState: {}, Rrate {}, Rain Total: {}, Cap:{}, Volt: {}".format(
                                 rainstate,
                                 rainrate,
                                 rain,
                                 supercap,
                                 solarvolt)
-
+                    out4 = {'RAW': davis_data}
-                    print("\n{} \n{} \n{}\n".format(out_id, out1, out3))
+                    with open('/home/pi/davis.rawdata', 'a+') as fh:
+                        now = datetime.datetime.strftime(datetime.datetime.now(), "%s")
+                        fh.write(now + ";" + str(out4) + "\n")
+                    #print("\n{} \n{} \n{} \n{}\n{}\n".format(out_id, out1, out2, out3, out4))
 
                     # Write the whole blob into Influx DB
                     influx_weather_client.write_points(influx_weather_write)

python/sysstats.py

@@ -1,35 +1,46 @@
 #!/usr/bin/python -u
 
 '''
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
     Type:           Python 3.x script
     Author:         Milan Toman (milan.v.toman@gmail.com)
     Description:    System stats
+    Version:        2.0 (SQLite)
 
-    TODO:           CPU util.
+    SQLite SCHEMA:
-
-
-    influxDB SCHEMA:
 
     DB status
 
-    ISS measure
+    TABLE raspi
     ----------------
-      voltage | solar or capacitor | state / lqi /  | battery or future_shit |
+      t_stamp | usage | host         | type
-      ----------------------------------------------------------------
+      -----------------------------------------------
-      field     tag                  field   tag
+      INT     | INT   | VARCHAR(50)  | VARCHAR(20)
 
-    RasPI
+	CREATE TABLE raspi(
+		t_stamp INT, 
+		usage INT, 
+		host VARCHAR(50), 
+		type VARCHAR(20));
+
+    TABLE network
     ----------------
-      usage | disk, mem, cpu, eth, wifi %
+      t_stamp | count | type         | nic         | host
-      ------------------------------------
+      ------------------------------------------------------------
-      field | tag
+      INT     | INT   | VARCHAR(20)  | VARCHAR(20) | VARCHAR(50)
+
+	CREATE TABLE network(
+		t_stamp INT, 
+		count INT, 
+		type VARCHAR(20), 
+		nic VARCHAR(20), 
+		host VARCHAR(50));
 
 
 
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
             Import libraries
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
 '''
 # mandatory
 import requests
@@ -41,107 +52,39 @@ import argparse
 import time
 import datetime
 import simplejson as json
-import influxdb
+import sqlite3
 import psutil
+import socket
 
 # optionally, future modules, locally available, I hate dependencies
 from pprint import pprint
 _SCRIPT_PATH = os.path.dirname(sys.argv[0])
-sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
+#sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
 #print(_SCRIPT_PATH + "/lib")
 
-#import ventilLogger
+global _hostname
-
+_hostname = socket.gethostname()
 
 '''
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
+            SQLite def
+------------------------------------------------------------------------
+'''
+_SQLiteDB = '/var/lib/plutonium/status.db'
+
+'''
+------------------------------------------------------------------------
             Define variables
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
 '''
 _VERSION = 2.0
-_NAME = u"Vantage Vue Decoding shite"
+_NAME = u"System stats collector"
-_LOG_DIR = _SCRIPT_PATH + '/log/'
-_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0])
-_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log'
-_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg'
-
-influx_status_write = []
-
-influx_host = 'localhost'
-influx_port = 8086
-influx_user = 'pi'
-influx_pwd = 'freedavis'
-status_db = 'status'
-
-'''
---------------------------------------------------------------------------------
-            Set up logging - disabled, need to enable this iin future
---------------------------------------------------------------------------------
-'''
 
 
 '''
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
-            Setup arguments and Options - not edited, sample shite
---------------------------------------------------------------------------------
-'''
-desc = u'''\
-DESCRIPTION:
-    Vantage Vue wireless data transfer decoder, V2
-    consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
-    for wtf is going on
-    '''
-epi = u'''\
-    ERROR CODES:
-        ?
-
-    EXAMPLES:
-        ?
-
-    '''
-formatter = argparse.RawDescriptionHelpFormatter
-arg_parser = argparse.ArgumentParser(description = desc,
-                                     formatter_class = formatter,
-                                     epilog = textwrap.dedent(epi))
-
-arg_parser.add_argument('-d', '--details',
-                        help = 'help',
-                        action='store_true')
-arg_parser.add_argument('-v', '--verbose',
-                        help = 'help',
-                        action='store_true')
-arg_parser.add_argument('-p', '--section',
-                        dest = 'section',
-                        default = ['last', 'count', 'diff'],
-                        choices = ['last', 'count', 'diff'],
-                        nargs = '+',
-                        type = str,
-                        help = 'help')
-arg_parser.add_argument('-s', '--snapusage',
-                        help = 'help',
-                        action='store_true')
-
-args = arg_parser.parse_args()
-if args.details:
-    _details = True
-else:
-    _details = False
-if args.verbose:
-    _more_details = True
-else:
-    _more_details = False
-if args.snapusage:
-    _SNAP_USAGE = True
-else:
-    _SNAP_USAGE = False
-try:
-    _sections = args.sections
-except:
-    _sections = ['last', 'count', 'diff']
-'''
---------------------------------------------------------------------------------
             Generic, standalone functions
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
 '''
 
 # Obvious shit, set up the client class
@@ -149,20 +92,35 @@ influx_status_client = influxdb.client.InfluxDBClient(
     influx_host, influx_port, influx_user, influx_pwd, status_db
 )
 
+
+    
+def create_project(conn, project):
+    """
+    Create a new project into the projects table
+    :param conn:
+    :param project:
+    :return: project id
+    """
+    sql = ''' INSERT INTO wind(name,begin_date,end_date)
+              VALUES(?,?,?) '''
+    cur = conn.cursor()
+    cur.execute(sql, project)
+    return cur.lastrowid
+
 '''
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
             Classes
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
 '''
 
 class DBwriter(object):
     def __init__(self):
-        __name__ = "Database writer class, Influx"
+        __name__ = "Database writer class, SQLite"    
 
-    def construct(self, connector, measurement, fields, tags):
+    def construct(self, DB, query):
         """ Takes values in a writes them to influxdb
 
-        requires:   list(connector): connector with all ticks to be written
+        requires:   list(connector): connector with ticks to be written
                                      at once
                     str(measurement): the measurement ID to be written
                     dict(fields): fields to be written in one tick
@@ -170,51 +128,109 @@ class DBwriter(object):
 
         returns:    list(result_connector)
         """
-        result_connector = connector
+        conn = sqlite3.connect(db_file)
-        result_connector.append({"measurement": measurement,
+        c = conn.cursor()
-                                 "fields": fields,
+        c.execute(query)        
-                                 "tags": tags}
+        conn.commit()
-                               )
+        conn.close()
         return result_connector
 
+class Stats(object):
+    def __init__(self):
+        __name__ = "Database writer class, SQLite"
+
+
+
 '''
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
             Main
---------------------------------------------------------------------------------
+------------------------------------------------------------------------
 '''
 if '__main__':
-    davis_writer = DBwriter()
+    status_writer = DBwriter()
     while True:
-        averaged_cpu = 0
+        averaged_cpu = psutil.cpu_percent()
+        mem_consumption = psutil.virtual_memory()[2]
+        disk_usage = psutil.disk_usage('/')[3]
+        print(psutil)
         # CPU stats
-        for timeout in range(1,5):
+        for timeout in range(1,15):
             if timeout == 1:
-                averaged_cpu = psutil.cpu_percent()
+                averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
                 mem_consumption = psutil.virtual_memory()[2]
                 disk_usage = psutil.disk_usage('/')[3]
+                if_counters = psutil.net_io_counters(pernic=True)
+                interfaces = if_counters.keys()
+
+                wlan_counters = if_counters["wlan0"]
+                wlan_sent = wlan_counters[0]
+                wlan_recv = wlan_counters[1]
+                wlan_error_in = wlan_counters[4]
+                wlan_error_out = wlan_counters[5]
+                wlan_drop_in = wlan_counters[6]
+                wlan_drop_out = wlan_counters[7]
+                eth_counters = if_counters["eth0"]
+                eth_sent = eth_counters[0]
+                eth_recv = eth_counters[1]
+                eth_error_in = eth_counters[4]
+                eth_error_out = eth_counters[5]
+                eth_drop_in = eth_counters[6]
+                eth_drop_out = eth_counters[7]
             else:
                 averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
             time.sleep(1)
         # Write the whole blob into Influx DB
-        influx_status_write = davis_writer.construct(
+        influx_status_write = status_writer.construct(
                 influx_status_write,
                 "RasPI",
                 {"usage": float(averaged_cpu)},
                 {"type": "cpu"}
             )
-        influx_status_write = davis_writer.construct(
+        influx_status_write = status_writer.construct(
                 influx_status_write,
                 "RasPI",
                 {"usage": float(mem_consumption)},
                 {"type": "mem"}
             )
-        influx_status_write = davis_writer.construct(
+        influx_status_write = status_writer.construct(
                 influx_status_write,
                 "RasPI",
                 {"usage": float(disk_usage)},
                 {"type": "disk"}
             )
-        print(influx_status_write)
+        influx_status_write = status_writer.construct(
+                influx_status_write,
+                "net",
+                {
+                    "b_out": float(wlan_sent),
+                    "b_in": float(wlan_recv),
+                    "e_out": float(wlan_error_out),
+                    "e_in": float(wlan_error_in),
+                    "drop_out": float(wlan_drop_out),
+                    "drop_in": float(wlan_drop_in),
+                },
+                {
+                    "type": "wlan0",
+                    "host": _hostname
+                }
+            )
+        influx_status_write = status_writer.construct(
+                influx_status_write,
+                "net",
+                {
+                    "b_out": float(eth_sent),
+                    "b_in": float(eth_recv),
+                    "e_out": float(eth_error_out),
+                    "e_in": float(eth_error_in),
+                    "drop_out": float(eth_drop_out),
+                    "drop_in": float(eth_drop_in),
+                },
+                {
+                    "type": "eth0",
+                    "host": _hostname
+                }
+            )
+        print("Writing values: {}".format(influx_status_write))
         influx_status_client.write_points(influx_status_write)
         influx_status_write = []
         averaged_cpu = 0