New version, completely
This commit is contained in:
589
python/davis_etl.py
Executable file
589
python/davis_etl.py
Executable file
@@ -0,0 +1,589 @@
|
||||
#!/usr/bin/python -u
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Type: Python 3.x script
|
||||
Author: Milan Toman (milan.v.toman@gmail.com)
|
||||
Description: Weather station (davis vantage vue) collector, coupled with
|
||||
the infamous arduino source on
|
||||
http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
|
||||
en-fucking-joy
|
||||
|
||||
TODO: Rainrate going bonkers, check that shite.
|
||||
|
||||
|
||||
influxDB SCHEMA:
|
||||
|
||||
DB weather
|
||||
measure wind
|
||||
----------------
|
||||
value | speed or direction or windgust
|
||||
---------------------------------------
|
||||
field tag
|
||||
|
||||
measure temphumi
|
||||
----------------
|
||||
temperature | humidity | external, internal | pressure
|
||||
---------------------------------------------------------
|
||||
field field tag field
|
||||
|
||||
measure rain
|
||||
----------------
|
||||
rain | rate / total / intensity | restart if zero, was it 65535 => before?
|
||||
---------------------------------------------
|
||||
field tag field(int)
|
||||
|
||||
|
||||
|
||||
DB status
|
||||
|
||||
ISS measure
|
||||
----------------
|
||||
voltage | solar or capacitor | state / lqi / | battery or future_shit |
|
||||
----------------------------------------------------------------
|
||||
field tag field tag
|
||||
|
||||
RasPI system
|
||||
----------------
|
||||
usage | disk, mem, cpu, eth, wifi %
|
||||
------------------------------------
|
||||
field | tag
|
||||
|
||||
|
||||
#
|
||||
TABLE vantage_vue_iss
|
||||
----------------------
|
||||
t_stamp | voltage | type | lqi | rssi | batt_low
|
||||
------------------------------------------------------------------
|
||||
INT | INT | VARCHAR(20) | TINYINT | TINYINT | BOOL
|
||||
|
||||
CREATE TABLE raspi(
|
||||
t_stamp INT,
|
||||
count INT,
|
||||
type VARCHAR(20),
|
||||
nic VARCHAR(20),
|
||||
host VARCHAR(50));
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Import libraries
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
# mandatory
|
||||
import requests
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import textwrap
|
||||
import argparse
|
||||
import time
|
||||
import datetime
|
||||
import serial
|
||||
import simplejson as json
|
||||
import influxdb
|
||||
|
||||
# optionally, future modules, locally available, I hate dependencies
|
||||
from pprint import pprint
|
||||
_SCRIPT_PATH = os.path.dirname(sys.argv[0])
|
||||
sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
|
||||
#print(_SCRIPT_PATH + "/lib")
|
||||
|
||||
#import ventilLogger
|
||||
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Define variables
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
_VERSION = 2.0
|
||||
_NAME = u"Vantage Vue Decoding shite"
|
||||
_LOG_DIR = _SCRIPT_PATH + '/log/'
|
||||
_LOG_FILE_ROOT = re.sub(u'./', '', sys.argv[0])
|
||||
_LOG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.log'
|
||||
_DEBUG_FILE = _LOG_DIR + _LOG_FILE_ROOT + u'.dbg'
|
||||
# finite loop implementation, tout for 43200 cycles
|
||||
tout = 0
|
||||
|
||||
_ABS_ZERO = 273.15
|
||||
_HEIGHT = 455
|
||||
temp = {}
|
||||
wind = {}
|
||||
humidity = 0
|
||||
pressure_adjusted = 0
|
||||
supercap = 0
|
||||
solarvolt = 0
|
||||
rainstate = 0
|
||||
rainrate = 0
|
||||
rain = 0
|
||||
influx_status_write = []
|
||||
influx_weather_write = []
|
||||
|
||||
influx_host = 'localhost'
|
||||
influx_port = 8086
|
||||
influx_user = 'pi'
|
||||
influx_pwd = 'Ventil6996'
|
||||
weather_db = 'weather_v2'
|
||||
status_db = 'status'
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Set up logging - disabled, need to enable this iin future
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Setup arguments and Options - not edited, sample shite
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
desc = u'''\
|
||||
DESCRIPTION:
|
||||
Vantage Vue wireless data transfer decoder, V2
|
||||
consult http://wp.spoton.cz/2017/11/24/davis-vantague-vue-arduino-and-a-raspberry-pi-3/
|
||||
for wtf is going on
|
||||
'''
|
||||
epi = u'''\
|
||||
ERROR CODES:
|
||||
?
|
||||
|
||||
EXAMPLES:
|
||||
?
|
||||
|
||||
'''
|
||||
formatter = argparse.RawDescriptionHelpFormatter
|
||||
arg_parser = argparse.ArgumentParser(description = desc,
|
||||
formatter_class = formatter,
|
||||
epilog = textwrap.dedent(epi))
|
||||
|
||||
arg_parser.add_argument('-d', '--details',
|
||||
help = 'help',
|
||||
action='store_true')
|
||||
arg_parser.add_argument('-v', '--verbose',
|
||||
help = 'help',
|
||||
action='store_true')
|
||||
arg_parser.add_argument('-p', '--section',
|
||||
dest = 'section',
|
||||
default = ['last', 'count', 'diff'],
|
||||
choices = ['last', 'count', 'diff'],
|
||||
nargs = '+',
|
||||
type = str,
|
||||
help = 'help')
|
||||
arg_parser.add_argument('-s', '--snapusage',
|
||||
help = 'help',
|
||||
action='store_true')
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
if args.details:
|
||||
_details = True
|
||||
else:
|
||||
_details = False
|
||||
if args.verbose:
|
||||
_more_details = True
|
||||
else:
|
||||
_more_details = False
|
||||
if args.snapusage:
|
||||
_SNAP_USAGE = True
|
||||
else:
|
||||
_SNAP_USAGE = False
|
||||
try:
|
||||
_sections = args.sections
|
||||
except:
|
||||
_sections = ['last', 'count', 'diff']
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Generic, standalone functions
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
# Obvious shit, set up the client class
|
||||
influx_weather_client = influxdb.client.InfluxDBClient(
|
||||
influx_host, influx_port, influx_user, influx_pwd, weather_db
|
||||
)
|
||||
influx_status_client = influxdb.client.InfluxDBClient(
|
||||
influx_host, influx_port, influx_user, influx_pwd, status_db
|
||||
)
|
||||
|
||||
def create_connection(db_file):
|
||||
""" create a database connection to the SQLite database
|
||||
specified by db_file
|
||||
:param db_file: database file
|
||||
:return: Connection object or None
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_file)
|
||||
return conn
|
||||
except Error as e:
|
||||
print(e)
|
||||
return None
|
||||
|
||||
def create_project(conn, project):
|
||||
"""
|
||||
Create a new project into the projects table
|
||||
:param conn:
|
||||
:param project:
|
||||
:return: project id
|
||||
"""
|
||||
sql = ''' INSERT INTO wind(name,begin_date,end_date)
|
||||
VALUES(?,?,?) '''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, project)
|
||||
return cur.lastrowid
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Classes
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
class davisDecoder(object):
|
||||
def __init__(self):
|
||||
__name__ = u'Davis value decoder class'
|
||||
self.height = _HEIGHT
|
||||
self.temp_dict = {}
|
||||
|
||||
def load_external_data(self):
|
||||
# Data external to the ISS
|
||||
self.pressure = float(davis_data['P'])
|
||||
self.inside_temp = round((float(davis_data['Ti'])\
|
||||
+ float(davis_data['Thtu'])) / 2, 2)
|
||||
self.inside_hum = davis_data['Hhtu']
|
||||
|
||||
def zero_fill(self, data):
|
||||
binary_data = format(int(data), '08b')
|
||||
msb = binary_data[0:4]
|
||||
lsb = binary_data[4:]
|
||||
result = {"MSB": msb, "LSB": lsb}
|
||||
return result
|
||||
|
||||
def davis_id(self, header):
|
||||
bin_header = self.zero_fill(header)
|
||||
davis_id = hex(int(bin_header['LSB'][1:], 2))
|
||||
raw_id = bin_header['MSB']
|
||||
battery_low = bin_header['LSB'][0]
|
||||
davis_packet_id = hex(int(raw_id, 2))
|
||||
result = {"davis_id": davis_id,
|
||||
"packet_id": davis_packet_id,
|
||||
"bat_low": battery_low}
|
||||
return result
|
||||
|
||||
def decode_wind(self, databytes):
|
||||
# wind speed in mph, i suppose. Let's convert it
|
||||
wind_speed = round(float(databytes['windspeed'] * 1.60934), 1)
|
||||
wind_direction_factor = round(float(360)/float(255), 1)
|
||||
wind_direction = databytes['winddir']
|
||||
wind_direction = float(wind_direction) * wind_direction_factor
|
||||
result = {"speed": wind_speed, "direction": wind_direction}
|
||||
return result
|
||||
|
||||
def decode_temp(self, temp):
|
||||
temp_f = (float(temp)) / float(160) # in Fahrenheit
|
||||
temp_c = round((temp_f - 32) * float(5)/float(9), 1)
|
||||
result = {"celsius": temp_c, "fahrenheit": temp_f}
|
||||
return result
|
||||
|
||||
def decode_humidity(self, hum):
|
||||
pass
|
||||
|
||||
def adjust_pressure(self, temp):
|
||||
sh = 0.0065 * self.height
|
||||
base = 1 - (sh) / (temp + sh + _ABS_ZERO)
|
||||
result = round(self.pressure * pow(base, -5.257), 2)
|
||||
return result
|
||||
|
||||
def supercap_decode(self, byte2, byte3):
|
||||
cap = (byte2 << 2) + (byte3 >> 6)
|
||||
result = float(cap / 100.00)
|
||||
return result
|
||||
|
||||
def solarvolt_decode(self, byte2, byte3):
|
||||
solar = (byte2 << 1) + (byte3 >> 7)
|
||||
result = float(solar)
|
||||
return result
|
||||
|
||||
def rain_decode(self, rain):
|
||||
result = float(rain & 0x7F)
|
||||
return result
|
||||
|
||||
def rainrate_decode(self, byte2, byte3):
|
||||
# if byte3(b2 here) is 0xFF, or 255, there is not rain
|
||||
print("b2:{} b3:{} = result:{}".format(byte2, byte3, byte2 + (byte3 >> 4 << 8)))
|
||||
if byte2 == 255:
|
||||
rainstate = 0
|
||||
rainrate = 0
|
||||
elif byte2 == 254:
|
||||
rainstate = 1
|
||||
rainrate = 0.1
|
||||
else:
|
||||
rainstate = 2
|
||||
if byte3 > 4:
|
||||
rainrate = 720 / ((byte3 >> 4 << 8) + byte2)
|
||||
else:
|
||||
rainrate = 0
|
||||
|
||||
result = {"state": float(rainstate), "rate": float(rainrate)}
|
||||
print(result)
|
||||
return result
|
||||
|
||||
class DBwriter(object):
|
||||
def __init__(self):
|
||||
__name__ = "Database writer class, Influx"
|
||||
|
||||
def construct(self, connector, measurement, fields, tags):
|
||||
""" Takes values in a writes them to influxdb
|
||||
|
||||
requires: list(connector): connector with all ticks to be written
|
||||
at once
|
||||
str(measurement): the measurement ID to be written
|
||||
dict(fields): fields to be written in one tick
|
||||
dict(tags): tags to be written with the fields
|
||||
|
||||
returns: list(result_connector)
|
||||
"""
|
||||
result_connector = connector
|
||||
result_connector.append({"measurement": measurement,
|
||||
"fields": fields,
|
||||
"tags": tags}
|
||||
)
|
||||
return result_connector
|
||||
|
||||
def base_construct(self, base_value_dict):
|
||||
""" Takes values in a writes them to influxdb
|
||||
|
||||
requires: dict(base_value_dict):
|
||||
{ "speed": float(),
|
||||
"direction": float(),
|
||||
"temperature": float(),
|
||||
"humidity": float()}
|
||||
base weather values being sent each time.
|
||||
Wind dir / strength, internal temp / humi
|
||||
|
||||
returns: list(base_connector)
|
||||
"""
|
||||
base_connector = [
|
||||
{
|
||||
"measurement": "wind",
|
||||
"fields": { "value": base_value_dict['speed'] },
|
||||
"tags": { "type": "speed" }
|
||||
},
|
||||
{
|
||||
"measurement": "wind",
|
||||
"fields": { "value": base_value_dict['direction'] },
|
||||
"tags": { "type": "direction" }
|
||||
},
|
||||
{
|
||||
"measurement": "temphumi",
|
||||
"fields": {
|
||||
"temperature": base_value_dict['temperature'],
|
||||
"humidity": base_value_dict['humidity']
|
||||
},
|
||||
"tags": { "type": "internal" }
|
||||
}]
|
||||
return base_connector
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Main
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
if '__main__':
|
||||
'''
|
||||
2 = Supercap voltage (Vue only)
|
||||
3 = ?
|
||||
4 = UV Index
|
||||
5 = Rain rate
|
||||
6 = Solar radiation
|
||||
7 = Solar Cell output (Vue only)
|
||||
8 = Temperature
|
||||
9 = Wind gust
|
||||
a = Humidity
|
||||
e = Rain
|
||||
|
||||
{'nxt': 64, 'P0': 1020.43, 'lqi': 6, 'b2': 3, 'P': 969.29, 'h': 144, 'Ti': 24.49,
|
||||
'cnt': 1, 'Hhtu': 28.68, 'b4': 129, 'b5': 247, 'b6': 36, 'b7': 255, 'b0': 1, 'b1': 10,
|
||||
'hop': 0, 'b3': 225, 'Thtu': 24.28, 'b8': 255, 'b9': 182, 'rssi': 45}
|
||||
|
||||
'''
|
||||
# TODO, make it work for any USB to serial port
|
||||
|
||||
davis_decoder = davisDecoder()
|
||||
davis_writer = DBwriter()
|
||||
try:
|
||||
with serial.Serial('/dev/ttyUSBdavis', 9600) as davis:
|
||||
# Now, let it run a couple times, end and restart via systemd
|
||||
while tout < 400:
|
||||
line = davis.readline()
|
||||
#print(line)
|
||||
davis_data = 0
|
||||
try:
|
||||
davis_data = eval(line)
|
||||
except SyntaxError as e_syntax:
|
||||
print("ERROR (syntax): {}".format(e_syntax))
|
||||
except TypeError as e_type:
|
||||
print("ERROR (Type): {}".format(e_type))
|
||||
except ValueError as e_value:
|
||||
print("ERROR (Type): {}".format(e_value))
|
||||
if davis_data != 0:
|
||||
# Raw data with every tick
|
||||
raw_header = davis_data['h']
|
||||
decoded_header = davis_decoder.davis_id(raw_header)
|
||||
davis_unit_id = decoded_header['davis_id']
|
||||
davis_packet_id = decoded_header['packet_id']
|
||||
|
||||
# Wind, mothafucka!
|
||||
raw_windspeed = davis_data['b0']
|
||||
raw_winddir = davis_data['b1']
|
||||
wind = davis_decoder.decode_wind({"windspeed": raw_windspeed,
|
||||
"winddir": raw_winddir})
|
||||
# Get data external to the ISS, from local PCB / internal
|
||||
# sensors and create base values for influx writing
|
||||
davis_decoder.load_external_data()
|
||||
influx_weather_write = davis_writer.base_construct(
|
||||
{ "speed": float(wind['speed']),
|
||||
"direction": float(wind['direction']),
|
||||
"temperature": float(davis_decoder.inside_temp),
|
||||
"humidity": float(davis_decoder.inside_hum)}
|
||||
)
|
||||
|
||||
# Wind gusts calculation
|
||||
if davis_packet_id == '0x9':
|
||||
raw_gusts = davis_data['b2']
|
||||
wind.update({"windgust": (raw_gusts * 1.60934)})
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"wind",
|
||||
{"value": float(wind['windgust'])},
|
||||
{"type": "windgust"}
|
||||
)
|
||||
|
||||
# 0x8 -> temperature
|
||||
if davis_packet_id == '0x8':
|
||||
raw_temp = (davis_data['b2'] << 8) + davis_data['b3']
|
||||
temp_dict = davis_decoder.decode_temp(raw_temp)
|
||||
temp = float(temp_dict['celsius'])
|
||||
pressure_adjusted = davis_decoder.adjust_pressure(temp)
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"temphumi",
|
||||
{"pressure": float(pressure_adjusted)},
|
||||
{"type" : "adjusted"}
|
||||
)
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"temphumi",
|
||||
{"pressure": float(davis_decoder.pressure)},
|
||||
{"type" : "raw"}
|
||||
)
|
||||
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"temphumi",
|
||||
{"temperature": float(temp)},
|
||||
{"type": "external"}
|
||||
)
|
||||
|
||||
# 0xa -> Humidity
|
||||
if davis_packet_id == '0xa':
|
||||
raw_humidity = (((davis_data['b3'] >> 4) & 0b0011) << 8)\
|
||||
+ davis_data['b2']
|
||||
humidity = round(int(raw_humidity) / float(10), 1)
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"temphumi",
|
||||
{"humidity": float(humidity)},
|
||||
{"type": "external"}
|
||||
)
|
||||
|
||||
# 0x2 -> SuperCap charge
|
||||
if davis_packet_id == '0x2':
|
||||
supercap = davis_decoder.supercap_decode(
|
||||
davis_data['b2'], davis_data['b3']
|
||||
)
|
||||
influx_status_write = davis_writer.construct(
|
||||
influx_status_write,
|
||||
"iss",
|
||||
{"voltage": float(supercap)},
|
||||
{"type": "capacitor"}
|
||||
)
|
||||
|
||||
# 0x7 -> SolarPanel Voltage
|
||||
if davis_packet_id == '0x7':
|
||||
solarvolt = davis_decoder.solarvolt_decode(
|
||||
davis_data['b2'], davis_data['b3']
|
||||
)
|
||||
influx_status_write = davis_writer.construct(
|
||||
influx_status_write,
|
||||
"iss",
|
||||
{"voltage": float(solarvolt)},
|
||||
{"type": "solar"}
|
||||
)
|
||||
|
||||
# 0xe -> Rain bucket tips -> https://www.carluccio.de/
|
||||
if davis_packet_id == '0xe':
|
||||
raw_rain = (davis_data['b2']) + (davis_data['b3'] >> 7 << 8)
|
||||
#raw_rain = davis_data['b2']
|
||||
rain = davis_decoder.rain_decode(raw_rain)
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"rain",
|
||||
{"value": float(rain)},
|
||||
{"type": "rain_bucket_tips"}
|
||||
)
|
||||
|
||||
# 0x5 -> Rain rate -> https://www.carluccio.de/
|
||||
if davis_packet_id == '0x5':
|
||||
rainrate_dict = davis_decoder.rainrate_decode(
|
||||
davis_data['b2'],
|
||||
davis_data['b3']
|
||||
)
|
||||
if rainrate_dict:
|
||||
rainstate = rainrate_dict['state']
|
||||
rainrate = rainrate_dict['rate']
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"rain",
|
||||
{"value": float(rainrate_dict['state'])},
|
||||
{"type": "rainstate"}
|
||||
)
|
||||
influx_weather_write = davis_writer.construct(
|
||||
influx_weather_write,
|
||||
"rain",
|
||||
{"value": float(rainrate_dict['rate'])},
|
||||
{"type": "rainrate"}
|
||||
)
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
out_id = decoded_header
|
||||
out1 = "temp: {}, wind: {}, humidity: {} ".format(
|
||||
temp,
|
||||
wind,
|
||||
humidity)
|
||||
out2 = "Padj: {}, Praw {}, Tins: {}, Humins: {}".format(
|
||||
pressure_adjusted,
|
||||
davis_decoder.pressure,
|
||||
davis_decoder.inside_temp,
|
||||
davis_decoder.inside_hum)
|
||||
out3 = "RainState: {}, Rrate {}, Rain Total: {}, Cap:{}, Volt: {}".format(
|
||||
rainstate,
|
||||
rainrate,
|
||||
rain,
|
||||
supercap,
|
||||
solarvolt)
|
||||
out4 = {'RAW': davis_data}
|
||||
with open('/home/pi/davis.rawdata', 'a+') as fh:
|
||||
now = datetime.datetime.strftime(datetime.datetime.now(), "%s")
|
||||
fh.write(now + ";" + str(out4) + "\n")
|
||||
#print("\n{} \n{} \n{} \n{}\n{}\n".format(out_id, out1, out2, out3, out4))
|
||||
|
||||
# Write the whole blob into Influx DB
|
||||
influx_weather_client.write_points(influx_weather_write)
|
||||
influx_status_client.write_points(influx_status_write)
|
||||
tout = tout + 1
|
||||
time.sleep(1)
|
||||
else:
|
||||
print("No data here, mate.")
|
||||
except serial.serialutil.SerialException as e:
|
||||
print("Serial Error {}".format(e))
|
||||
120
python/lib/ventilLogger.py
Normal file
120
python/lib/ventilLogger.py
Normal file
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/python -u
|
||||
|
||||
'''
|
||||
--------------------------------------------------------------------------------
|
||||
Type: Python 3.x module
|
||||
Author: Milan Toman (milan.v.toman@gmail.com)
|
||||
Description: Logger module, writes debug, logs and warning into ./log dir
|
||||
Used many times, doesn't change, let us create a module.
|
||||
Module is not ready, needs work.
|
||||
|
||||
TODO:
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Import libraries
|
||||
--------------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
class ventilLogger(object):
|
||||
|
||||
def __init__(self):
|
||||
__name__ = 'Zi logger! By Ventil!!!'
|
||||
|
||||
def all_other_shit(self):
|
||||
# Check log directory and create if non-existent
|
||||
if os.path.isdir(_LOG_DIR):
|
||||
# print "INFO: Log directory \"{}\" exists.".format(_LOG_DIR)
|
||||
files = os.listdir(_LOG_DIR)
|
||||
logfile_dict = {}
|
||||
for file in files:
|
||||
if _LOG_FILE_ROOT in file:
|
||||
file_path = os.path.join(_LOG_DIR, file)
|
||||
file_stats = os.stat(file_path)
|
||||
file_mtime = file_stats.st_mtime
|
||||
"""
|
||||
if datetime.datetime.now() - \
|
||||
file_stats.st_mtime > datetime.timedelta(hours=24)
|
||||
"""
|
||||
try:
|
||||
logfile_dict.update({file_path: file_mtime})
|
||||
except:
|
||||
logfile_dict = {file_path: file_mtime}
|
||||
else:
|
||||
pass
|
||||
sorted_list_keys = sorted(logfile_dict, key=logfile_dict.get)
|
||||
# select the last 30 log files to keep, delete the rest.
|
||||
files_to_keep = sorted_list_keys[-30:]
|
||||
for filename in sorted_list_keys:
|
||||
if filename not in files_to_keep:
|
||||
#print("Deleting {}".format(filename))
|
||||
os.remove(filename)
|
||||
else:
|
||||
#print("Not deleting {}".format(filename))
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
os.mkdir(_LOG_DIR)
|
||||
# print "INFO: Created logging directory \"{}\"".format(_LOG_DIR)
|
||||
except () as error:
|
||||
print(u"FATAL: Unable to create " +\
|
||||
u"logging directory \"{}\"".format(_LOG_DIR))
|
||||
raise SystemError(u"Unable to create log directory %s", error)
|
||||
|
||||
# Check for previous logs and rename if any
|
||||
if os.path.isfile(_LOG_FILE):
|
||||
timestapmp_logfile = os.path.getmtime(_LOG_FILE)
|
||||
date_logfile = datetime.datetime.fromtimestamp(timestapmp_logfile)
|
||||
_LOG_RENAME = _LOG_FILE + "." + date_logfile.strftime("%Y%m%d%H%M%S")
|
||||
os.rename(_LOG_FILE, _LOG_RENAME)
|
||||
if os.path.isfile(_DEBUG_FILE):
|
||||
timestapmp_logfile = os.path.getmtime(_DEBUG_FILE)
|
||||
date_logfile = datetime.datetime.fromtimestamp(timestapmp_logfile)
|
||||
_DEBUG_RENAME = _DEBUG_FILE + "." + date_logfile.strftime("%Y%m%d%H%M%S")
|
||||
os.rename(_DEBUG_FILE, _DEBUG_RENAME)
|
||||
|
||||
# Cleanup if more than _MAX_LOGS / _MAX_LOGS_SIZE logs are present
|
||||
# TODO
|
||||
|
||||
# Setup formatting
|
||||
_basic_format = "%(asctime)s %(name)s %(levelname)s %(message)s"
|
||||
_basic_formatter = logging.Formatter(_basic_format)
|
||||
_debug_format = "%(asctime)s %(name)s[%(process)d] \
|
||||
(%(funcName)s) %(levelname)s %(message)s"
|
||||
_debug_formatter = logging.Formatter(_debug_format)
|
||||
_console_format = "%(name)s %(levelname)s: %(message)s"
|
||||
_console_formatter = logging.Formatter(_console_format)
|
||||
|
||||
# Make logging readable with module hierarchy
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
# Setting up handlers for stdout / file logging and debug
|
||||
# Logfile
|
||||
basic_handler = logging.FileHandler(_LOG_FILE)
|
||||
basic_handler.setLevel(logging.ERROR)
|
||||
basic_handler.setFormatter(_basic_formatter)
|
||||
logger.addHandler(basic_handler)
|
||||
|
||||
# Debug file
|
||||
debug_handler = logging.FileHandler(_DEBUG_FILE)
|
||||
debug_handler.setLevel(logging.DEBUG)
|
||||
debug_handler.setFormatter(_debug_formatter)
|
||||
logger.addHandler(debug_handler)
|
||||
|
||||
# Console
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.CRITICAL)
|
||||
console_handler.setFormatter(_console_formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Just for debugging
|
||||
# print _LOG_FILE, _DEBUG_FILE
|
||||
# logger.debug(u'debug message')
|
||||
# logger.info(u'info message')
|
||||
# logger.warn(u'warn message')
|
||||
# logger.error(u'error message')
|
||||
# logger.critical(u'critical message')
|
||||
236
python/sysstats.py
Normal file
236
python/sysstats.py
Normal file
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/python -u
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
Type: Python 3.x script
|
||||
Author: Milan Toman (milan.v.toman@gmail.com)
|
||||
Description: System stats
|
||||
Version: 2.0 (SQLite)
|
||||
|
||||
SQLite SCHEMA:
|
||||
|
||||
DB status
|
||||
|
||||
TABLE raspi
|
||||
----------------
|
||||
t_stamp | usage | host | type
|
||||
-----------------------------------------------
|
||||
INT | INT | VARCHAR(50) | VARCHAR(20)
|
||||
|
||||
CREATE TABLE raspi(
|
||||
t_stamp INT,
|
||||
usage INT,
|
||||
host VARCHAR(50),
|
||||
type VARCHAR(20));
|
||||
|
||||
TABLE network
|
||||
----------------
|
||||
t_stamp | count | type | nic | host
|
||||
------------------------------------------------------------
|
||||
INT | INT | VARCHAR(20) | VARCHAR(20) | VARCHAR(50)
|
||||
|
||||
CREATE TABLE network(
|
||||
t_stamp INT,
|
||||
count INT,
|
||||
type VARCHAR(20),
|
||||
nic VARCHAR(20),
|
||||
host VARCHAR(50));
|
||||
|
||||
|
||||
|
||||
------------------------------------------------------------------------
|
||||
Import libraries
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
# mandatory
|
||||
import requests
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import textwrap
|
||||
import argparse
|
||||
import time
|
||||
import datetime
|
||||
import simplejson as json
|
||||
import sqlite3
|
||||
import psutil
|
||||
import socket
|
||||
|
||||
# optionally, future modules, locally available, I hate dependencies
|
||||
from pprint import pprint
|
||||
_SCRIPT_PATH = os.path.dirname(sys.argv[0])
|
||||
#sys.path.append(_SCRIPT_PATH + "/home/pi/test/lib")
|
||||
#print(_SCRIPT_PATH + "/lib")
|
||||
|
||||
global _hostname
|
||||
_hostname = socket.gethostname()
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
SQLite def
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
_SQLiteDB = '/var/lib/plutonium/status.db'
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
Define variables
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
_VERSION = 2.0
|
||||
_NAME = u"System stats collector"
|
||||
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
Generic, standalone functions
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
# Obvious shit, set up the client class
|
||||
influx_status_client = influxdb.client.InfluxDBClient(
|
||||
influx_host, influx_port, influx_user, influx_pwd, status_db
|
||||
)
|
||||
|
||||
|
||||
|
||||
def create_project(conn, project):
|
||||
"""
|
||||
Create a new project into the projects table
|
||||
:param conn:
|
||||
:param project:
|
||||
:return: project id
|
||||
"""
|
||||
sql = ''' INSERT INTO wind(name,begin_date,end_date)
|
||||
VALUES(?,?,?) '''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, project)
|
||||
return cur.lastrowid
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
Classes
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
|
||||
class DBwriter(object):
|
||||
def __init__(self):
|
||||
__name__ = "Database writer class, SQLite"
|
||||
|
||||
def construct(self, DB, query):
|
||||
""" Takes values in a writes them to influxdb
|
||||
|
||||
requires: list(connector): connector with ticks to be written
|
||||
at once
|
||||
str(measurement): the measurement ID to be written
|
||||
dict(fields): fields to be written in one tick
|
||||
dict(tags): tags to be written with the fields
|
||||
|
||||
returns: list(result_connector)
|
||||
"""
|
||||
conn = sqlite3.connect(db_file)
|
||||
c = conn.cursor()
|
||||
c.execute(query)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return result_connector
|
||||
|
||||
class Stats(object):
|
||||
def __init__(self):
|
||||
__name__ = "Database writer class, SQLite"
|
||||
|
||||
|
||||
|
||||
'''
|
||||
------------------------------------------------------------------------
|
||||
Main
|
||||
------------------------------------------------------------------------
|
||||
'''
|
||||
if '__main__':
|
||||
status_writer = DBwriter()
|
||||
while True:
|
||||
averaged_cpu = psutil.cpu_percent()
|
||||
mem_consumption = psutil.virtual_memory()[2]
|
||||
disk_usage = psutil.disk_usage('/')[3]
|
||||
print(psutil)
|
||||
# CPU stats
|
||||
for timeout in range(1,15):
|
||||
if timeout == 1:
|
||||
averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
|
||||
mem_consumption = psutil.virtual_memory()[2]
|
||||
disk_usage = psutil.disk_usage('/')[3]
|
||||
if_counters = psutil.net_io_counters(pernic=True)
|
||||
interfaces = if_counters.keys()
|
||||
|
||||
wlan_counters = if_counters["wlan0"]
|
||||
wlan_sent = wlan_counters[0]
|
||||
wlan_recv = wlan_counters[1]
|
||||
wlan_error_in = wlan_counters[4]
|
||||
wlan_error_out = wlan_counters[5]
|
||||
wlan_drop_in = wlan_counters[6]
|
||||
wlan_drop_out = wlan_counters[7]
|
||||
eth_counters = if_counters["eth0"]
|
||||
eth_sent = eth_counters[0]
|
||||
eth_recv = eth_counters[1]
|
||||
eth_error_in = eth_counters[4]
|
||||
eth_error_out = eth_counters[5]
|
||||
eth_drop_in = eth_counters[6]
|
||||
eth_drop_out = eth_counters[7]
|
||||
else:
|
||||
averaged_cpu = (averaged_cpu + psutil.cpu_percent()) / 2
|
||||
time.sleep(1)
|
||||
# Write the whole blob into Influx DB
|
||||
influx_status_write = status_writer.construct(
|
||||
influx_status_write,
|
||||
"RasPI",
|
||||
{"usage": float(averaged_cpu)},
|
||||
{"type": "cpu"}
|
||||
)
|
||||
influx_status_write = status_writer.construct(
|
||||
influx_status_write,
|
||||
"RasPI",
|
||||
{"usage": float(mem_consumption)},
|
||||
{"type": "mem"}
|
||||
)
|
||||
influx_status_write = status_writer.construct(
|
||||
influx_status_write,
|
||||
"RasPI",
|
||||
{"usage": float(disk_usage)},
|
||||
{"type": "disk"}
|
||||
)
|
||||
influx_status_write = status_writer.construct(
|
||||
influx_status_write,
|
||||
"net",
|
||||
{
|
||||
"b_out": float(wlan_sent),
|
||||
"b_in": float(wlan_recv),
|
||||
"e_out": float(wlan_error_out),
|
||||
"e_in": float(wlan_error_in),
|
||||
"drop_out": float(wlan_drop_out),
|
||||
"drop_in": float(wlan_drop_in),
|
||||
},
|
||||
{
|
||||
"type": "wlan0",
|
||||
"host": _hostname
|
||||
}
|
||||
)
|
||||
influx_status_write = status_writer.construct(
|
||||
influx_status_write,
|
||||
"net",
|
||||
{
|
||||
"b_out": float(eth_sent),
|
||||
"b_in": float(eth_recv),
|
||||
"e_out": float(eth_error_out),
|
||||
"e_in": float(eth_error_in),
|
||||
"drop_out": float(eth_drop_out),
|
||||
"drop_in": float(eth_drop_in),
|
||||
},
|
||||
{
|
||||
"type": "eth0",
|
||||
"host": _hostname
|
||||
}
|
||||
)
|
||||
print("Writing values: {}".format(influx_status_write))
|
||||
influx_status_client.write_points(influx_status_write)
|
||||
influx_status_write = []
|
||||
averaged_cpu = 0
|
||||
Reference in New Issue
Block a user