ventil/freedavis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

working project, maybe

Browse Source
This commit is contained in:
Milan Toman
2019-08-22 19:24:35 +02:00
parent 8689dd8c63
commit a454a4aac5
152 changed files with 43419 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
doc/source/conf.py Normal file
View File

@@ -0,0 +1,195 @@
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
sys.setrecursionlimit(1500)
# -- Project information -----------------------------------------------------
project = 'Plutonium reporter'
copyright = '2019, Milan \'Ventil\' Toman'
author = 'Milan \'Ventil\' Toman'
# The short X.Y version
version = ''
# The full version, including alpha/beta/rc tags
release = '1.0'
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode',
'sphinx.ext.githubpages',
'sphinx.ext.intersphinx'
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
#html_theme_options = {
#'navbar_title': "Plutonium reporter DOC",
#'navbar_site_name': "Plutonium reporter",
#'navbar_links': [
# ("Live", "https://bastart.spoton.cz", True)
# ],
#'navbar_sidebarrel': True
#
#}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'Plutoniumreporterdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'Plutoniumreporter.tex', 'Plutonium reporter Documentation',
'Milan \'Ventil\' Toman', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'plutoniumreporter', 'Plutonium reporter Documentation',
[author], 1)
]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'Plutoniumreporter', 'Plutonium reporter Documentation',
author, 'Plutoniumreporter', 'One line description of project.',
'Miscellaneous'),
]
# -- Options for Epub output -------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = project
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
# -- Extension configuration -------------------------------------------------
# -- Options for intersphinx extension ---------------------------------------
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}

244
doc/source/db_structure.rst Normal file
View File

@@ -0,0 +1,244 @@
Structure of InfluxDB
=====================
influxDB SCHEMA:
DB weather
measure wind
----------------
value | speed or direction or windgust
---------------------------------------
field tag
measure temphumi
----------------
temperature | humidity | external, internal | pressure
---------------------------------------------------------
field field tag field
measure rain
----------------
rain | rate / total / intensity | restart if zero, was it 65535 => before?
---------------------------------------------
field tag field(int)
DB status
ISS measure
----------------
voltage | solar or capacitor | state / lqi / | battery or future_shit |
----------------------------------------------------------------
field tag field tag
RasPI system
----------------
usage | disk, mem, cpu, eth, wifi %
------------------------------------
field | tag
SQLite SCHEMA:
DB status
TABLE raspi
----------------
t_stamp | usage | host | type
-----------------------------------------------
INT | INT | VARCHAR(50) | VARCHAR(20)
CREATE TABLE raspi(
t_stamp INT,
usage INT,
host VARCHAR(50),
type VARCHAR(20));
TABLE network
----------------
t_stamp | count | type | nic | host
------------------------------------------------------------
INT | INT | VARCHAR(20) | VARCHAR(20) | VARCHAR(50)
CREATE TABLE network(
t_stamp INT,
count INT,
type VARCHAR(20),
nic VARCHAR(20),
host VARCHAR(50));
TABLE vantage_vue_iss
----------------------
t_stamp | voltage | type | lqi | rssi | batt_low
------------------------------------------------------------------
INT | INT | VARCHAR(20) | TINYINT | TINYINT | BOOL
-------------------------------------------------------------------------------
ENERGY
-------------------------------------------------------------------------------
CREATE CONTINUOUS QUERY cq_power_1m on voltage BEGIN
SELECT max(power) AS p_max, min(power) AS p_min,
mean(power) as power, mean(voltage) AS voltage
INTO voltage.monthly.mppt_aggregated
FROM voltage.realtime.mppt
GROUP BY time(1m),type
END
CREATE CONTINUOUS QUERY "cq_power_1h" ON "voltage" BEGIN
SELECT max("power") AS p_max,min(power) AS p_min,
mean(power) as power, mean(voltage) AS voltage
INTO "yearly"."mppt_aggregated"
FROM realtime.mppt
GROUP BY time(1h), type
END
CREATE CONTINUOUS QUERY "cq_power_6h" ON "voltage" BEGIN
SELECT max("power") AS p_max,min(power) AS p_min,
mean(power) as power, mean(voltage) AS voltage
INTO "infinite"."mppt_aggregated"
FROM realtime.mppt
GROUP BY time(6h), type
END
drop continuous query cq_power_30m on voltage
show retention policies on weather_v2
create retention policy realtime on weather_v2 duration 168h replication 1 shard duration 1h
create retention policy monthly on weather_v2 duration 720h replication 1 shard duration 24h
create retention policy yearly on weather_v2 duration 8760h replication 1 shard duration 168h
create retention policy infinite on weather_v2 duration 0s replication 1 shard duration 720h
select time,value into realtime.rain from autogen.rain where time > now()-1w group by *
select time,humidity,pressure,temperature into realtime.temphumi from autogen.temphumi where time > now()-1w group by *
select time,humidity,temperature into realtime.usense from autogen.usense where time > now()-1w group by *
select time,value into realtime.wind from autogen.wind where time > now()-1w group by *
alter retention policy realtime on weather_v2 default
-------------------------------------------------------------------------------
WIND - DONE
-------------------------------------------------------------------------------
CREATE CONTINUOUS QUERY "cq_rain_10m" ON "weather_v2" BEGIN
SELECT max("rainrate") AS rrate_max, mean(rainrate) AS rrate, max(rain_bucket_tips) AS rain_tips
INTO "monthly"."rainrate_aggregated"
FROM realtime.rain
GROUP BY time(10m)
END
CREATE CONTINUOUS QUERY "cq_rain_1h" ON "weather_v2" BEGIN
SELECT max("rainrate") AS rrate_max, mean(rainrate) AS rrate, max(rain_bucket_tips) AS rain_tips
INTO "yealy"."rainrate_aggregated"
FROM realtime.rain
GROUP BY time(10m)
END
-------------------------------------------------------------------------------
RAIN - DONE
-------------------------------------------------------------------------------
CREATE CONTINUOUS QUERY "cq_rain_10m" ON "weather_v2" BEGIN
SELECT max("value") AS val_max, mean(value) AS value
INTO "monthly"."rainrate_aggregated"
FROM realtime.rain
GROUP BY type,time(10m)
END
CREATE CONTINUOUS QUERY "cq_rain_1h" ON "weather_v2" BEGIN
SELECT max("value") AS val_max, mean(value) AS value
INTO "yearly"."rainrate_aggregated"
FROM realtime.rain
GROUP BY type,time(1h)
END
CREATE CONTINUOUS QUERY "cq_rain_6h" ON "weather_v2" BEGIN
SELECT max("value") AS val_max, mean(value) AS value
INTO "infinite"."rainrate_aggregated"
FROM realtime.rain
GROUP BY type,time(6h)
END
-------------------------------------------------------------------------------
TEMPHUMI - DONE
-------------------------------------------------------------------------------
CREATE CONTINUOUS QUERY "cq_temphumi_10m" ON "weather_v2" BEGIN
SELECT
max("humidity") AS humidity_max,
min("humidity") AS humidity_min,
mean("humidity") AS humidity,
max("temperature") AS temperature_max,
min("temperature") AS temperature_min,
mean("temperature") AS temperature
INTO "monthly"."temphumi_aggregated"
FROM realtime.temphumi
GROUP BY type, time(10m)
END
CREATE CONTINUOUS QUERY "cq_temphumi_1h" ON "weather_v2" BEGIN
SELECT
max("humidity") AS humidity_max,
min("humidity") AS humidity_min,
mean("humidity") AS humidity,
max("temperature") AS temperature_max,
min("temperature") AS temperature_min,
mean("temperature") AS temperature
INTO "yearly"."temphumi_aggregated"
FROM realtime.temphumi
GROUP BY type, time(1h)
END
CREATE CONTINUOUS QUERY "cq_temphumi_6h" ON "weather_v2" BEGIN
SELECT
max("humidity") AS humidity_max,
min("humidity") AS humidity_min,
mean("humidity") AS humidity,
max("temperature") AS temperature_max,
min("temperature") AS temperature_min,
mean("temperature") AS temperature
INTO "infinite"."temphumi_aggregated"
FROM realtime.temphumi
GROUP BY type, time(6h)
END
-------------------------------------------------------------------------------
USENSE - DONE
-------------------------------------------------------------------------------
CREATE CONTINUOUS QUERY "cq_usense_6h" ON "weather_v2" BEGIN
SELECT mean("battery") AS battery, mean(humidity) AS humidity, mean(temperature) AS temperature
INTO "yearly"."usense_aggregated"
FROM realtime.usense
GROUP BY type,time(6h)
END
CREATE CONTINUOUS QUERY "cq_usense_12h" ON "weather_v2" BEGIN
SELECT mean("battery") AS battery, mean(humidity) AS humidity, mean(temperature) AS temperature
INTO "infinite"."usense_aggregated"
FROM realtime.usense
GROUP BY type,time(12h)
END
-------------------------------------------------------------------------------
WIND
-------------------------------------------------------------------------------
-------------------------------------------------------------------------------
STATUS
-------------------------------------------------------------------------------
POLICIES
--------
create retention policy realtime on status duration 168h replication 1 shard duration 1h
create retention policy monthly on status duration 720h replication 1 shard duration 24h
create retention policy yearly on status duration 8760h replication 1 shard duration 168h
create retention policy infinite on status duration 0s replication 1 shard duration 720h
select time,usage into realtime.RasPI_aggregated from autogen.RasPI where time > now()-1w group by *
select time,voltage into realtime.iss_aggregated from autogen.iss where time > now()-1w group by *
--
SELECT NON_NEGATIVE_DERIVATIVE(max(*)) as traffic INTO "monthly"."net_aggregated" FROM autogen.net WHERE time > now()-2w GROUP BY time(30s)
CQ
--
CREATE CONTINUOUS QUERY "cq_net_1m" ON "status" BEGIN
SELECT NON_NEGATIVE_DERIVATIVE(max(*)) as traffic
INTO "monthly"."net_aggregated"
FROM realtime.net
WHERE time > now()-1m
GROUP BY time(30s)
END
alter retention policy realtime on status default

31
doc/source/examples.rst Normal file
View File

@@ -0,0 +1,31 @@
.. Plutonium reporter documentation master file, created by
sphinx-quickstart on Wed Apr 3 15:53:15 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
==============================================
!
==============================================
.. toctree::
:maxdepth: 3
:caption: Contents:
Dynamic data
=====================
.. automodule:: modules.dynamic
:members:
Dynamic Weather
================
.. autoclass:: modules.dynamic.DynamicWeather
:members:
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

95
doc/source/index.rst Normal file
View File

@@ -0,0 +1,95 @@
.. Plutonium reporter documentation master file, created by
sphinx-quickstart on Wed Apr 3 15:53:15 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
==============================================
DOC: Plutonium reporter
==============================================
.. toctree::
:maxdepth: 3
:caption: Contents:
introduction
Scope
=====
This document covers the **software part** of the reporter, although it consists
of a multitude of hw technologies, please keep that in mind.
Purpose
========
Delivering aggregated and comprehensive representation of data-value pairs
in such a way, that even a complete idiot can read them. *This project is
created by AND for me.*
In short, this application spins up a web server and on its address plots and
displays values gathered from various sources.
An example granted: https://bastart.spoton.cz
Sources
--------
The sources for feeding the **Plutonium** include:
- Davis Vantage vue weather station (with a couple HW mods)
- Victron MPPT solar converter (Utilizing the Victron Direct RS232 protocol)
- CPU, MEM, DISK, (W)LAN statistics, plotted
- Custom ESP8266 data loggers
- Temp / Humidity + battery logging
- Current monitoring for LED lighting
Discrepancies and hardware dependencies
---------------------------------------
Obviously, all the monitoring cannot be done without proper HW equipment.
Although this server is primarily aimed at use on a Raspberry PI, it can be
installed on an old notebook, or similar, as the platform is Python3 and thus
independent of the OS.
RasPI statistics
++++++++++++++++
- Just the RasPI, should work out of the box
Davis Vantage Vue
+++++++++++++++++
- Obviously the Davis Vantage Vue weather station
- The CC1101 / wireless version
- arduino mini / Uno @ 3.3V
- CC1101 receiver with a couple other components
- Data structure received from davis:
2 = Supercap voltage (Vue only)
3 = ?
4 = UV Index
5 = Rain rate
6 = Solar radiation
7 = Solar Cell output (Vue only)
8 = Temperature
9 = Wind gust
a = Humidity
e = Rain
{'nxt': 64, 'P0': 1020.43, 'lqi': 6, 'b2': 3, 'P': 969.29, 'h': 144, 'Ti': 24.49,
'cnt': 1, 'Hhtu': 28.68, 'b4': 129, 'b5': 247, 'b6': 36, 'b7': 255, 'b0': 1, 'b1': 10,
'hop': 0, 'b3': 225, 'Thtu': 24.28, 'b8': 255, 'b9': 182, 'rssi': 45}
Solar / MPPT
++++++++++++
- Victron MPPT solar charge controller (Bluesolar)
- RS232 -> USB or similar, to get the data to RasPi
ESP8266 stuff
+++++++++++++
- basically anything that can feed into the influxDB.
Indices and tables
===================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

134
doc/source/introduction.rst Normal file
View File

@@ -0,0 +1,134 @@
.. Plutonium reporter documentation master file, created by
sphinx-quickstart on Wed Apr 3 15:53:15 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Introduction to Plutonium
+++++++++++++++++++++++++
.. toctree::
:maxdepth: 5
:caption: Contents:
Directory structure
===================
Following structure is applied to this project. Some directories are
minified in this view on purpose.
::
.
├── chttpd.py
├── config
│   └── plutonium.ini
├── config.py
├── doc ... (documentation in sphinx)
├── index.py
├── localdeploy.sh
├── modules
│   ├── dynamic.py
│   ├── status.py
│   ├── temphumi.py
│   ├── voltage.py
│   └── weather.py
├── static
│   ├── css
│   │   ├── bootstrap.css
│   │   └── dygraph.css ...
│   ├── img
│   │   ├── battery_0.png ...
│   └── js
│   ├── solar_graph.js
│   ├── status_graph.js
│   ├── temphum_graph.js
│   └── weather_graph.js ...
├── templates
│   ├── footer.html
│   ├── header.html
│   ├── landing_page.html
│   ├── status_admin.html
│   ├── temphumi_admin.html
│   ├── top_menu.html
│   ├── voltage_admin.html
│   └── weather_admin.html
└── TODO.txt
CherryPy configuration (config.py)
==================================
Configuration is stored in a separate file statically, so each submodule can
load the same configuration. This should be variables, that are project-wide.
CherryPy configuration file (plutonium.ini)
-------------------------------------------
Configuration file, .ini style. Option = value. File resides in ./config
directory. It is read by confi.py and parsed into a dict(), available
throughout the project.
::
_server_protocol = https
_server_name = bastart.spoton.cz
_server_port = 80
_server_bind_ip = 0.0.0.0
_influx_host = localhost
_influx_port = 8086
_influx_user = pi
_influx_pwd = password
_influx_weather_db = weather_v2
_influx_status_db = status
_influx_voltage_db = voltage
_influx_IoT_db = weather_v2
Configuration classes and functions
------------------------------------
.. automodule:: config
:members:
CherryPy server (chttpd.py)
===========================
The server uses CherryPy module. For more information, please consult the
Cherrypy documentation.
CHTTPD.py is also the executable, that can be launched as a standalone
application by simply typing ./chttpd.py, or python3 chttpd.py
.. automodule:: chttpd
:members:
Modules and web paths
---------------------
Modules are located in the `modules` directory, hence the imports from a
subdirectory
::
from modules import voltage
from modules import weather
from modules import dynamic
from modules import status
from modules import temphumi
As can be seen, each class / module is mounted under a specific web path. This
is the preferred way of future expansion modules.
::
cherrypy.tree.mount(voltage.EnergyInfo(), "/", conf)
cherrypy.tree.mount(voltage.EnergyInfo(), "/energy", conf)
cherrypy.tree.mount(weather.WeatherInfo(), "/weather", conf)
cherrypy.tree.mount(status.StatusInfo(), "/status", conf)
cherrypy.tree.mount(dynamic.Expose(), "/data", conf)
cherrypy.tree.mount(temphumi.PuerhInfo(), "/temphumi", conf)
Index (index.py)
================
Reserved for future use. Currently not displayed, as the EnergyInfo() class is
mounted under root(/) of the web, defined in chttpd.py
::
cherrypy.tree.mount(voltage.EnergyInfo(), "/", conf)
.. automodule:: index
:members:
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`