2019-08-02 01:26:38 -07:00
#!/usr/bin/env python3
2019-08-06 05:53:07 -07:00
from bottle import get , post , request , Bottle , run , template , static_file
2019-08-06 06:50:28 -07:00
import threading , time , json , zlib , gnupg , socket , psutil , os , sys , pymysql , queue , _thread , matplotlib . pyplot as plt , matplotlib . cm as cm , numpy
2019-07-26 04:55:47 -07:00
2019-08-05 05:54:48 -07:00
#gpg=gnupg.GPG('/usr/bin/gpg',gnupghome='/home/pi/.gnupg')
gpg = gnupg . GPG ( )
2019-07-26 04:55:47 -07:00
2019-08-01 05:55:25 -07:00
pathname = os . path . dirname ( sys . argv [ 0 ] )
abspath = os . path . abspath ( pathname )
configfile = abspath + " /config.json "
try :
cf = open ( configfile , " r " )
except :
cf = open ( configfile + " .template " , " r " )
log_conf = json . load ( cf )
cf . close ( )
2019-08-02 01:26:38 -07:00
parameter = { " device " : socket . gethostname ( ) , " allowed_ip " : { " 127.0.0.1 " : " 25A4CF79414F10FD " } , " gpg_keyid " : " 25A4CF79414F10FD " }
2019-08-01 05:55:25 -07:00
for n in parameter :
if n in log_conf :
parameter [ n ] = log_conf [ n ]
if " sqlserver " in log_conf :
hostname = " banana "
if " host " in log_conf [ ' sqlserver ' ] :
hostname = log_conf [ ' sqlserver ' ] [ ' host ' ]
port = 24049
if " port " in log_conf [ ' sqlserver ' ] :
port = int ( log_conf [ ' sqlserver ' ] [ ' port ' ] )
2019-08-01 21:17:09 -07:00
clientlist = parameter [ ' allowed_ip ' ]
try :
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
except :
clientlist = parameter [ ' allowed_ip ' ]
else :
mycursor = mydb . cursor ( pymysql . cursors . DictCursor )
mycursor . execute ( " select ip,keyid from clients; " )
myclientlist = mycursor . fetchall ( )
if len ( myclientlist ) > 0 :
tcl = { }
for i in myclientlist :
if len ( gpg . list_keys ( i [ ' keyid ' ] ) ) > 0 :
tcl [ i [ ' ip ' ] ] = i [ ' keyid ' ]
if len ( tcl ) > 0 :
clientlist = tcl
2019-08-01 05:55:25 -07:00
#sqlinsert="insert into measures (time,id,value) values ({0:d},{1:d},{2:d})"
sqlinsert = " insert into measures (time,id,value) values "
2019-08-15 05:28:12 -07:00
hsqlinsert = " insert into hourly_measures (time,id,value) values "
dsqlinsert = " insert into daily_measures (time,id,value) values "
2019-08-01 05:55:25 -07:00
measdata = { }
_HASH = " hash "
_SIGNEDGPG = " signed_gpg "
_PAYLOAD = " payload "
_MEASURES = " measure "
_BEGINSIGNATURE = " -----BEGIN PGP SIGNATURE----- "
_BEGINMESSAGE = " -----BEGIN PGP SIGNED MESSAGE----- "
_BEGINHASH = " Hash: "
_JSONDATA = " data "
_JSONSIGNEDDATA = " signed_data "
_JSONENCRYPTDATA = " encrypted_data "
2019-08-02 01:26:38 -07:00
def analyse_jsonin ( json_in , hash_id ) :
measdata = { }
if _JSONDATA in json_in :
measdata = json_in [ _JSONDATA ]
if _JSONSIGNEDDATA in json_in :
vpgp = gpg . verify ( json_in [ _JSONSIGNEDDATA ] )
if hash_id != vpgp . key_id :
print ( " signature does not fit hash id " )
else :
signed_in = json_in [ _JSONSIGNEDDATA ] . split ( " \n " )
signed_in [ signed_in . index ( _BEGINSIGNATURE ) : ] = " "
del signed_in [ signed_in . index ( _BEGINMESSAGE ) ]
del signed_in [ signed_in . index ( " " ) ]
for h in signed_in :
if _BEGINHASH in h :
del signed_in [ signed_in . index ( h ) ]
if len ( signed_in ) > 0 :
measdata = json . loads ( signed_in [ 0 ] )
if _JSONENCRYPTDATA in json_in :
dpgp = gpg . decrypt ( json_in [ _JSONENCRYPTDATA ] )
if hash_id != dpgp . key_id :
print ( " signature of encrypted data does not fit hash id " )
else :
measdata = json . loads ( dpgp . data )
if len ( measdata ) == 0 :
print ( " no data available " )
else :
_thread . start_new_thread ( insert_sql , ( measdata , ) )
2019-08-01 05:55:25 -07:00
def insert_sql ( measdata ) :
2019-08-15 05:28:12 -07:00
# tsi=sqlinsert
tsi = " "
2019-08-13 05:20:30 -07:00
tsi_count = 0
for h in measdata : # iterate over each variable stored in json
2019-08-01 05:55:25 -07:00
md = measdata [ h ]
2019-08-13 05:20:30 -07:00
for m in md [ ' measures ' ] : # iterate over each measurement for given variable
2019-08-01 05:55:25 -07:00
try :
stime = int ( m )
except :
print ( " wrong entry " )
else :
tsi = tsi + ' ( ' + str ( m ) + ' , ' + h + ' , ' + str ( md [ ' measures ' ] [ m ] ) + ' ), '
2019-08-13 05:20:30 -07:00
tsi_count = tsi_count + 1
2019-08-01 05:55:25 -07:00
tsi = tsi [ : - 1 ] + ' ; '
try :
2019-08-01 21:17:09 -07:00
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
2019-08-01 05:55:25 -07:00
except :
print ( " could not connect to mysql " )
with open ( " missed.sql " , " a " ) as sqlstore :
sqlstore . write ( tsi )
else :
mycursor = mydb . cursor ( pymysql . cursors . DictCursor )
2019-08-01 21:17:09 -07:00
for h in measdata :
mycursor . execute ( ' select id from ids where id= ' + h + ' ; ' )
2019-08-02 01:26:38 -07:00
t = mycursor . fetchall ( )
2019-08-01 21:17:09 -07:00
if len ( t ) == 0 :
hsql = ' insert into ids (id,device,varname,sensor,sensorsub,i2c) values ( ' + h + ' , '
hm = measdata [ h ]
ht = { }
for i in [ ' device ' , ' varname ' , ' sensor ' , ' sensorsub ' ] :
if i in hm :
2019-08-02 01:26:38 -07:00
hsql = hsql + " ' " + hm [ i ] + " ' " + ' , '
else :
2019-08-01 21:17:09 -07:00
hsql = hsql + " \' \' , "
if ' i2c ' in hm :
try :
2019-08-02 01:26:38 -07:00
hsql = hsql + " ' " + str ( hm [ ' i2c ' ] ) + " ' "
2019-08-01 21:17:09 -07:00
except :
2019-08-02 01:26:38 -07:00
hsql = hsql + " ' 0 ' "
2019-08-01 21:17:09 -07:00
else :
hsql = hsql + 0
hsql = hsql + ' ); '
try :
mycursor . execute ( hsql )
except :
print ( " could not insert new var_id " )
2019-08-02 01:26:38 -07:00
print ( hsql )
2019-08-15 05:28:12 -07:00
mycursor . execute ( sqlinsert + tsi )
mycursor . execute ( hsqlinsert + tsi )
mycursor . execute ( dsqlinsert + tsi )
delhtime = int ( ( time . time ( ) - 3600 ) * 1000 )
deldtime = int ( ( time . time ( ) - 86400 ) * 1000 )
mycursor . execute ( " delete from hourly_measures where time < " + str ( delhtime ) )
mycursor . execute ( " delete from daily_measures where time < " + str ( deldtime ) )
2019-08-13 05:20:30 -07:00
print ( str ( tsi_count ) + " new measures inserted " )
2019-08-01 05:55:25 -07:00
mycursor . close ( )
mydb . commit ( )
mydb . close ( )
2019-08-02 01:26:38 -07:00
# print(tsi)
# print(measdata)
2019-08-01 05:55:25 -07:00
2019-08-02 05:52:32 -07:00
2019-07-26 04:55:47 -07:00
app = Bottle ( )
2019-08-02 05:52:32 -07:00
#bottlesqp=bottle_mysql.Plugin(read_default_file='~/.my.cnf',db="rasolar")
#app.install(plugin)
2019-07-26 04:55:47 -07:00
@app.get ( ' / ' )
def approot ( ) :
return '''
2019-08-13 05:20:30 -07:00
< a href = " /ids " > Uebersicht < / a > < / br >
< a href = " /solar " > Solarübersicht < / a > < / br >
< a href = " /cpu " > CPU - Temperaturen < / a > < / br >
2019-07-26 04:55:47 -07:00
'''
2019-08-02 05:52:32 -07:00
@app.get ( ' /ids ' )
def show_ids ( ) :
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
mycursor = mydb . cursor ( pymysql . cursors . DictCursor )
2019-08-13 05:20:30 -07:00
starttime = time . time ( )
mycursor . execute ( ' select gm.*,measures.value,ids.varname,ids.device,ids.sensor,ids.i2c,ids.sensorsub from (select id, max(time) as time,count(time) as count from measures group by id) gm join measures on measures.id=gm.id and measures.time=gm.time join ids on ids.id=gm.id where gm.time>= ((select max(time) from measures)-6400000); ' )
print ( " ids sql duration: " + str ( time . time ( ) - starttime ) )
2019-08-02 05:52:32 -07:00
row = mycursor . fetchall ( )
mycursor . close ( )
mydb . close ( )
return template ( ' ids.tpl ' , measdata = row )
2019-08-09 11:15:59 -07:00
@app.get ( ' /solar ' )
def show_solar ( ) :
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
2019-08-09 11:21:32 -07:00
myc = mydb . cursor ( )
2019-08-09 11:15:59 -07:00
myc . execute ( " select gm.time,measures.value/1000,ids.varname from (select id, max(time) as time from measures group by id) gm join measures on measures.id=gm.id and measures.time=gm.time join ids on ids.id=gm.id where ids.sensor= ' tristar ' ; " )
ts = myc . fetchall ( )
myc . close ( )
mydb . close ( )
tt = { }
zs = 0
2019-08-09 11:20:14 -07:00
for tims , value , varname in ts :
2019-08-09 11:15:59 -07:00
tt [ varname ] = value
2019-08-09 11:22:36 -07:00
if int ( tims ) > zs :
2019-08-09 11:20:14 -07:00
zs = int ( tims )
2019-08-09 11:18:28 -07:00
return template ( ' solar.tpl ' , solardata = tt , zeitstempel = time . strftime ( ' % H: % M: % S % Y- % m- %d ' , time . localtime ( zs / 1000 ) ) )
2019-08-09 11:15:59 -07:00
2019-08-13 05:20:30 -07:00
@app.get ( ' /temp ' )
2019-08-13 06:45:26 -07:00
def show_temperature ( ) :
2019-08-13 05:20:30 -07:00
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
myc = mydb . cursor ( )
2019-08-13 07:13:41 -07:00
myc . execute ( " select ids.id,ids.device,ids.varname from ids where ids.sensor in ( ' BME280 ' , ' TSL2591 ' , ' VEML6070 ' ) " )
2019-08-13 05:20:30 -07:00
ti = myc . fetchall ( )
2019-08-13 06:47:51 -07:00
myids = " ( "
2019-08-13 05:20:30 -07:00
myvars = { }
mydev = { }
2019-08-13 06:46:47 -07:00
for ids , device , varname in ti :
2019-08-13 05:20:30 -07:00
myids = myids + " ' " + str ( ids ) + " ' , "
myvars [ ids ] = varname
mydev [ ids ] = device
myids = myids [ : - 1 ] + ' ) '
2019-08-13 06:45:26 -07:00
print ( myids )
print ( " select gm.time,measures.value/1000,ids.varname,ids.device from (select id, max(time) as time from measures where id in " + myids + " group by id) gm join measures on measures.id=gm.id and measures.time=gm.time join ids on ids.id=gm.id; " )
2019-08-13 05:20:30 -07:00
myc . execute ( " select gm.time,measures.value/1000,ids.varname,ids.device from (select id, max(time) as time from measures where id in " + myids + " group by id) gm join measures on measures.id=gm.id and measures.time=gm.time join ids on ids.id=gm.id; " )
ts = myc . fetchall ( )
myc . close ( )
mydb . close ( )
tt = { }
zs = 0
2019-08-13 07:13:41 -07:00
thp = { " rasolar " : { " temperature " : 0 , " humidity " : 0 , " pressure " : 0 , " time " : 0 } , " ragps " : { " temperature " : 0 , " humidity " : 0 , " pressure " : 0 , " time " : 0 } , " ragarden " : { " temperature " : 0 , " humidity " : 0 , " pressure " : 0 , " time " : 0 } }
2019-08-13 05:20:30 -07:00
for tims , value , varname , device in ts :
tt [ varname ] = value
2019-08-13 06:50:49 -07:00
try :
thp [ device ] [ varname ] = value
except :
thp [ device ] = { varname : value }
2019-08-13 05:20:30 -07:00
try :
if thp [ device ] [ ' time ' ] < tims :
thp [ device ] [ ' time ' ] = tims
except :
thp [ device ] [ ' time ' ] = tims
if int ( tims ) > zs :
zs = int ( tims )
return template ( ' temperature.tpl ' , tempdata = thp , zeitstempel = time . strftime ( ' % H: % M: % S % Y- % m- %d ' , time . localtime ( zs / 1000 ) ) )
2019-08-09 11:15:59 -07:00
2019-08-15 23:02:38 -07:00
@app.get ( ' /clients ' )
def show_clients ( ) :
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
myc = mydb . cursor ( pymysql . cursors . DictCursor )
no_clients = myc . execute ( ' select distinct ids.device from ids join hourly_measures on hourly_measures.id=ids.id; ' )
if no_clients > 0 :
t_cl = myc . fetchall ( )
client_list = { }
for cl in t_cl :
client_list . append ( cl )
return template ( clients . tpl , td = str ( int ( 1000 * time . time ( ) ) ) )
else :
return '''
< ! DOCTYPE html > < html > < head >
< title > Übersicht Tristar < / title >
< / head > < body >
< p > Aktuell keine Rechner verbunden < / p >
< / body > < / html >
'''
@app.get ( ' /client/<client> ' )
def forward_client ( client ) :
response = request . get ( ' http:// ' + client + ' :8080/ ' )
if response . status_code == 200 :
return response . content
else :
return '''
< ! DOCTYPE html > < html > < head >
< title > Keine Verbindung < / title >
< / head > < body >
< p > Kein Anschluss unter dieser Himbeere < / p >
< / body > < / html >
'''
2019-08-05 05:54:48 -07:00
@app.get ( ' /cpu ' )
def show_ids ( ) :
2019-08-13 05:20:30 -07:00
starttime = time . time ( )
2019-08-05 05:54:48 -07:00
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
2019-08-07 05:54:11 -07:00
myc = mydb . cursor ( pymysql . cursors . DictCursor )
myc . execute ( ' select * from ids where sensor= " CPU " ; ' )
channels = myc . fetchall ( )
myc . close ( )
2019-08-06 05:53:07 -07:00
cpu_col = cm . rainbow ( numpy . linspace ( 0 , 1 , len ( channels ) ) )
plt . figure ( figsize = [ 6 , 8 ] )
2019-08-07 05:54:11 -07:00
mycursor = mydb . cursor ( )
2019-08-06 05:53:07 -07:00
for j in range ( len ( channels ) ) :
2019-08-09 07:52:20 -07:00
mycursor . execute ( ' select ((select max(time) from measures)-measures.time)/1000 as time,measures.value/1000 as value from measures join ids on ids.id=measures.id where measures.time>= ((select max(time) from measures)-24*6400000) and ids.sensor= " CPU " and ids.id= ' + str ( channels [ j ] [ ' id ' ] ) + ' ; ' )
2019-08-07 05:54:11 -07:00
row = numpy . array ( mycursor . fetchall ( ) )
plt . plot ( row [ : , 0 ] , row [ : , 1 ] , color = cpu_col [ j ] , label = channels [ j ] [ ' varname ' ] + ' ; ' + channels [ j ] [ ' device ' ] )
2019-08-06 05:53:07 -07:00
plt . legend ( )
2019-08-13 05:20:30 -07:00
print ( str ( time . time ( ) - starttime ) + " s for fetching and display " )
2019-08-06 05:53:07 -07:00
plt . title ( ' CPU Verlauf ' )
2019-08-13 05:20:30 -07:00
plt . savefig ( " svg/cpu.svg " )
2019-08-05 05:54:48 -07:00
mycursor . close ( )
mydb . close ( )
if len ( row ) > 0 :
return template ( ' cputemp.tpl ' , measdata = row )
2019-08-06 05:53:07 -07:00
@app.get ( ' /svg/<svgfile> ' )
def show_svg ( svgfile ) :
2019-08-13 05:20:30 -07:00
return static_file ( " svg/ " + svgfile , root = abspath )
2019-08-06 05:53:07 -07:00
2019-08-05 05:54:48 -07:00
@app.get ( ' /graph/<mid> ' )
def show_graph ( mid ) :
2019-08-13 05:20:30 -07:00
starttime = time . time ( )
2019-08-05 05:54:48 -07:00
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
2019-08-07 05:54:11 -07:00
mycursor = mydb . cursor ( )
2019-08-13 05:20:30 -07:00
print ( str ( time . time ( ) - starttime ) + ' sql open ' )
2019-08-15 05:28:12 -07:00
mycursor . execute ( ' select ((select max(time) from hourly_measures)-hourly_measures.time)/6400000 as time,hourly_measures.value/1000 as value from hourly_measures where id= %s ' , str ( mid ) )
2019-08-07 05:54:11 -07:00
row = numpy . array ( mycursor . fetchall ( ) )
2019-08-13 05:20:30 -07:00
print ( str ( time . time ( ) - starttime ) + ' sql fetched ' )
2019-08-05 05:54:48 -07:00
mycursor . close ( )
mydb . close ( )
2019-08-13 05:20:30 -07:00
print ( str ( time . time ( ) - starttime ) + ' sql closed ' )
2019-08-06 05:53:07 -07:00
plt . figure ( figsize = [ 6 , 8 ] )
2019-08-07 05:54:11 -07:00
plt . plot ( row [ : , 0 ] , row [ : , 1 ] )
2019-08-13 05:20:30 -07:00
print ( str ( time . time ( ) - starttime ) + ' picture ' )
plt . savefig ( " svg/ " + mid + " .svg " )
print ( str ( time . time ( ) - starttime ) + ' saved ' )
2019-08-05 05:54:48 -07:00
if len ( row ) > 0 :
2019-08-06 05:53:07 -07:00
return template ( ' verlauf.tpl ' , measdata = row , mid = mid )
2019-08-02 05:52:32 -07:00
2019-08-01 05:55:25 -07:00
@app.post ( ' /data/<hash_id> ' )
def dataimport ( hash_id ) :
2019-08-02 01:26:38 -07:00
# print(hash_id)
2019-08-01 05:55:25 -07:00
timestart = time . time ( )
# check if request comes from allowed ip
2019-08-02 01:26:38 -07:00
if ( request . remote_addr in clientlist ) :
2019-08-01 05:55:25 -07:00
# hash must be the used gpg key id
2019-08-02 01:26:38 -07:00
if ( hash_id in clientlist [ request . remote_addr ] ) :
# print("correct id")
2019-08-01 05:55:25 -07:00
# check, if json is transmitted
try :
json_in = json . loads ( request . json )
2019-08-02 01:26:38 -07:00
# print(json_in)
2019-08-01 05:55:25 -07:00
except :
print ( " no json " )
else :
2019-08-02 01:26:38 -07:00
print ( time . time ( ) - timestart )
_thread . start_new_thread ( analyse_jsonin , ( json_in , hash_id , ) )
2019-08-01 05:55:25 -07:00
else :
print ( " wrong id " )
else :
print ( " not allowed client address " )
2019-07-26 04:55:47 -07:00
2019-08-02 07:06:35 -07:00
run ( app , host = " " , port = 8081 )
2019-08-07 05:54:11 -07:00
mydb = pymysql . connect ( read_default_file = " ~/.my.cnf " , database = " rasolar " )
#mycursor=mydb.cursor(pymysql.cursors.DictCursor)
mycursor = mydb . cursor ( pymysql . cursors . DictCursor )
mycursor . execute ( ' select gm.*,measures.value,ids.varname,ids.device,ids.sensor,ids.i2c,ids.sensorsub from (select id, max(time) as time,count(time) as count from measures group by id) gm join measures on measures.id=gm.id and measures.time=gm.time join ids on ids.id=gm.id where gm.time>= ((select max(time) from measures)-24*6400000); ' )
row = mycursor . fetchall ( )
mycursor . close ( )
mydb . close ( )