Commit 6719f9ae authored by Holger Brand's avatar Holger Brand
Browse files

Merge branch 'release/Release-1-1-0-0' into main

parents 2306c2e9 4e97fa31
......@@ -2,15 +2,7 @@
# -*- coding: utf-8 -*-
"""
The programm can be used to log MQTT-Topics to SQLite database.
Derived from template MQTT-Queue-Template.py
It can also be used to query the SQLite database via MQTT, example:
Topic: SQLQuery
Payload:
{
"SQLQuery": "select datetime, data from data where topic_id=(select topic_id from topics where datetime > '2022-03-25 09:30:00' AND topic_name = 'ADIO/ADIO_AI0') ORDER BY datetime ASC",
"SQLResponseTopic": "Test/SQLQueryResponse"
}
Refer to README.md for more details.
Copyright 2022 GSI Helmholtzzentrum für Schwerionenforschung GmbH
Dr. Holger Brand, EEL, Planckstraße 1, 64291 Darmstadt, Germany
......@@ -45,6 +37,7 @@ topic_sqlstatus = client_id+"/SQLStatus"
topic_status = client_id+"/Status"
topic_version = client_id+"/Version"
topics_to_log = []
log_all = False
q = queue.Queue()
......@@ -148,6 +141,7 @@ def command_processor():
logging.debug('topics_in_table: ' + str(topics_in_table))
topic_ids = []
topic_names = []
topic_previous = {}
for topic in topics_in_table:
topic_names.append(topic[1])
logging.debug('topic_names: ' + str(topic_names))
......@@ -164,7 +158,7 @@ def command_processor():
if command == "Command":
logging.info('Received command ' + command + ' Payload: ' + payload)
elif topic_sqlquery.find(topic)>=0: # is SQLQuery topic
logging.info('Received sqlquery ' + topic + ' Payload: ' + payload)
logging.debug('Received sqlquery ' + topic + ' Payload: ' + payload)
try:
request = json.loads(payload)
logging.debug('Request:' + str(request))
......@@ -190,15 +184,39 @@ def command_processor():
pass
else:# topic_sqlquery.find(topic)>=0: #is topic for DB logging
if topic in topic_names:
#Insert topic data
logging.debug('Topic: ' + topic + ' found in topics_in_table')
max_data_id = db_insert_data(db, dbc, max_data_id, topic, payload, topic_names)
logging.debug('log_all=' + str(log_all))
logging.debug('payload=' + str(payload))
try:
previous = topic_previous[topic]
logging.debug('previous=' + str(topic_previous[topic]))
except KeyError as e:
logging.debug('Previous value of ' + topic + ' not jet existing')
previous = None
pass
if not previous or payload != previous or log_all: #Insert topic data
logging.debug('Log topic: ' + topic + ' to database')
max_data_id = db_insert_data(db, dbc, max_data_id, topic, payload, topic_names)
pass
if topic not in topic_previous :
logging.debug('Topic: ' + topic + ' not found, add to in topic_previous')
topic_previous.update({topic : payload})
pass
if topic_previous[topic] != payload:
logging.debug('Topic: ' + topic + ' replace value in topic_previous')
topic_previous[topic] = payload
pass
else:
#Insert topic into table topics, then insert data
logging.debug('Topic: ' + topic + ' not found in topics_in_table')
if db_check_payload_type(payload):
max_topic_id, topics_in_table, topic_names = db_insert_topic(db, dbc, max_topic_id, topics_in_table, topic_names, topic)
max_data_id = db_insert_data(db, dbc, max_data_id, topic, payload, topic_names)
if topic not in topic_previous :
topic_previous.update({topic : payload})
pass
else:
topic_previous[topic] = payload
pass
pass
else:
logging.debug('Topic: ' + topic + ' ignored due to unsupported datatype.')
......@@ -368,12 +386,14 @@ if __name__ == '__main__':
epilog = 'Copyright 2021 GSI Helmholtzzentrum für Schwerionenforschung GmbH',\
prog = 'MQTT-SQLite-Logger'\
)
parser.add_argument('-a', '--all', dest='log_all', default=False)
parser.add_argument('-b', '--broker', dest='broker', default='localhost')
parser.add_argument('--debug', dest='debug_level', default=logging.INFO)
parser.add_argument('-d', '--database', dest='database_filename', default='MQTT-SQLite.sqlite')
parser.add_argument('-l', '--log', dest='log_filename', default='MQTT-SQLite-Logger.log')
parser.add_argument('-t', '--topics', dest='topics_filename', default='MQTT-Topics.json')
args = parser.parse_args()
log_all = args.log_all
broker = args.broker
logging.basicConfig(
filename='MQTT-SQLite-Logger.log', filemode='w+',
......
{
"topics": [
"ADIO/ADIO_AI0",
"ADIO/ADIO_DI0"
"CSPP-Template/SystemMonitor_Memory",
"CSPP-Template/SystemMonitor_CPU-Load"
]
}
......@@ -6,8 +6,8 @@ It can also be used to query the SQLite database via MQTT.
Define topics to log in JSON format, Wildcards (+,#) are suported, example:
{
"topics": [
"ADIO/ADIO_AI0",
"ADIO/ADIO_DI0"
"CSPP-Template/SystemMonitor_Memory",
"CSPP-Template/SystemMonitor_CPU-Load"
]
}
......@@ -44,7 +44,10 @@ Acknowledgements
Revision History
----------------
Revision 1.0.0.0 - 25.03.2022 H.Brand@gsi.de
Revision 1.1.0.0 - 08.04.2022 H.Brand@gsi.de
- Log all or changed values only to SQLite
Revision 1.0.0.1 - 25.03.2022 H.Brand@gsi.de
- Logging to SQLite
- Query SQLite (Topic: SQLQuery)
- Prepared to handle commands (Topic: Command)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment