rework of super-server using jinja2

This commit is contained in:
Kevin Francoise
2023-01-19 19:28:14 +01:00
parent 2ba2c988ea
commit 2bef6efea6
6 changed files with 447 additions and 643 deletions

View File

@@ -1,65 +1,25 @@
from zabbix_api import ZabbixAPI
from os import path
import asyncio
import json
import operator
import aiohttp_jinja2
import jinja2
import aiohttp
import aiohttp.web
import json
import settings
import time
import datetime
import logging
import sys
import os.path
import random
import argparse
import re
import config
import os
import json
import asyncio
import socket
from aiohttp import web
from zabbix_api import ZabbixAPI
from functools import lru_cache
stdio_handler = logging.StreamHandler()
stdio_handler.setLevel(logging.INFO)
_logger = logging.getLogger('aiohttp.access')
_logger.addHandler(stdio_handler)
_logger.setLevel(logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument("--zabbix_ip", help="Zabbix Frontend IP.", required=True)
parser.add_argument("--zabbix_url", help="Zabbix URL. Used to build triggers URL.", required=True)
parser.add_argument("--alert_limit", help="Number of alerts to retrieve.", required=True)
parser.add_argument("--zabbix_hostgroup", help="Search for hostgroups which correspond to this parameter (Wildcard allowed).", required=True)
parser.add_argument("--zabbix_min_severity", help="Minimum trigger severity to retrieve.", required=True)
parser.add_argument("--zabbix_login", help="Login to connect to the Zabbix API.", required=True)
parser.add_argument("--zabbix_pass", help="Password to connect to the Zabbix API.", required=True)
parser.add_argument("--list_zabbix_servers", help="List of Zabbix Servers to check", required=True, nargs="+")
parser.add_argument("--zabbix_timeout", help="Timeout to the API.", required=True, type=int)
parser.add_argument("--port", help="Listen Port.", required=True)
args = parser.parse_args()
ZABBIX_API = 'http://'+args.zabbix_ip
ZABBIX_FRONTEND = args.zabbix_url + '/'
ZABBIX_LOGIN = args.zabbix_login
ZABBIX_PASS = args.zabbix_pass
LIMIT = args.alert_limit
HOSTGROUP = args.zabbix_hostgroup
SEVERITY = args.zabbix_min_severity
TIMEOUT = args.zabbix_timeout
PORT = args.port
ZABBIX_SERVERS_CHECK = args.list_zabbix_servers
_logger.info('\n[ENVIRONMENT VARIABLES]\n[ZABBIX API] - {}'.format(ZABBIX_API))
_logger.info('[ZABBIX FRONTEND] - {}'.format(ZABBIX_FRONTEND))
_logger.info('[ALERT LIMIT] - {}'.format(LIMIT))
_logger.info('[HOSTGROUP] - {}'.format(HOSTGROUP))
_logger.info('[MIN SEVERITY] - {}'.format(SEVERITY))
_logger.info('[TIMEOUT] - {}'.format(TIMEOUT))
_logger.info('[PORT] - {}'.format(PORT))
_logger.info('[ZABBIX SERVERS TO CHECK] - {}\n'.format(ZABBIX_SERVERS_CHECK))
zapi = None
_logger.setLevel(logging.INFO)
def convert_seconds(seconds):
time = "a few sec"
@@ -87,92 +47,160 @@ def convert_seconds(seconds):
time = str(round(seconds / 31536000)) + " years"
return time
def severity_badge(severity):
if severity == 5:
badge = '<span class="octicon octicon-flame"> </span>'
elif severity == 4:
badge = '<span class="badge badge-pill badge-danger">4</span>'
elif severity == 3:
badge = '<span class="badge badge-pill badge-warning">3</span>'
elif severity == 2:
badge = '<span class="badge badge-pill badge-primary">2</span>'
elif severity == 1:
badge = '<span class="badge badge-pill badge-info">1</span>'
def zabbix_login():
global zapi
x = 0
while x < 10:
try:
zapi = ZabbixAPI(server=settings.ZABBIX_API, timeout=int(settings.TIMEOUT))
zapi.login(settings.ZABBIX_LOGIN, settings.ZABBIX_PASS)
except Exception as e:
x+=1
_logger.error("[ERR] - {} Retry #{}: {}".format(datetime.datetime.now(), x, e))
time.sleep(x)
continue
break
if x >= 10:
sys.exit("Can't connect to Zabbix API.")
def zabbix_call(request, method):
global zapi
x = 0
while x < 10:
try:
if method == 'hostgroup':
resp = zapi.hostgroup.get(request)
elif method == 'triggers':
resp = zapi.trigger.get(request)
except Exception as e:
x+=1
_logger.error("[ERR] - {} Retry #{}: {}".format(datetime.datetime.now(), x, e))
time.sleep(x)
continue
break
if x >= 10:
sys.exit("Can't perform calls to Zabbix API.")
else:
badge = '<span class="badge badge-pill badge-light">0</span>'
return badge
return resp
def severity_css(severity):
css = ""
if severity == 5:
css = 'class="disaster"'
return css
def read_file(file_to_update):
if os.path.exists(file_to_update):
with open(file_to_update, 'r') as f:
out = json.load(f)
f.close()
return out
return False
def html_response(data):
response = ""
for x in range(len(data)):
seconds = int(time.time()) - int(data[x]['lastchange'])
since = convert_seconds(seconds)
css_class = severity_css(int(data[x]['priority']))
badge = severity_badge(int(data[x]['priority']))
if data[x]['team'] not in config.COLOR_TEAM:
color = "#%06x;" % random.randint(0, 0xFFFFFF)
config.COLOR_TEAM[data[x]['team']] = color
else:
color = config.COLOR_TEAM[data[x]['team']]
response += "<tr "+css_class+"><td><span class='badge badge-pill' style='background-color:"+color+"'>"+data[x]['team']+"</span></td><td>"+data[x]['host']+"</td><td><a href='"+ZABBIX_FRONTEND+"zabbix.php?action=problem.view&filter_set=1&filter_triggerids%5B%5D="+data[x]['triggerid']+"' target='_blank' "+css_class+">"+data[x]['description']+"</a></td><td>"+badge+"</td><td><span class='badge badge-pill badge-light'>"+since+"</span></td></tr>"
def write_file(notes, file_to_update):
with open(file_to_update, 'w+') as f:
json.dump(notes, f)
f.close()
return response
def display_notes(request):
note_list = []
notes = read_file('./data/motd.json')
teams = request.match_info.get('teams')
if notes:
for ts in notes:
for note in notes[ts]:
if note['team'] in teams or note['team'] == 'all':
date_note = datetime.datetime.utcfromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
note_list.append({"lvl": note["lvl"], "date": date_note, "msg": note['msg'], "name": note['name'], "ts": ts })
return note_list
def capitalize_nth(s, n):
if len(s) == 2:
return s[:n].upper() + s[n:].capitalize()
def get_hostgroup_color(hostgroup):
if hostgroup not in settings.COLOR_TEAM:
color = "#%06x;" % random.randint(0, 0xFFFFFF)
settings.COLOR_TEAM[hostgroup] = color
else:
return s.title()
color = settings.COLOR_TEAM[hostgroup]
return color
def get_hostgroups():
groups = []
for hg in settings.HOSTGROUP:
request = dict()
request['output'] = 'extend'
request['search'] = dict()
request['search']['name'] = hg
request['searchWildcardsEnabled'] = 1
resp = zabbix_call(request, 'hostgroup')
for x in range(len(resp)):
groups.append(resp[x]['name'])
return groups
def construct_menu():
MENU = ""
def get_ttl_hash(seconds=45):
return round(time.time() / seconds)
for key in config.CONTENT.keys():
MENU += "<li class='nav-item'><a class='nav-link' href='/"+key+"'>"+ capitalize_nth(key.replace('Team-', ''),2) +"</a></li>"
return MENU
def construct_form():
FORM = "<div class='col'><select class='form-control' name='team'><option value='all'>All teams</option>"
for key in config.CONTENT.keys():
FORM += "<option value='"+key+"'>"+key+"</option>"
FORM += "</select></div>"
return FORM
async def check_servers():
global ZABBIX_SERVERS_CHECK
while True:
_logger.info('[INFO] - Checking Zabbix Servers: {}'.format(ZABBIX_SERVERS_CHECK))
j = {}
for ip in ZABBIX_SERVERS_CHECK:
port = 10051
if ':' in ip:
i = ip.split(':')
ip = i[0]
port = int(i[1].strip())
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
result = sock.connect_ex((ip, port))
if result == 0:
_logger.info("[INFO] - Port {} OK: {}".format(port, ip))
else:
_logger.error("[ERR] - Port {} KO: {}".format(port, ip))
j[ip] = result
sock.close()
if os.path.exists('./data/zabbix-servers.json'):
out = read_file('./data/zabbix-servers.json')
out.update(j)
write_file(out, './data/zabbix-servers.json')
else:
write_file(j, './data/zabbix-servers.json')
await asyncio.sleep(60)
@lru_cache()
def get_problems(ttl_hash = None):
del ttl_hash
problems = []
limit = settings.LIMIT
groups = [group.lower() for group in get_hostgroups()]
try:
zapi.logged_in()
except Exception as e:
_logger.info('[INFO] - {}: Connection to Zabbix API'.format(datetime.datetime.now()))
zabbix_login()
team_list = settings.HOSTGROUP
groupids = []
for hg in team_list:
req = dict()
req['output'] = 'extend'
req['search'] = dict()
req['search']['name'] = hg
req['searchWildcardsEnabled'] = 1
resp = zabbix_call(req, 'hostgroup')
for x in range(len(resp)):
groupids.append(int(resp[x]['groupid']))
req = dict()
req['limit'] = limit
req['groupids'] = groupids
req['monitored'] = 1
req['maintenance'] = 0
req['active'] = 1
req['min_severity'] = settings.SEVERITY
req['output'] = "extend"
req['expandData'] = 1
req['selectHosts'] = "extend"
req['selectGroups'] = "extend"
req['expandDescription'] = 1
req['only_true'] = 1
req['skipDependent'] = 1
req['withUnacknowledgedEvents'] = 1
req['withLastEventUnacknowledged'] = 1
req['selectTags'] = "extend"
req['filter'] = dict()
req['filter']['value'] = 1
req['sortfield'] = ["priority","lastchange"]
req['sortorder'] = ["DESC"]
resp = zabbix_call(req, 'triggers')
for x in range(len(resp)):
data = resp[x]
if len(data['hosts']) > 0:
# Loop on problems
for z in range(len(data['groups'])):
if int(data['groups'][z]['groupid']) in groupids:
hostgroup = data['groups'][z]['name']
color = get_hostgroup_color(hostgroup)
since = convert_seconds(int(time.time()) - int(data['lastchange']))
problems.append({'description': data['description'], 'host': data['hosts'][0]['host'], 'priority': data['priority'], 'triggerid': data['triggerid'], 'since': since, 'hostgroup': hostgroup, 'color': color, 'lastchange': data['lastchange']})
for y in range(len(data['tags'])):
tag_value = data['tags'][y]['value']
if tag_value.lower() in groups:
color = get_hostgroup_color(tag_value)
since = convert_seconds(int(time.time()) - int(data['lastchange']))
problems.append({'description': data['description'], 'host': data['hosts'][0]['host'], 'priority': data['priority'], 'triggerid': data['triggerid'], 'since': since, 'hostgroup': tag_value, 'color': color, 'lastchange': data['lastchange']})
_logger.info('[INFO] - {}: Refresh...'.format(datetime.datetime.now()))
return problems
async def post_note(request):
data = await request.post()
@@ -201,19 +229,6 @@ async def post_note(request):
_logger.info("[ADD] - {}".format(j))
return aiohttp.web.HTTPFound(location=url, text='{}'.format(ts), content_type='text/html')
def read_file(file_to_update):
if os.path.exists(file_to_update):
with open(file_to_update, 'r') as f:
out = json.load(f)
return out
f.close()
return False
def write_file(notes, file_to_update):
with open(file_to_update, 'w+') as f:
json.dump(notes, f)
f.close()
async def del_note(request):
data = await request.post()
note_id = data['note_id']
@@ -224,195 +239,90 @@ async def del_note(request):
write_file(out, './data/motd.json')
return aiohttp.web.HTTPFound(location=url)
async def show_alerts(request):
global CONTENT
global NAVBAR
global JS_CONTENT
global TEMPLATE_FOOTER
global TOTAL_ALERTS
data_list = []
html_content = ""
html_notes = "<table class='table table-borderless'>"
html_check = "<table class='table table-borderless table-sm'>"
response = ""
config.TEMPLATE_HEAD = config.TEMPLATE_HEAD.replace('FORM_TEAM', construct_form())
config.NAVBAR = config.NAVBAR.replace('LIST', construct_menu())
config.NAVBAR = re.sub('\[[0-9]+\]', '['+config.TOTAL_ALERTS+']', config.NAVBAR)
url = str(request.url)
if '/tv/' in url:
config.TEMPLATE_FOOTER = config.TEMPLATE_FOOTER.replace('show', 'hide')
else:
config.TEMPLATE_FOOTER = config.TEMPLATE_FOOTER.replace('hide', 'show')
teams = request.match_info.get('teams')
if not teams:
teams = []
notes = read_file('./data/motd.json')
if notes:
for ts in notes:
for note in notes[ts]:
if note['team'] in teams or note['team'] == 'all':
date_note = datetime.datetime.utcfromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
html_notes += "<tr class='bg-"+note['lvl']+"'><td class='text-left'><span class='octicon octicon-clock'></span> "+date_note+"</td><td class='text-center'> "+note['msg']+"</td><td class='text-right'><span class='octicon octicon-hubot'></span><i> (by "+note['name']+")</i></td><td class='text-right'><form action='/del' method='post' accept-charset='utf-8' enctype='application/x-www-form-urlencoded'><input type='text' class='form-control url_note' name='url' hidden readonly><input type='text' name='note_id' value='"+ts+"' readonly hidden><button type='submit' class='btn btn-outline-light btn-sm' id='del_note' ><span class='octicon octicon-trashcan'></span></button></form></td></tr>"
html_notes += '</table>'
check_zbx = read_file('./data/zabbix-servers.json')
if check_zbx:
for ip in check_zbx:
if check_zbx[ip] != 0:
html_check += "<tr class='bg-danger'><td class='text-center align-left'><span class='octicon octicon-alert'></span> Zabbix Server: "+ ip + " seems UNREACHABLE! <span class='octicon octicon-alert'></span></td></tr>"
html_check += '</table>'
IMAGE='<div class="container-fluid image-container"><img src="/images/zabbix_logo_500x131.png" width="30%"/></div>'
if teams:
team_list = teams.split('+')
for team in team_list:
team_alerts = config.CONTENT.get(team, {})
data_list.extend(team_alerts)
if data_list:
data = sorted(data_list,
key = lambda i: (i['priority'], i['lastchange']),
reverse=True)
result = html_response(data)
html_content = config.TEMPLATE_HEAD.replace('[alerts]', '['+str(len(data))+']').replace('NAVBAR', config.NAVBAR).replace('NOTES', html_notes).replace('CHECK', html_check) + result + config.TEMPLATE_FOOTER
else:
html_content = config.TEMPLATE_HEAD.replace('NAVBAR', config.NAVBAR).replace('NOTES', html_notes).replace('CHECK', html_check) + '<div class="alert alert-dark text-center" role="alert">No Alerts for '+str(teams)+'! <span class="octicon octicon-thumbsup"></span></div>'+IMAGE+' ' + config.TEMPLATE_FOOTER
else:
html_content = config.TEMPLATE_HEAD.replace('NAVBAR', config.NAVBAR) + response + config.TEMPLATE_FOOTER
return aiohttp.web.Response(text=html_content, content_type='text/html')
def zabbix_login():
global zapi
x = 0
while x < 10:
try:
zapi = ZabbixAPI(server=ZABBIX_API, timeout=int(TIMEOUT))
zapi.login(ZABBIX_LOGIN, ZABBIX_PASS)
except Exception as e:
x+=1
_logger.error("[ERR] - {} Retry #{}: {}".format(datetime.datetime.now(), x, e))
time.sleep(x)
continue
break
if x >= 10:
sys.exit("Can't connect to Zabbix API.")
def call_zabbix(request, method):
global zapi
x = 0
while x < 10:
try:
if method == 'hostgroup':
resp = zapi.hostgroup.get(request)
elif method == 'triggers':
resp = zapi.trigger.get(request)
except Exception as e:
x+=1
_logger.error("[ERR] - {} Retry #{}: {}".format(datetime.datetime.now(), x, e))
time.sleep(x)
continue
break
if x >= 10:
sys.exit("Can't perform calls to Zabbix API.")
else:
return resp
async def process_zabbix_queue():
global CONTENT
global TOTAL_ALERTS
global HOSTGROUP
global LIMIT
global SEVERITY
global zapi
async def check_servers():
while True:
try:
zapi.logged_in()
except Exception as e:
_logger.info('[INFO] - {}: Connection to Zabbix API'.format(datetime.datetime.now()))
zabbix_login()
request = dict()
request['output'] = 'extend'
request['search'] = dict()
request['search']['name'] = [HOSTGROUP]
request['searchWildcardsEnabled'] = 1
resp = call_zabbix(request, 'hostgroup')
groupids = []
for x in range(len(resp)):
config.CONTENT[resp[x]['name']] = []
groupids.append(resp[x]['groupid'])
request = dict()
request['limit'] = LIMIT
request['groupids'] = groupids
request['monitored'] = 1
request['maintenance'] = 0
request['active'] = 1
request['min_severity'] = SEVERITY
request['output'] = "extend"
request['expandData'] = 1
request['selectHosts'] = "extend"
request['selectGroups'] = "extend"
request['expandDescription'] = 1
request['only_true'] = 1
request['skipDependent'] = 1
request['withUnacknowledgedEvents'] = 1
request['withLastEventUnacknowledged'] = 1
request['selectTags'] = "extend"
request['filter'] = dict()
request['filter']['value'] = 1
request['sortfield'] = ["priority","lastchange"]
request['sortorder'] = ["DESC"]
_logger.info(request)
resp = call_zabbix(request, 'triggers')
config.TOTAL_ALERTS = str(len(resp))
_logger.info('[NB ALERTS] - {}'.format(config.TOTAL_ALERTS))
hostgroup_to_search = HOSTGROUP.replace('*','')
for x in range(len(resp)):
data = resp[x]
if len(data['hosts']) > 0:
for z in range(len(data['groups'])):
if hostgroup_to_search in data['groups'][z]['name']:
group = data['groups'][z]['name']
config.CONTENT[group].append({'description': data['description'], 'host': data['hosts'][0]['host'], 'priority': data['priority'], 'triggerid': data['triggerid'], 'lastchange': data['lastchange'], 'team': group})
for y in range(len(data['tags'])):
if 'team' == data['tags'][y]['tag']:
team = data['tags'][y]['value']
config.CONTENT[team].append({'description': data['description'], 'host': data['hosts'][0]['host'], 'priority': data['priority'], 'triggerid': data['triggerid'], 'lastchange': data['lastchange'], 'team': team})
_logger.info('[INFO] - {}: Refresh...'.format(datetime.datetime.now()))
await asyncio.sleep(15)
_logger.info('[INFO] - Checking Zabbix Servers: {}'.format(settings.ZABBIX_SERVERS_CHECK))
j = {}
for ip in settings.ZABBIX_SERVERS_CHECK:
port = 10051
if ':' in ip:
i = ip.split(':')
ip = i[0]
port = int(i[1].strip())
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
result = sock.connect_ex((ip, port))
if result == 0:
_logger.info("[INFO] - Port {} OK: {}".format(port, ip))
else:
_logger.error("[ERR] - Port {} KO: {}".format(port, ip))
j[ip] = result
sock.close()
if os.path.exists('./data/zabbix-servers.json'):
out = read_file('./data/zabbix-servers.json')
out.update(j)
write_file(out, './data/zabbix-servers.json')
else:
write_file(j, './data/zabbix-servers.json')
await asyncio.sleep(60)
async def start_background_tasks(app):
app['dispatch'] = app.loop.create_task(process_zabbix_queue())
app['dispatch'] = app.loop.create_task(check_servers())
async def cleanup_background_tasks(app):
app['dispatch'].cancel()
await app['dispatch']
@aiohttp_jinja2.template('index.html')
def display_alerts(request):
url = str(request.url)
if '/tv/' in url:
tv_mode = True
else:
tv_mode = False
teams = request.match_info.get('teams')
try:
zapi.logged_in()
except Exception as e:
_logger.info('[INFO] - {}: Connection to Zabbix API'.format(datetime.datetime.now()))
zabbix_login()
if teams:
team_list = teams.lower().split('+')
check_servers = read_file('./data/zabbix-servers.json')
notes = display_notes(request)
problems = get_problems(ttl_hash=get_ttl_hash())
_logger.info(len(problems))
problems = [problem for problem in problems if problem.get('hostgroup').lower() in team_list]
problems = sorted(problems,
key = lambda i: (i['priority'], i['lastchange']),
reverse=True)
_logger.info('[NB ALERTS] - {}'.format(len(problems)))
context = {'alerts': problems, 'total_alerts': len(problems), 'zabbix_url': settings.ZABBIX_URL, "hostgroups": get_hostgroups(), "notes": notes, "tv_mode": tv_mode, "check_servers": check_servers}
return aiohttp_jinja2.render_template(
'index.html', request, context)
def main():
app = web.Application()
aiohttp_jinja2.setup(app,
loader=jinja2.FileSystemLoader('templates'))
app = aiohttp.web.Application(logger=_logger)
app.add_routes([aiohttp.web.get('/', show_alerts),
app.add_routes([
web.get('/', display_alerts),
web.get('/{teams}', display_alerts),
web.get('/tv/{teams}', display_alerts),
aiohttp.web.post('/post', post_note),
aiohttp.web.post('/del', del_note),
aiohttp.web.get('/{teams}', show_alerts),
aiohttp.web.get('/tv/{teams}', show_alerts),
aiohttp.web.static('/images', 'images'),
aiohttp.web.static('/css', 'css'),
aiohttp.web.static('/js', 'js')])
aiohttp.web.static('/js', 'js')
])
app.on_startup.append(start_background_tasks)
app.on_cleanup.append(cleanup_background_tasks)
aiohttp.web.run_app(app, port=PORT)
aiohttp.web.run_app(app, port=settings.PORT)
if __name__ == '__main__':
main()
main()