Zabbix Alert Monitor

#!/usr/bin/python

import sys
import logging
import smtplib
import string
from email.mime.text import MIMEText
from datetime import datetime, timedelta
from calendar import timegm
from pyzabbix import ZabbixAPI
from array import array

###################################################

# [ CONFIG START ]

# Zabbix Server Info
URL = 'https://host.your.org/zabbix'
API_USER = 'api'
API_PASSWORD = 'api_password' # avoid or escape ( \' )  apostrophes in password

# Action IDs to check in comma seperated array: [1, 2, 3]
CHECK_ACTIONS = [3]

# after ALERTS_PER alerts are generated for an action within MINUTES_DISABLE minutes, the action will be disabled
# action will be re-enabled after no alerts have been generated for that action for MINUTES_RECOVERY minutes.
#
# NOTES: -Both problem and recovery operation alerts get counted, unless you filter them out (see below)
#    -Once the action is disabled, no alerts are generated. So it will always recover after MINUTES_RECOVERY.
#    -MINUTES_RECOVERY must be >= MINUTES DISABLE to prevent flapping
#
ALERTS_PER = 5
MINUTES_DISABLE = 5
MINUTES_RECOVERY = 15

# Ignore alerts where string found in subject, array format: ["ignore me", "ignore me too"]
ALERT_IGNORE = ["Resolved:"]

# SMTP server info for being alerted when action status changes
SMTP_SERVER = "10.11.12.13"
SMTP_PORT = 25
SMTP_SSL = 0
SMTP_USER = "anonymous" # leave as "anonymous" for no logon
SMTP_PASS = "anonymous"

# recipients in array format: ["recip1@a.com", "recip2@b.com"]
SMTP_TO = ["email1@your.org","email2@your.org"]
SMTP_FROM = "zabbix_monitor@your.org"
SMTP_SUBJECT = "Zabbix Alert Monitor"

SMTP_ALERT_WHEN_DISABLED = 1
SMTP_ALERT_WHEN_ENABLED = 1
SMTP_ALERT_ERRORS = 1


# [ CONFIG END ]

###################################################

DEBUG=False
if DEBUG:
    stream = logging.StreamHandler(sys.stdout)
    stream.setLevel(logging.DEBUG)
    log = logging.getLogger('pyzabbix')
    log.addHandler(stream)
    log.setLevel(logging.DEBUG)


def unix_time(dttm=None):
    if dttm is None:
       dttm = datetime.utcnow()

    return timegm(dttm.utctimetuple())


def send_email(sub, body):
    mail_server = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
    if SMTP_USER != "anonymous":
        mail_server.login(SMTP_USER, SMTP_PASS)
    if SMTP_SSL == 1:
        mail_server.starttls()
    msg = MIMEText(body)
    msg['Subject'] = sub
    msg['From'] = SMTP_FROM
    msg['To'] = ", ".join(SMTP_TO)
    mail_server.sendmail(SMTP_FROM, SMTP_TO, msg.as_string())
    mail_server.quit

if MINUTES_RECOVERY < MINUTES_DISABLE or ALERTS_PER < 1:
    print("Config error with MINUTES_RECOVERY, MINUTES_DISABLE, or ALERTS_PER")
    quit()

# init the API and Authenticate
zapi = ZabbixAPI(URL)
zapi.timeout = 5.1
zapi.login(API_USER, API_PASSWORD)

disable_start_time = datetime.utcnow() - timedelta(minutes = MINUTES_DISABLE)
recovery_start_time = datetime.utcnow() - timedelta(minutes = MINUTES_RECOVERY)
recovery_enable_time = (datetime.now() + timedelta(minutes = MINUTES_RECOVERY)).strftime("%Y-%m-%d %H:%M:%S")


# DISABLE CHECKS
print("")
print("Action disable checks for alerts starting at:", disable_start_time.strftime("%Y-%m-%d %H:%M:%S"), "(UTC)...")
disable_start_time = unix_time(disable_start_time)

seen_eventids = []
for action_id in CHECK_ACTIONS:
    for action in zapi.action.get(actionids=action_id):
        if action['status'] != '1':
            alert_count = 0
            skip_count = 0
            for alert in zapi.alert.get(actionids=action_id, time_from=disable_start_time):
                ignore_alert = 0
                for str_ignore in ALERT_IGNORE:
                    if str_ignore.lower() in alert['subject'].lower():
                        ignore_alert = 1

                if alert['status'] != '0' and ignore_alert == 0:
                    if alert['eventid'] in seen_eventids:
                        print('Skipped due to duplicate event: ' + alert['subject'])
                        skip_count += 1
                    else:
                        print(alert['subject'])
                        alert_count += 1
                        seen_eventids.append(alert['eventid'])
                else:
                    skip_count += 1

            print("Alerts for action " + str(action_id) + " (\"" + action['name'] + "\"): skipped: " + str(alert_count) + ", counted: " + str(skip_count))

            if alert_count >= ALERTS_PER:
                err = 0
                try:
                    zapi.action.update(
                        actionid=action_id,
                        status=1
                    )
                except:
                    err = 1

                if err == 0:
                    subject = SMTP_SUBJECT + ": " + action['name'] + " DISABLED"
                    message = "Action \"" + action['name'] + "\" has been disabled and will be re-enabled after " + recovery_enable_time
                    print(message)
                    if SMTP_ALERT_WHEN_DISABLED == 1:
                        send_email(subject, message)
                else:
                    subject = SMTP_SUBJECT + ": ERROR"
                    message = "ERROR disabling: " + action['name']
                    print(message)
                    if SMTP_ALERT_ERRORS == 1:
                        send_email(subject, message)
        else:
            print("Action " + str(action_id) + " (\"" + action['name'] + "\") is currently disabled")

# RECOVERY CHECKS

print("")
print("Action recover checks for alerts starting at:", recovery_start_time.strftime("%Y-%m-%d %H:%M:%S"), "(UTC)...")

recovery_start_time = unix_time(recovery_start_time)

for action_id in CHECK_ACTIONS:
    for action in zapi.action.get(actionids=action_id):
        if action['status'] != '0':
            alert_count = 0
            for alert in zapi.alert.get(actionids=action_id, time_from=recovery_start_time):
                print(alert['subject'])
                alert_count += 1
                last_eventid = alert['eventid']

            print("Alerts counted for action " + str(action_id) + " (\"" + action['name'] + "\"): " + str(alert_count))

            if alert_count == 0:
                                err = 0
                                try:
                                        zapi.action.update(
                                                actionid=action_id,
                                                status=0
                                        )
                                except:
                                        err = 1

                                if err == 0:
                                        subject = SMTP_SUBJECT + ": " + action['name'] + " ENABLED"
                                        message = "Action " + action['name'] + " has been re-enabled"
                                        print(message)
                                        if SMTP_ALERT_WHEN_ENABLED == 1:
                                                send_email(subject, message)
                                else:
                                        subject = SMTP_SUBJECT + ": ERROR"
                                        message = "ERROR enabling: " + action['name']
                                        print(message)
                                        if SMTP_ALERT_ERRORS == 1:
                                                send_email(subject, message)

        else:
            print("Action " + str(action_id) + " (\"" + action['name'] + "\") is not currently disabled")

print("")
# fin