src/logalert.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import logging
   5 import os
   6 import shlex
   7 import subprocess
   8 import time
   9
  10 MAX_LINES = 10
  11 ALERT_INTERVAL = 86400
  12
  13 def follow(filename):
  14     while True:
  15         try:
  16             with open(filename, "r") as f:
  17                 logging.info("Re-attached to log file.")
  18                 for line in f: pass
  19                 while True:
  20                     line = f.readline()
  21                     if not line:
  22                         if not os.path.exists(filename):
  23                             break
  24                         else:
  25                             time.sleep(1.0)
  26                             yield None
  27                     else:
  28                         yield line.rstrip("\n")
  29         except FileNotFoundError:
  30             time.sleep(1.0)
  31             yield None
  32
  33 def feed_handler(data):
  34     try:
  35         handler = subprocess.Popen(shlex.split(args.handler),
  36                                    stdin=subprocess.PIPE,
  37                                    stdout=subprocess.PIPE,
  38                                    stderr=subprocess.PIPE,
  39                                    encoding='UTF-8')
  40         out_data, err_data = handler.communicate("%s\n" % data)
  41         if handler.returncode != 0:
  42             logging.warning("Handler exited with non-zero return code %d! (%s)" %
  43                             (handler.returncode, err_data))
  44     except Exception as e:
  45         logging.error("Error feeding handler: %s" % str(e))
  46
  47 def create_msg(title, icon, logfile, text, lines):
  48     msg = "<b>%s</b> <i>%s</i> %s" % (title, logfile, icon)
  49     msg += "<br>%s" % text
  50     msg += "<br><pre>"
  51     for line in lines: msg += line + "\n"
  52     msg += "</pre>"
  53     return msg
  54
  55 logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
  56                     level=logging.INFO,
  57                     datefmt='%m/%d/%Y %H:%M:%S')
  58
  59 parser = argparse.ArgumentParser(description='Alert on excessive number of error log lines.')
  60 parser.add_argument('logfile', type=str, help='logfile to be watched')
  61 parser.add_argument('handler', type=str,
  62                     help='alert will be delivered to standard input of handler')
  63 parser.add_argument('-s', '--interval-size', type=int, default=600, dest='interval_size',
  64                     help='sample interval size in seconds (default: 600)')
  65 parser.add_argument('-n', '--num-intervals', type=int, default=6, dest='num_intervals',
  66                     help='number of intervals to keep in history (default: 6)')
  67
  68 args = parser.parse_args()
  69
  70 kept_times = []
  71 lines = []
  72 last_slot_time = None
  73 error_state = False
  74 last_alert_time = 0
  75
  76 for line in follow(args.logfile):
  77     time_now = time.time()
  78     slot_now = int(time_now) // args.interval_size
  79
  80     if line != None:
  81         if not last_slot_time or slot_now > last_slot_time:
  82             kept_times.append(slot_now)
  83             last_slot_time = slot_now
  84         lines.append(line)
  85         if len(lines) > MAX_LINES:
  86             lines.pop(0)
  87
  88     while len(kept_times) > 0 and \
  89           kept_times[0] <= slot_now - (args.num_intervals + 1):
  90         kept_times.pop(0)
  91
  92     intervals = [False] * (args.num_intervals + 1)
  93     for kept_time in kept_times:
  94         intervals[slot_now - kept_time] = True
  95
  96     logging.debug(intervals)
  97
  98     if not False in intervals[1:]:
  99         if not error_state or time_now - last_alert_time > ALERT_INTERVAL:
 100             last_alert_time = time_now
 101             feed_handler( create_msg("Log Alert",
 102                                      "&#9760;",
 103                                      args.logfile,
 104                                      "Number of errors exceeded!",
 105                                      lines) )
 106         if not error_state:
 107             logging.warning("Entering error state!")
 108             error_state = True
 109
 110     else:
 111         if error_state:
 112             logging.info("Leaving error state.")
 113             error_state = False