src/logalert.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import logging
   5 import os
   6 import shlex
   7 import subprocess
   8 import time
   9
  10 from line import Line
  11
  12 MAX_LINES = 10
  13 ALERT_INTERVAL = 86400
  14
  15 def follow(filename):
  16     while True:
  17         try:
  18             with open(filename, "r") as f:
  19                 logging.info("Re-attached to log file.")
  20                 for line in f: pass
  21                 while True:
  22                     line = f.readline()
  23                     if not line:
  24                         if not os.path.exists(filename):
  25                             break
  26                         else:
  27                             time.sleep(1.0)
  28                             yield None
  29                     else:
  30                         yield line.rstrip("\n")
  31         except FileNotFoundError:
  32             time.sleep(1.0)
  33             yield None
  34
  35 def feed_handler(data):
  36     try:
  37         handler = subprocess.Popen(shlex.split(args.handler),
  38                                    stdin=subprocess.PIPE,
  39                                    stdout=subprocess.PIPE,
  40                                    stderr=subprocess.PIPE,
  41                                    encoding='UTF-8')
  42         out_data, err_data = handler.communicate("%s\n" % data)
  43         if handler.returncode != 0:
  44             logging.warning("Handler exited with non-zero return code %d! (%s)" %
  45                             (handler.returncode, err_data))
  46     except Exception as e:
  47         logging.error("Error feeding handler: %s" % str(e))
  48
  49 def create_msg(title, icon, logfile, text, lines):
  50     msg = "<b>%s</b> <i>%s</i> %s" % (title, logfile, icon)
  51     msg += "<br>%s" % text
  52     msg += "<br><pre>"
  53     for line in lines: msg += line + "\n"
  54     msg += "</pre>"
  55     return msg
  56
  57 logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
  58                     level=logging.INFO,
  59                     datefmt='%m/%d/%Y %H:%M:%S')
  60
  61 parser = argparse.ArgumentParser(description='Alert on excessive number of error log lines.')
  62 parser.add_argument('logfile', type=str, help='logfile to be watched')
  63 parser.add_argument('handler', type=str,
  64                     help='alert will be delivered to standard input of handler')
  65 parser.add_argument('-s', '--interval-size', type=int, default=600, dest='interval_size',
  66                     help='sample interval size in seconds (default: 600)')
  67 parser.add_argument('-n', '--num-intervals', type=int, default=6, dest='num_intervals',
  68                     help='number of intervals to keep in history (default: 6)')
  69
  70 args = parser.parse_args()
  71
  72 kept_times = []
  73 lines = []
  74 last_slot_time = None
  75 error_state = False
  76 last_alert_time = 0
  77
  78 for line in follow(args.logfile):
  79     time_now = time.time()
  80     slot_now = int(time_now) // args.interval_size
  81
  82     if line != None:
  83         if not last_slot_time or slot_now > last_slot_time:
  84             kept_times.append(slot_now)
  85             last_slot_time = slot_now
  86         lines.append(line)
  87         if len(lines) > MAX_LINES:
  88             lines.pop(0)
  89
  90     while len(kept_times) > 0 and \
  91           kept_times[0] <= slot_now - (args.num_intervals + 1):
  92         kept_times.pop(0)
  93
  94     intervals = [False] * (args.num_intervals + 1)
  95     for kept_time in kept_times:
  96         intervals[slot_now - kept_time] = True
  97
  98     logging.debug(intervals)
  99
 100     if not False in intervals[1:]:
 101         if not error_state or time_now - last_alert_time > ALERT_INTERVAL:
 102             last_alert_time = time_now
 103             feed_handler( create_msg("Log Alert",
 104                                      "&#9760;",
 105                                      args.logfile,
 106                                      "Number of errors exceeded!",
 107                                      lines) )
 108         if not error_state:
 109             logging.warning("Entering error state!")
 110             error_state = True
 111
 112     else:
 113         if error_state:
 114             logging.info("Leaving error state.")
 115             error_state = False
 116             feed_handler( create_msg("Log Un-Alert",
 117                                      "&#127774;",
 118                                      args.logfile,
 119                                      "Log is back to normal.",
 120                                      lines) )