src/logalert.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import logging
   5 import os
   6 import shlex
   7 import subprocess
   8 import time
   9
  10 from line import Line
  11
  12 MAX_LINES = 10
  13
  14 def follow(filename):
  15     while True:
  16         try:
  17             with open(filename, "r") as f:
  18                 logging.info("Re-attached to log file.")
  19                 for line in f: pass
  20                 while True:
  21                     line = f.readline()
  22                     if not line:
  23                         if not os.path.exists(filename):
  24                             break
  25                         else:
  26                             time.sleep(1.0)
  27                             yield None
  28                     else:
  29                         yield line.rstrip("\n")
  30         except FileNotFoundError:
  31             time.sleep(1.0)
  32             yield None
  33
  34 def feed_handler(data):
  35     try:
  36         handler = subprocess.Popen(shlex.split(args.handler),
  37                                    stdin=subprocess.PIPE,
  38                                    stdout=subprocess.PIPE,
  39                                    stderr=subprocess.PIPE,
  40                                    encoding='UTF-8')
  41         out_data, err_data = handler.communicate("%s\n" % data)
  42         if handler.returncode != 0:
  43             logging.warning("Handler exited with non-zero return code %d! (%s)" %
  44                             (handler.returncode, err_data))
  45     except Exception as e:
  46         logging.error("Error feeding handler: %s" % str(e))
  47
  48 def create_msg(title, icon, logfile, text, lines):
  49     msg = "<b>%s</b> <i>%s</i> %s" % (title, logfile, icon)
  50     msg += "<br>%s" % text
  51     msg += "<br><pre>"
  52     for line in lines: msg += line + "\n"
  53     msg += "</pre>"
  54     return msg
  55
  56 logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
  57                     level=logging.INFO,
  58                     datefmt='%m/%d/%Y %H:%M:%S')
  59
  60 parser = argparse.ArgumentParser(description='Alert on excessive number of error log lines.')
  61 parser.add_argument('logfile', type=str, help='logfile to be watched')
  62 parser.add_argument('handler', type=str,
  63                     help='alert will be delivered to standard input of handler')
  64 parser.add_argument('-s', '--interval-size', type=int, default=600, dest='interval_size',
  65                     help='sample interval size in seconds (default: 600)')
  66 parser.add_argument('-n', '--num-intervals', type=int, default=6, dest='num_intervals',
  67                     help='number of intervals to keep in history (default: 6)')
  68
  69 args = parser.parse_args()
  70
  71 kept_times = []
  72 lines = []
  73 last_time = None
  74 error_state = False
  75
  76 for line in follow(args.logfile):
  77     now = int(time.time()) // args.interval_size
  78
  79     if line != None:
  80         if not last_time or now > last_time:
  81             kept_times.append(now)
  82             last_time = now
  83         lines.append(line)
  84         if len(lines) > MAX_LINES:
  85             lines.pop(0)
  86
  87     while len(kept_times) > 0 and \
  88           kept_times[0] <= now - args.num_intervals:
  89         kept_times.pop(0)
  90
  91     intervals = [False] * args.num_intervals
  92     for kept_time in kept_times:
  93         intervals[now - kept_time] = True
  94
  95     logging.debug(intervals)
  96
  97     if not False in intervals:
  98         if not error_state:
  99             logging.warning("Entering error state!")
 100             error_state = True
 101             feed_handler( create_msg("Log Alert",
 102                                      "&#9760;",
 103                                      args.logfile,
 104                                      "Number of errors exceeded!",
 105                                      lines) )
 106     else:
 107         if error_state:
 108             logging.info("Leaving error state.")
 109             error_state = False
 110             feed_handler( create_msg("Log Un-Alert",
 111                                      "&#127774;",
 112                                      args.logfile,
 113                                      "Log is back to normal.",
 114                                      lines) )