src/logalert.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import logging
   5 import os
   6 import shlex
   7 import subprocess
   8 import time
   9
  10 MAX_LINES = 10
  11 ALERT_INTERVAL = 86400
  12
  13 def follow(path):
  14     while True:
  15         try:
  16             fd = os.open(path, os.O_RDONLY)
  17             current_ino = os.fstat(fd).st_ino
  18             with os.fdopen(fd, "r") as f:
  19                 logging.info("Re-attached to log file.")
  20                 for line in f: pass
  21                 while True:
  22                     line = f.readline()
  23                     if not line:
  24                         if os.stat(path).st_ino != current_ino or \
  25                            os.stat(path).st_size < f.tell():
  26                             break
  27                         else:
  28                             time.sleep(1.0)
  29                             yield None
  30                     else:
  31                         yield line.rstrip("\n")
  32         except FileNotFoundError:
  33             time.sleep(1.0)
  34             yield None
  35
  36 def feed_handler(data):
  37     try:
  38         handler = subprocess.Popen(shlex.split(args.handler),
  39                                    stdin=subprocess.PIPE,
  40                                    stdout=subprocess.PIPE,
  41                                    stderr=subprocess.PIPE,
  42                                    encoding='UTF-8')
  43         out_data, err_data = handler.communicate("%s\n" % data)
  44         if handler.returncode != 0:
  45             logging.warning("Handler exited with non-zero return code %d! (%s)" %
  46                             (handler.returncode, err_data))
  47     except Exception as e:
  48         logging.error("Error feeding handler: %s" % str(e))
  49
  50 def create_msg(title, icon, logfile, text, lines):
  51     msg = "<b>%s</b> <i>%s</i> %s" % (title, logfile, icon)
  52     msg += "<br>%s" % text
  53     msg += "<br><pre>"
  54     for line in lines: msg += line + "\n"
  55     msg += "</pre>"
  56     return msg
  57
  58 logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
  59                     level=logging.INFO,
  60                     datefmt='%m/%d/%Y %H:%M:%S')
  61
  62 parser = argparse.ArgumentParser(description='Alert on excessive number of error log lines.')
  63 parser.add_argument('logfile', type=str, help='logfile to be watched')
  64 parser.add_argument('handler', type=str,
  65                     help='alert will be delivered to standard input of handler')
  66 parser.add_argument('-s', '--interval-size', type=int, default=600, dest='interval_size',
  67                     help='sample interval size in seconds (default: 600)')
  68 parser.add_argument('-n', '--num-intervals', type=int, default=6, dest='num_intervals',
  69                     help='number of intervals to keep in history (default: 6)')
  70
  71 args = parser.parse_args()
  72
  73 kept_times = []
  74 lines = []
  75 last_slot_time = None
  76 error_state = False
  77 last_alert_time = 0
  78
  79 for line in follow(args.logfile):
  80     time_now = time.time()
  81     slot_now = int(time_now) // args.interval_size
  82
  83     if line != None:
  84         if not last_slot_time or slot_now > last_slot_time:
  85             kept_times.append(slot_now)
  86             last_slot_time = slot_now
  87         lines.append(line)
  88         if len(lines) > MAX_LINES:
  89             lines.pop(0)
  90
  91     while len(kept_times) > 0 and \
  92           kept_times[0] <= slot_now - (args.num_intervals + 1):
  93         kept_times.pop(0)
  94
  95     intervals = [False] * (args.num_intervals + 1)
  96     for kept_time in kept_times:
  97         intervals[slot_now - kept_time] = True
  98
  99     logging.debug(intervals)
 100
 101     if not False in intervals[1:]:
 102         if not error_state or time_now - last_alert_time > ALERT_INTERVAL:
 103             last_alert_time = time_now
 104             feed_handler( create_msg("Log Alert",
 105                                      "&#9760;",
 106                                      args.logfile,
 107                                      "Number of errors exceeded!",
 108                                      lines) )
 109         if not error_state:
 110             logging.warning("Entering error state!")
 111             error_state = True
 112
 113     else:
 114         if error_state:
 115             logging.info("Leaving error state.")
 116             error_state = False