fblog.py

from datetime import datetime, timedelta
import operator
import time

# Locale
from locale import LC_ALL, getlocale, setlocale
locale = getlocale()
setlocale(LC_ALL,'en_US')

def trace(source):
  for item in source:
    print item
    yield item

def generate(func):
  def gen_func(s):
    for item in s:
      yield func(item)
  return gen_func

def fblog(log_file=None, only=None, exclude=None):

  def group_entries(lines):
    entry = []
    for line in lines:
      if not line[0].isspace():
        if entry:
          yield tuple(entry)
          entry = []
      entry.append(line)
    if entry:
      yield tuple(entry)

  def parse_entry(entry):
    (server,timestamp) = entry[0].split('\t')
    timestamp = datetime.strptime(timestamp.strip(),'%a %b %d %H:%M:%S %Y')
    msg = [m.strip() for m in entry[1:]]
    key = msg[0]
    if key.startswith('Database:') and len(msg) > 1:
      key = msg[1]
    return (timestamp, key, msg)

  parse_entries = generate(parse_entry)

  log_file = log_file if log_file else open('firebird.log')
  non_empty = (line for line in log_file if line.lstrip())
  entries = (x for x in group_entries(non_empty))
  tuples = (x for x in parse_entries(entries))

  columns = ['timestamp','key','msg']
  output = (dict(zip(columns,t)) for t in tuples)
  if only:
    return (item for item in output if only in item['key'])
  elif exclude:
    return (item for item in output if exclude not in item['key'])
  else:
    return output

# utility generators

def normalize_key(entry):
  key = entry['key']
  entry['key'] = ' '.join(x for x in key.split() if (x.upper() != x) or x.isdigit())
  return entry
normalize_keys = generate(normalize_key)

def count_entries(entries):
  msgs = {}
  for entry in entries:
    key = entry['key']
    msgs[key] = msgs.get(key,0) + 1
  for key, count in msgs.items():
    yield (count, key)

def unique(iterable):
  seen = set()
  for element in iterable:
    k = element['key']
    if k not in seen:
      seen.add(k)
      yield element

def between(seq,from_time=None,to_time=None):
  for item in seq:
    if not (from_time or to_time):
      yield item
    else:
      send = not from_time or (from_time and item['timestamp'] >= from_time)
      send = send and (not to_time or (to_time and item['timestamp'] <= to_time))
      if send:
        yield item

def follow(thefile):
  thefile.seek(0,2)
  # Go to the end of the file
  while True:
    line = thefile.readline()
    if not line:
      time.sleep(1)
      # Sleep briefly
      continue
    yield line

# utility functions

def d(datestr):
  return datetime.strptime(datestr,"%d.%m.%y")

def dt(datetimestr):
  return datetime.strptime(datetimestr,"%d.%m.%y %H:%M:%S")

# report functions

def brief(seq):
  for item in seq:
    yield "%s - %s" % (item['timestamp'], item['key'])

def full(seq):
  for item in seq:
    yield str(item['timestamp'])
    for line in item['msg']:
      yield "   "+line

def frequency(seq):
  counts = [x for x in count_entries(seq)]
  counts.sort(key=operator.itemgetter(0),reverse=True)
  for count, msg in counts:
    yield "%8i %s" % (count, msg)

def show(lines):
  for line in lines: print line

Previous topic

merge_all.py

Next topic

Example 1