import re from subprocess import Popen, PIPE from datetime import datetime import sys import os.path class LdapLogStats: def __init__( self, oldStats = None, file = None, server = None, outputDiff = 0 ): # Set this to 1 if you want it to log how long each operation takes self.debug = 0 # This holds all the timers for the debugging self.timers = {} # Lenght of time the script has to run in(Seconds). self.timeLimit = 99999 # Where to put the output log self.outputLog = '/var/log/ldapanalyzer.log' # Where to put the old stats file if oldStats == None: self.oldStatsPath = '/tmp/oldStats' else: self.oldStatsPath = oldStats self.scriptTimer( "ScriptTime" ) if file == None: self.logfile = '/var/log/syslog' else: self.logfile = file if server == None: self.server = "" else: self.server = " | grep " + server if self.debug: print "Server: " + self.server + " LogFile: " + self.logfile self.timer( "GrepCommand" ) conn = 'conn=[0-9]+ ' now = datetime.now() day = str( now.day ) month = now.strftime( "%b" ) #day = '8' #month = 'Sep' if len( day ) < 2: day = ' ' + day cmd = 'egrep "' + conn + '" ' + self.logfile + ' | grep "' + month + ' ' + day + ' ' + self.server + '"' if self.debug: print cmd p = Popen( cmd, shell=True, stdout=PIPE, stderr=PIPE ) self.timer( "GrepCommand" ) self.timer( "Readlines" ) self.log = p.stdout.readlines(); self.timer( "Readlines" ) self.timer( "WalkLog" ) stats, connections, lastLine = self.walkLog( self.log ) self.timer( "WalkLog" ) self.timer( "IterateStats" ) for key,val in stats.items(): if key == 'SSLConns': # Both TLS and SSL both have the TLS established so just subtract the # of TLS conns from SSl to get the correct ammount stat = "SSLConns:" + str( stats["SSLConns"] - stats["TLSConns"] ) + " " else: stat = key + ":" + str( val ) + " " sys.stdout.write( stat ) self.timer( "IterateStats" ) self.scriptTimer( "ScriptTime" ) self.writeLog( str( datetime.now() ) + ': Total Conns Processed: ' + str( stats["TotalConns"] ) + "\n" + str( self.timers ) + "\n" ) self.writeOldStats( stats, lastLine ) def writeOldStats( self, stats, lastLine ): f = open( self.oldStatsPath, 'w' ) statstring = "" for key,val in stats.items(): statstring += key + ":" + str( val ) + " " f.write( statstring + '\n' ) f.write( lastLine ) f.close() def readLastStats( self ): if not os.path.isfile( self.oldStatsPath ): return ({}, None) f = open( self.oldStatsPath, 'r' ) lastStats = f.readlines() f.close() # If there are less than 2 lines the lastStats file is wrong if len( lastStats ) < 2: return ({}, None) # Because logs can get rotated we need to make sure we are reading the same log file cmd = 'grep -F "' + lastStats[1] + '" ' + self.logfile p = Popen( cmd, shell=True, stdout=PIPE, stderr=PIPE ) if p.stdout.read() == '': return ({}, None) oldStats = {} for item in lastStats[0].split( ): type, amount = item.split( ':' ) oldStats[type] = int( amount ) return (oldStats, lastStats[1]) def writeLog( self, stuff ): log = open( self.outputLog, 'a' ) log.write( stuff ) if self.debug: print stuff log.close() def scriptTimer( self, whatIsTimed ): try: start = self.timers[whatIsTimed] self.timers[whatIsTimed] = (datetime.now() - start).seconds except: self.timers[whatIsTimed] = datetime.now() def timer( self, whatIsTimed ): if self.debug: try: start = self.timers[whatIsTimed] self.timers[whatIsTimed] = (datetime.now() - start).seconds except: self.timers[whatIsTimed] = datetime.now() def walkLog( self, pLog = None ): # Get the old stats to append to oldStats, lastLine = self.readLastStats() walkStart = datetime.now(); openedConnections = {} if oldStats == {}: stats = { "TotalConns" : 0, "Binds" : 0, "SSLConns" : 0, "TLSConns" : 0, "Searches" : 0, "NoEncryptConns" : 0, "SSLFail" : 0, "TLSNoneFail" : 0 } else: stats = oldStats if pLog == None: log = self.log else: log = pLog # Opened Connection connStart = re.compile( '.*conn=([0-9]+) fd=[0-9]+ ACCEPT.*' ) # SSL Conn connSSL = re.compile( '.*conn=([0-9]+) fd=[0-9]+ TLS established tls_ssf=[0-9]+ ssf=[0-9]+' ) # TLS Conn connTLS = re.compile( '.*conn=([0-9]+) op=[0-9]+ STARTTLS' ) # Bind Attempt connBind = re.compile( '.*conn=([0-9]+) op=[0-9]+ BIND.*mech=SIMPLE.*' ) # Search Attempt connSRCH = re.compile( '.*conn=([0-9]+) op=[0-9]+ SEARCH RESULT .*' ) # SSL Failed Attempt connSSLFail = re.compile( '.*conn=([0-9]+) fd=[0-9]+ closed \(connection lost\)' ) # TLS & None Failed Attempt connTLSNoneFail = re.compile( '.*conn=([0-9]+) fd=[0-9]+ closed \(TLS negotiation failure\)' ) # Keep the last line for the oldStats lastDateTime = "" # Stats of how many lines were actually processed processedLines = 0 if lastLine != None: foundLastLine = 0 else: foundLastLine = 1 if self.debug: print str( len( log ) ) + " lines to process" # Loop through the log file for logLine in log: # Grab each line as at the end this will be the last line if not foundLastLine: if lastLine.strip() == logLine.strip(): if self.debug: print "Beginning to process lines" foundLastLine = 1 continue # Grab each line as at the end this will be the last line lastDateTime = logLine processedLines += 1 if (datetime.now() - walkStart).seconds >= self.timeLimit: self.writeLog( "Exiting over " + str( self.timeLimit ) + " second run time" ) sys.exit( -1 ) # Search Attempt connNum = connSRCH.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["Search"] += 1 stats["Searches"] += 1 continue except KeyError: ''' Do Nothing ''' # Opened connection connNum = connStart.search( logLine ) if connNum != None: openedConnections[connNum.group(1)] = { "Bind" : 0, "SSLSuccess" : 0, "TLSSuccess" : 0, "NoEncryptSuccess" : 0, "SSLFail" : 0, "TLSNoneFail" : 0, "Search" : 0 } stats["TotalConns"] += 1 continue # Bind Attempt connNum = connBind.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["Bind"] += 1 stats["Binds"] += 1 continue except KeyError: ''' Do Nothing ''' # SSL Connection connNum = connSSL.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["SSLSuccess"] += 1 stats["SSLConns"] += 1 continue except KeyError: ''' Do Nothing ''' # SSL Fail connNum = connSSLFail.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["SSLFail"] += 1 stats["SSLFail"] += 1 continue except KeyError: ''' Do Nothing ''' # TLS Connection connNum = connTLS.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["TLSSuccess"] += 1 stats["TLSConns"] += 1 continue except KeyError: ''' Do Nothing ''' # TLS & None Fail connNum = connTLSNoneFail.search( logLine ) if connNum != None: connNum = connNum.group(1) try: openedConnections[connNum]["TLSNoneFail"] += 1 stats["TLSNoneFail"] += 1 continue except KeyError: ''' Do Nothing ''' self.timer( "NoEncryptSuccessLoop" ) for connNum, conn in openedConnections.items(): if conn["SSLSuccess"] == 0 and conn["TLSSuccess"] == 0: openedConnections[connNum]["NoEncryptSuccess"] += 1 stats["NoEncryptConns"] += 1 self.timer( "NoEncryptSuccessLoop" ) if self.debug: print str( processedLines ) + " actually processed" return ( stats, openedConnections, lastDateTime ) arglen = len( sys.argv ) if arglen == 2: lls = LdapLogStats( sys.argv[1] ) elif arglen == 3: lls = LdapLogStats( sys.argv[1], sys.argv[2] ) elif arglen == 4: lls = LdapLogStats( sys.argv[1], sys.argv[2], sys.argv[3] ) else: lls = LdapLogStats( )