python分析apache访问日志脚本分享

Arce

#!/usr/bin/env python 
# coding=utf-8 
   
#------------------------------------------------------ 
# Name:   Apache 日志分析脚本 
# Purpose:   此脚本只用来分析Apache的访问日志 
# Version:   2.0 
# Author:  LEO 
# Created:   2013-4-26 
# Modified:   2013-5-4 
# Copyright:  (c) LEO 2013 
#------------------------------------------------------ 
   
import sys 
import time 
   
#该类是用来打印格式 
class displayFormat(object): 
   
  def format_size(self,size): 
  '''格式化流量单位''' 
  KB = 1024   
  MB = 1048576  
  GB = 1073741824  
  TB = 1099511627776 
  if size >= TB : 
    size = str(size / TB) + 'T' 
  elif size < KB : 
    size = str(size) + 'B' 
  elif size >= GB and size < TB: 
    size = str(size / GB) + 'G' 
  elif size >= MB and size < GB : 
    size = str(size / MB) + 'M' 
  else : 
    size = str(size / KB) + 'K' 
  return size 
   
  formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s' 
   
  def transverse_line(self) : 
  '''输出横线''' 
  print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10) 
   
  def head(self): 
  '''输出头部信息''' 
  print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503') 
   
  def error_print(self) : 
  '''输出错误信息''' 
  print 
  print 'Usage : ' + sys.argv[0] + ' ApacheLogFilePath [Number]' 
  print 
  sys.exit(1) 
   
  def execut_time(self): 
  '''输出脚本执行的时间''' 
  print 
  print "Script Execution Time: %.3f second" % time.clock() 
  print 
   
#该类是用来生成主机信息的字典 
class hostInfo(object): 
   
  host_info = ['200','404','500','302','304','503','403','times','size'] 
   
  def __init__(self,host): 
  self.host = host = {}.fromkeys(self.host_info,0) 
   
  def increment(self,status_times_size,is_size): 
  '''该方法是用来给host_info中的各个值加1''' 
  if status_times_size == 'times': 
    self.host['times'] += 1 
  elif is_size: 
    self.host['size'] = self.host['size'] + status_times_size 
  else: 
    self.host[status_times_size] += 1 
   
  def get_value(self,value): 
  '''该方法是取到各个主机信息中对应的值''' 
  return self.host[value] 
   
#该类是用来分析文件 
class fileAnalysis(object): 
  def __init__(self): 
  '''初始化一个空字典''' 
  self.report_dict = {} 
  self.total_request_times,self.total_traffic,self.total_200,  
  self.total_404,self.total_500,self.total_403,self.total_302,  
  self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0 
   
  def split_eachline_todict(self,line): 
  '''分割文件中的每一行，并返回一个字典''' 
  split_line = line.split() 
  split_dict = {'remote_host':split_line[0],'status':split_line[-2],'bytes_sent':split_line[-1],} 
  return split_dict 
   
  def generate_log_report(self,logfile): 
  '''读取文件，分析split_eachline_todict方法生成的字典''' 
  for line in logfile: 
    try: 
    line_dict = self.split_eachline_todict(line) 
    host = line_dict['remote_host'] 
    status = line_dict['status'] 
    except ValueError : 
    continue 
    except IndexError : 
    continue 
   
    if host not in self.report_dict : 
    host_info_obj = hostInfo(host) 
    self.report_dict[host] = host_info_obj 
    else : 
    host_info_obj = self.report_dict[host] 
   
    host_info_obj.increment('times',False)   
    if status in host_info_obj.host_info :  
    host_info_obj.increment(status,False)  
    try: 
    bytes_sent = int(line_dict['bytes_sent'])  
    except ValueError: 
    bytes_sent = 0 
    host_info_obj.increment(bytes_sent,True) 
  return self.report_dict 
   
  def return_sorted_list(self,true_dict): 
  '''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序''' 
  for host_key in true_dict : 
    host_value = true_dict[host_key] 
    times = host_value.get_value('times')  
    self.total_request_times = self.total_request_times + times  
    size = host_value.get_value('size')  
    self.total_traffic = self.total_traffic + size  
   
    o200 = host_value.get_value('200') 
    o404 = host_value.get_value('404') 
    o500 = host_value.get_value('500') 
    o403 = host_value.get_value('403') 
    o302 = host_value.get_value('302') 
    o304 = host_value.get_value('304') 
    o503 = host_value.get_value('503') 
   
    true_dict[host_key] = {'200':o200,'404':o404,'500':o500,'403':o403,'302':o302,'304':o304,  
          '503':o503,'times':times,'size':size} 
   
    self.total_200 = self.total_200 + o200 
    self.total_404 = self.total_404 + o404 
    self.total_500 = self.total_500 + o500 
    self.total_302 = self.total_302 + o302 
    self.total_304 = self.total_304 + o304 
    self.total_503 = self.total_503 + o503 
   
  sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],t[1]['size']),reverse=True) 
  return sorted_list 
   
class Main(object): 
  def main(self) : 
  '''主调函数''' 
  display_format = displayFormat() 
  arg_length = len(sys.argv) 
  if arg_length == 1 : 
    display_format.error_print() 
  elif arg_length == 2 or arg_length == 3: 
    infile_name = sys.argv[1] 
    try : 
    infile = open(infile_name,'r') 
    if arg_length == 3 : 
      lines = int(sys.argv[2]) 
    else : 
      lines = 0 
    except IOError,e : 
    print 
    print e 
    display_format.error_print() 
    except ValueError : 
    print 
    print "Please Enter A Volid Number !!" 
    display_format.error_print() 
  else : 
    display_format.error_print() 
   
  fileAnalysis_obj = fileAnalysis() 
  not_true_dict = fileAnalysis_obj.generate_log_report(infile) 
  log_report = fileAnalysis_obj.return_sorted_list(not_true_dict) 
  total_ip = len(log_report) 
  if lines : 
    log_report = log_report[0:lines] 
  infile.close() 
   
  print 
  total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic) 
  total_request_times = fileAnalysis_obj.total_request_times 
  print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d' 
     % (total_ip,total_traffic,total_request_times) 
  print 
  display_format.head() 
  display_format.transverse_line() 
   
  for host in log_report : 
    times = host[1]['times'] 
    times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100 
    print display_format.formatstring % (host[0], 
             display_format.format_size(host[1]['size']), 
             times,str(times_percent)[0:5], 
             host[1]['200'],host[1]['404'], 
             host[1]['500'],host[1]['403'], 
             host[1]['302'],host[1]['304'],host[1]['503']) 
               
  if (not lines) or total_ip == lines : 
    display_format.transverse_line() 
    print display_format.formatstring % (total_ip,total_traffic,  
             total_request_times,'100%', 
             fileAnalysis_obj.total_200, 
             fileAnalysis_obj.total_404, 
             fileAnalysis_obj.total_500,  
             fileAnalysis_obj.total_403, 
             fileAnalysis_obj.total_302,  
             fileAnalysis_obj.total_304, 
             fileAnalysis_obj.total_503) 
  display_format.execut_time() 
   
if __name__ == '__main__': 
  main_obj = Main() 
  main_obj.main()