python实时监控logstash日志代码

实时读取logstash日志，有异常错误keywork即触发报警。

# /usr/bin/env python3# -*- coding: utf-8 -*-# __author__ = caozhi# create_time 2018-11-12,update_time 2018-11-15# version = 1.0# 录像高可用报警# 1 读取日志 使用游标移动# 2 线上业务日志文件会切割，切割后，读取上一个切割的日志import osimport sysimport jsonimport requestsimport timeimport recini = conf.ini’log_file = logstash.log’def readconf(): try: with open(cini, ’r+’) as f: CONF = json.load(f) except: CONF = {'seek': 0, 'inode': 922817, 'last_file': logstash.log'} writeconf(CONF=CONF) print(’conf.ini 配置文件缺失，自动创建一个新的配置文件’) return CONFdef writeconf(CONF): with open(cini, ’w+’) as e: json.dump(CONF, e)def read_log(log_file, seek): try: f = open(log_file, ’r’) except FileNotFoundError: f = open(logstash.log’, ’r’) seek = 0 print(’上一个文件读取失败了，请检查切割的日志文件’) except: print(’日志文件打开错误，退出程序’) sys.exit()f.seek(seek)line = f.readline()new_seek = f.tell()if new_seek == seek: print(’没有追加日志，退出程序’) sys.exit()while line: try: logstash = json.loads(line) except: CONF = {'seek': 0, 'inode': 922817, 'last_file': '/data/logs/lmrs/logstash.log'} writeconf(CONF=CONF) print(’json数据加载错误，重新创建一个新的配置文件’) sys.exit() #if ’’’re.search(time.strftime('%Y:%H:%M', time.localtime()), logstash.get(’log_time’)) and ’’’logstash.get(’rtype’) == 6 and logstash.get(’uri’) == ’/publish’ and logstash.get(’event’) == 0: if logstash.get(’rtype’) == 6 and logstash.get(’uri’) == ’/publish’ and logstash.get(’event’) == 0: value = 1 stream = logstash.get(’name’) print(’{} {}’.format(value, stream)) record(value=value, stream=stream) else: value = 0 stream = 0 line = f.readline()seek = f.tell()f.closereturn value, stream, seekdef record(value, stream): data = [] record = {} record[’metric’] = ’recording_high_availability_monitor’ record[’endpoint’] = os.uname()[1] record[’timestamp’] = int(time.time()) record[’step’] = 60 record[’value’] = value record[’counterType’] = ’GAUGE’ record[’Tags’] = ’{}={}’.format(int(time.time()), stream) data.append(record)if data: print(’这是data的json数据’) print(data) falcon_request = requests.post('http://127.0.0.1:1988/v1/push', data=json.dumps(data)) #falcon_request = requests.post('http://127.0.0.1:1988/v1/push', json=data) print(’json参数请求返回状态码为：’ + str(falcon_request.status_code)) print(’json参数请求返回为：’ + str(falcon_request.text))if __name__ == ’__main__’: print() print(’***************************************’) print(’本次执行脚本时间：{}’.format(time.strftime('%Y%m%d_%H%M', time.localtime()))) CONF = readconf() print(’first_CONF :{}’.format(CONF)) print(’NO1.log_file’,log_file) last_inode = CONF[’inode’] inode = os.stat(log_file).st_ino print(’last_inode: {} inode: {}’.format(last_inode, inode))if inode == last_inode: seek = CONF[’seek’] next_file = 0else: log_file = CONF[’last_file’] + time.strftime('-%Y%m%d_', time.localtime()) + str(time.strftime('%H%M', time.localtime()))[:-1] + ’0’ next_file = 1 seek = CONF[’seek’]print(’NO2.log_file’,log_file)value, stream, seek = read_log(log_file=log_file,seek=seek)if next_file: CONF[’seek’] = 0else: CONF[’seek’] = seekCONF[’inode’] = os.stat(logstash.log’).st_inowriteconf(CONF=CONF)print(’last_CONF :{}’.format(CONF))

补充知识：logstash 调用exec

我就废话不多说了，还是直接看代码吧！

[elk@Vsftp logstash]$ cat t3.conf input { stdin { } } filter { grok { match => [ 'message','(?m)s*%{TIMESTAMP_ISO8601:time}s*(?<Level>(S+)).*'] } date { match => ['time', 'yyyy-MM-dd HH:mm:ss,SSS'] } mutate { add_field =>['type','tailong'] add_field =>['messager','%{type}-%{message}'] remove_field =>['message'] }} output { if ([Level] == 'ERROR' or [messager] =~ 'Exception' ) and [messager] !~ '温金服务未连接' and [messager] !~ '调用温金代理系统接口错误' and [messager] !~ 'BusinessException' { exec { command => '/bin/smail.pl '%{messager}' '%{type}' ' } } stdout { codec =>rubydebug } } Vsftp:/root# cat /bin/smail.pl #!/usr/bin/perl use Net::SMTP;use HTTP::Date qw(time2iso str2time time2iso time2isoz); use Data::Dumper;use Getopt::Std;use vars qw($opt_d );getopts(’d:’);# mail_user should be your_mail@163.com $message= '@ARGV'; $env='$opt_d'; sub send_mail{ my $CurrTime = time2iso(time()); my $to_address = shift; my $mail_user = ’zhao.yangjian@163.com’; my $mail_pwd = ’xx’; my $mail_server = ’smtp.163.com’; my $from = 'From: $mail_usern'; my $subject = 'Subject: zjcap infon'; my $info = '$CurrTime--$message'; my $message = <<CONTENT; $infoCONTENT my $smtp = Net::SMTP->new($mail_server); $smtp->auth($mail_user, $mail_pwd) || die 'Auth Error! $!'; $smtp->mail($mail_user); $smtp->to($to_address); $smtp->data(); # begin the data $smtp->datasend($from); # set user $smtp->datasend($subject); # set subject $smtp->datasend('nn'); $smtp->datasend('$messagen'); # set content $smtp->dataend(); $smtp->quit();}; send_mail (’zhao.yangjian@163.com’); 2017-01-12 10:19:19,888 jjjjj Exception{ '@version' => '1', '@timestamp' => '2017-01-12T02:19:19.888Z', 'host' => 'Vsftp', 'time' => '2017-01-12 10:19:19,888', 'Level' => 'jjjjj', 'type' => 'tailong', 'messager' => 'tailong-2017-01-12 10:19:19,888 jjjjj Exception'}

以上这篇python实时监控logstash日志代码就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持好吧啦网。