bro-http_log.conf

########################
# logstash Configuration Files - Bro IDS Logs
# MIT License, so do what you want with it!
#
# For use with logstash, elasticsearch, and kibana to analyze logs
#
# Usage: Reference this config file for your instance of logstash to parse Bro http logs
#
# Limitations: Standard bro log delimiter is tab.
#
#######################

input {
  file {
    type => "bro-http_log"
    start_position => "end"
    sincedb_path => "/var/tmp/.bro_http_sincedb"

    #Edit the following path to reflect the location of your log files. You can also change the extension if you use something else
    path => "/opt/nsm/bro/logs/current/http.log"
  }
}

filter {

  #Let's get rid of those header lines; they begin with a hash
  if [message] =~ /^#/ {
    drop { }
  }

  #Now, using the csv filter, we can define the Bro log fields
  if [type] == "bro-http_log" {
    csv {

      #http.log:#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
      columns => ["ts","uid","id.orig_h","id.orig_p","id.resp_h","id.resp_p","trans_depth","method","host","uri","referrer","user_agent","request_body_len","response_body_len","status_code","status_msg","info_code","info_msg","filename","tags","username","password","proxied","orig_fuids","orig_mime_types","resp_fuids","resp_mime_types"]

      #If you use a custom delimiter, change the following value in between the quotes to your delimiter. Otherwise, leave the next line alone.
      separator => "	"
    }

    #Let's convert our timestamp into the 'ts' field, so we can use Kibana features natively
    date {
      match => [ "ts", "UNIX" ]
    }

    # add geoip attributes
    geoip {
      source => "id.orig_h"
      target => "orig_geoip"
    }
    geoip {
      source => "id.resp_h"
      target => "resp_geoip"
    }

    mutate {
      convert => [ "id.orig_p", "integer" ]
      convert => [ "id.resp_p", "integer" ]
      convert => [ "trans_depth", "integer" ]
      convert => [ "request_body_len", "integer" ]
      convert => [ "response_body_len", "integer" ]
      convert => [ "status_code", "integer" ]
      convert => [ "info_code", "integer" ]
      rename =>  [ "id.orig_h", "id_orig_host" ]
      rename =>  [ "id.orig_p", "id_orig_port" ]
      rename =>  [ "id.resp_h", "id_resp_host" ]
      rename =>  [ "id.resp_p", "id_resp_port" ]
    }
  }
}

output {
  # stdout { codec => rubydebug }
  elasticsearch { hosts => localhost }
}