JustPaste.it - Share Text & Images the Easy Way

For the command:

ubuntu@ip-172-31-25-208:~$ echo "2015-08-22T20:59:30.322751Z External-LB-Gateway 106.216.129.73:25217 10.131.6.233:9088 0.01728 0.000015 0.000023 - - 266 41 "- - - " "-" - -" | java -jar logstash-1.3.3-flatjar.jar agent -f test.conf -v -- web

output is:

Match data {:match=>{"message"=>"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} %{IP:backend_ip}:%{NUMBER:backend_port:int} %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{NUMBER:elb_status_code:int} %{NUMBER:backend_status_code:int} %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} %{QS:request}"}, :level=>:info}
Grok compile {:field=>"message", :patterns=>["%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} %{IP:backend_ip}:%{NUMBER:backend_port:int} %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{NUMBER:elb_status_code:int} %{NUMBER:backend_status_code:int} %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} %{QS:request}"], :level=>:info}
filter received {:event=>#<LogStash::Event:0x390261c8 @cancelled=false, @data={"message"=>"2015-08-22T20:59:30.322751Z External-LB-Gateway 106.216.129.73:25217 10.131.6.233:9088 0.01728 0.000015 0.000023 - - 266 41 - - - - - -", "@version"=>"1", "@timestamp"=>"2015-10-30T21:11:36.351Z", "host"=>"ip-172-31-25-208"}>, :level=>:info}
Pipeline started {:level=>:info}
Plugin is finished {:plugin=><LogStash::Filters::Grok match=>{"message"=>"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} %{IP:backend_ip}:%{NUMBER:backend_port:int} %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{NUMBER:elb_status_code:int} %{NUMBER:backend_status_code:int} %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} %{QS:request}"}, tag_on_failure=>["_grokparsefailure"]>, :level=>:info}
Plugin is finished {:plugin=><LogStash::Filters::Date match=>["ISO8601"], target=>"@timestamp">, :level=>:info}
Starting embedded ElasticSearch local node. {:level=>:info}
log4j, [2015-10-30T21:11:37.352] INFO: org.elasticsearch.node: [Stardust] version[0.90.9], pid[15228], build[a968646/2013-12-23T10:35:28Z]
log4j, [2015-10-30T21:11:37.353] INFO: org.elasticsearch.node: [Stardust] initializing ...
log4j, [2015-10-30T21:11:37.358] INFO: org.elasticsearch.plugins: [Stardust] loaded [], sites []
log4j, [2015-10-30T21:11:38.892] INFO: org.elasticsearch.node: [Stardust] initialized
log4j, [2015-10-30T21:11:38.892] INFO: org.elasticsearch.node: [Stardust] starting ...
log4j, [2015-10-30T21:11:38.988] INFO: org.elasticsearch.transport: [Stardust] bound_address {inet[/0:0:0:0:0:0:0:0:9302]}, publish_address {inet[/172.31.25.208:9302]}
log4j, [2015-10-30T21:11:42.034] INFO: org.elasticsearch.cluster.service: [Stardust] new_master [Stardust][vmkpXZVMRYiQ_uiaMRhSLw][inet[/172.31.25.208:9302]], reason: zen-disco-join (elected_as_master)
log4j, [2015-10-30T21:11:42.059] INFO: org.elasticsearch.discovery: [Stardust] elasticsearch/vmkpXZVMRYiQ_uiaMRhSLw
log4j, [2015-10-30T21:11:42.075] INFO: org.elasticsearch.http: [Stardust] bound_address {inet[/0:0:0:0:0:0:0:0:9201]}, publish_address {inet[/172.31.25.208:9201]}
log4j, [2015-10-30T21:11:42.076] INFO: org.elasticsearch.node: [Stardust] started
New ElasticSearch output {:cluster=>nil, :host=>"localhost", :port=>"9300-9305", :embedded=>true, :level=>:info}
log4j, [2015-10-30T21:11:42.121] INFO: org.elasticsearch.node: [Peepers] version[0.90.9], pid[15228], build[a968646/2013-12-23T10:35:28Z]
log4j, [2015-10-30T21:11:42.121] INFO: org.elasticsearch.node: [Peepers] initializing ...
log4j, [2015-10-30T21:11:42.122] INFO: org.elasticsearch.plugins: [Peepers] loaded [], sites []
log4j, [2015-10-30T21:11:42.361] INFO: org.elasticsearch.node: [Peepers] initialized
log4j, [2015-10-30T21:11:42.362] INFO: org.elasticsearch.node: [Peepers] starting ...
log4j, [2015-10-30T21:11:42.373] INFO: org.elasticsearch.transport: [Peepers] bound_address {inet[/0:0:0:0:0:0:0:0:9303]}, publish_address {inet[/172.31.25.208:9303]}
log4j, [2015-10-30T21:11:42.484] INFO: org.elasticsearch.gateway: [Stardust] recovered [1] indices into cluster_state
log4j, [2015-10-30T21:11:46.137] WARN: org.elasticsearch.discovery.zen.ping.unicast: [Peepers] failed to send ping to [[#zen_unicast_1#][inet[localhost/127.0.0.1:9300]]]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [][inet[localhost/127.0.0.1:9300]][discovery/zen/unicast] request_id [0] timed out after [3750ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
log4j, [2015-10-30T21:11:46.137] WARN: org.elasticsearch.discovery.zen.ping.unicast: [Peepers] failed to send ping to [[#zen_unicast_2#][inet[localhost/127.0.0.1:9301]]]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [][inet[localhost/127.0.0.1:9301]][discovery/zen/unicast] request_id [1] timed out after [3750ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
log4j, [2015-10-30T21:11:46.909] INFO: org.elasticsearch.cluster.service: [Stardust] added {[Peepers][qPl5na9uS_y51kT8oU6nUA][inet[/172.31.25.208:9303]]{client=true, data=false},}, reason: zen-disco-receive(join from node[[Peepers][qPl5na9uS_y51kT8oU6nUA][inet[/172.31.25.208:9303]]{client=true, data=false}])
log4j, [2015-10-30T21:11:46.916] INFO: org.elasticsearch.cluster.service: [Peepers] detected_master [Stardust][vmkpXZVMRYiQ_uiaMRhSLw][inet[/172.31.25.208:9302]], added {[Stardust][vmkpXZVMRYiQ_uiaMRhSLw][inet[/172.31.25.208:9302]],}, reason: zen-disco-receive(from master [[Stardust][vmkpXZVMRYiQ_uiaMRhSLw][inet[/172.31.25.208:9302]]])
log4j, [2015-10-30T21:11:46.917] INFO: org.elasticsearch.discovery: [Peepers] elasticsearch/qPl5na9uS_y51kT8oU6nUA
log4j, [2015-10-30T21:11:46.918] INFO: org.elasticsearch.node: [Peepers] started
Automatic template management enabled {:manage_template=>"true", :level=>:info}
Found existing Logstash template match. {:has_template=>true, :name=>"logstash-*", :alt=>"logstash*", :templates=>"{\"logstash\"=>\"logstash-*\"}", :level=>:info}
output received {:event=>#<LogStash::Event:0x390261c8 @cancelled=false, @data={"message"=>"2015-08-22T20:59:30.322751Z External-LB-Gateway 106.216.129.73:25217 10.131.6.233:9088 0.01728 0.000015 0.000023 - - 266 41 - - - - - -", "@version"=>"1", "@timestamp"=>"2015-10-30T21:11:36.351Z", "host"=>"ip-172-31-25-208"}>, :level=>:info}
Plugin is finished {:plugin=><LogStash::Outputs::ElasticSearch index=>"logstash-%{+YYYY.MM.dd}", template_name=>"logstash", port=>"9300-9305", embedded_http_port=>"9200-9300", protocol=>"node">, :level=>:info}
Pipeline shutdown complete. {:level=>:info}

where my **test.conf** is:

input { stdin { } }
filter {
if [type] == "elb" {
grok {
match => [ "message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} %{IP:backend_ip}:%{NUMBER:backend_port:int} %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{NUMBER:elb_status_code:int} %{NUMBER:backend_status_code:int} %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} %{QS:request}" ]
}
date {
match => [ "timestamp", "ISO8601" ]
}
}
}
output {
elasticsearch { embedded => true }
}