input { stdin { codec => json_lines { charset => "UTF-8" } } } filter { date { # Parse Eliot timestamp filed into the special @timestamp field Logstash # expects: match => [ "timestamp", "UNIX" ] target => ["@timestamp"] } } output { # Stdout output for debugging: stdout { codec => rubydebug } elasticsearch { # Documents in ElasticSearch are identified by tuples of (index, mapping # type, document_id). # References: # - http://logstash.net/docs/1.3.2/outputs/elasticsearch # - http://stackoverflow.com/questions/15025876/what-is-an-index-in-elasticsearch # We make the document id unique (for a specific index/mapping type pair) by # using the relevant Eliot fields. This means replaying messages will not # result in duplicates, as long as the replayed messages end up in the same # index (see below). document_id => "%{task_uuid}_%{task_level}" # By default logstash sets the index to include the current date. When we # get to point of replaying log files on startup for crash recovery we might # want to use the last modified date of the file instead of current date, # otherwise we'll get documents ending up in wrong index. #index => "logstash-%{+YYYY.MM.dd}" index_type => "Eliot" # In a centralized ElasticSearch setup we'd be specifying host/port # or some such. In this setup we run it ourselves: embedded => true } }