Linux:
bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
bin/kafka-server-start.sh -daemon config/server.properties
Windows:
bin\windows\zookeeper-server-start.bat config\zookeeper.properties
bin\windows\kafka-server-start.bat config\server.properties
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
- Edit file connect-file-source.properties
name=local-file-source
connector.class=FileStreamSource
tasks.max=1
file=/tmp/myfile.txt
topic=test2
- Edit file connect-file-sink.properties
name=local-file-sink
connector.class=FileStreamSink
tasks.max=1
file=/tmp/myout.txt
topics=test2
--------------------------------------
bin/connect-standalone.sh config/connect-standalone.properties config/connect-file-source.properties config/connect-file-sink.properties
--------------------------------------
bin\windows\connect-standalone.bat config/connect-standalone.properties config/connect-file-source.properties config/connect-file-sink.properties
---------------------------------------
echo Sunday >>/tmp/myfile.txt
------------------------------------
filebeat.prospectors:
- input_type: log
paths:
- /tmp/myfile.txt
document_type: myexample
exclude_files: ['\.gz$']
ignore_older: 2m
close_inactive: 1m
fields:
env: SCHOOL
fields_under_root: true
tail_files: true
output.kafka:
hosts: ["localhost:9092"]
topic: "test_log"
partition.round_robin:
reachable_only: false
required_acks: 0
compression: gzip
max_message_bytes: 30000
--------------
input {
kafka {
bootstrap_servers=>"localhost:9092"
topics=>["test_metric"]
type=>"metricsets"
}
kafka {
bootstrap_servers=>"localhost:9092"
topics=>["test_log"]
type=>"myexample"
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "%{type}-%{+YYYY.MM.dd}"
}
}
-----------------------------
output.kafka:
hosts: ["localhost:9092"]
topic: "test_metric"
partition.round_robin:
reachable_only: false
required_acks: 0
compression: gzip
max_message_bytes: 30000
www.griddlers.net/images/rasto/Mimacom_Openslava_dataprocessing.pptx