# Install ElasticSearch 1.4.2, LogStash 1.4.2, Kibana 3.1.2 on a CS2.2-SSD Cloudlet ($.03/hr) running Linux CentOS 6.6 64-bit

cloudscript elk_single_stack

version = _latest

result_template = elk_result_template

globals

elk_hostname = 'elk'

elk_instance_type = 'CS2.2-SSD' # 2GB RAM, 2 vCore(s), 50GB SSD, 10Gbps

elk_image_type = 'Linux CentOS 6.6 64-bit'

elk_slice_user = 'elk'

elasticsearch_version = 'elasticsearch-1.4.2'

logstash_version = 'logstash-1.4.2'

kibana_version = 'kibana-3.1.2'

# passwords setup

server_password = lib::random_password()

console_password = lib::random_password()

thread elk_setup

tasks = [elk_server_setup]

task elk_server_setup

#-----------------

# Create elk keys

#-----------------

# Create elk server root password key

/key/password elk_server_password_key read_or_create

key_group = _SERVER

password = server_password

# Create elk server console key

/key/password elk_server_console_key read_or_create

key_group = _CONSOLE

password = console_password

#----------------------------

# Create elk storage slice,

# bootstrap script and recipe

#-----------------------------

# Create storage slice keys

/key/token elk_slice_key read_or_create

username = elk_slice_user

# Create slice to store script in cloudstorage

/storage/slice elk_slice read_or_create

keys = [elk_slice_key]

# Create slice container to store script in cloudstorage

/storage/container elk_container => [elk_slice] read_or_create

slice = elk_slice

# Place script data in cloudstorage

/storage/object elk_bootstrap_object => [elk_slice, elk_container] read_or_create

container_name = 'elk_container'

file_name = 'bootstrap_elk.sh'

slice = elk_slice

content_data = elk_bootstrap_data

# Associate the cloudstorage object with the elk script

/orchestration/script elk_bootstrap_script => [elk_slice, elk_container, elk_bootstrap_object] read_or_create

data_uri = 'cloudstorage://elk_slice/elk_container/bootstrap_elk.sh'

script_type = _SHELL

encoding = _STORAGE

# Create the recipe and associate the script

/orchestration/recipe elk_bootstrap_recipe read_or_create

scripts = [elk_bootstrap_script]

#-----------------------

# Create the elk server

#-----------------------

/server/cloud elk_server read_or_create

hostname = '{{ elk_hostname }}'

image = '{{ elk_image_type }}'

service_type = '{{ elk_instance_type }}'

keys = [elk_server_password_key, elk_server_console_key]

recipes = [elk_bootstrap_recipe]

text_template elk_bootstrap_data

#!/bin/sh

# check if running as root

if [ "$EUID" -ne 0 ]

then echo "ERROR: must have root permissions to execute the commands"

exit

fi

#

# Install packages

#

# get latest package list

yum --quiet --assumeyes update

# install wget

yum --quiet --assumeyes install wget

# install iptable services

yum --quiet --assumeyes install iptables-services

# install httpd

yum --quiet --assumeyes install httpd

# install and setup Java

yum --quiet --assumeyes install java

export JRE_HOME=/usr/lib/jvm/jre

export PATH=$PATH:/usr/lib/jvm/jre/bin

# install and setup Elasticsearch

cd /opt/

wget https://download.elasticsearch.org/elasticsearch/elasticsearch/{{ elasticsearch_version }}.tar.gz

tar -xf {{ elasticsearch_version }}.tar.gz

ln -s /opt/{{ elasticsearch_version }} /opt/elasticsearch

# modify Elasticsearch config

echo "http.cors.enabled: true" >> /opt/elasticsearch/config/elasticsearch.yml

# start up Elasticsearch in the background

/opt/elasticsearch/bin/elasticsearch -d

# install and setup Logstash

cd /opt/

wget https://download.elasticsearch.org/logstash/logstash/{{ logstash_version }}.tar.gz

tar -xf {{ logstash_version }}.tar.gz

ln -s /opt/{{ logstash_version }} /opt/logstash

# prepare a logstash.conf file for Apache logs

cat <<\EOF>/opt/logstash/logstash.conf

input {

file {

path => "/tmp/access_log"

start_position => beginning

sincedb_path => "/opt/logstash/sincedb.log"

}

}

filter {

if [path] =~ "access" {

mutate { replace => { "type" => "apache_access" } }

grok {

match => { "message" => "%{COMBINEDAPACHELOG}" }

}

}

date {

match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]

}

}

output {

elasticsearch {

host => localhost

}

stdout { codec => rubydebug }

}

EOF

# start logstash

nohup /bin/bash /opt/logstash/bin/logstash agent -f /opt/logstash/logstash.conf -l /opt/logstash/logstash.log > /dev/null 2>&1 &

# install and setup Kibana

cd /opt/

wget https://download.elasticsearch.org/kibana/kibana/{{ kibana_version }}.tar.gz

tar -xf {{ kibana_version }}.tar.gz

# copy the contents of the extracted directory to your webserver root directory

cp -R /opt/{{ kibana_version }}/* /var/www/html

chown -R apache:apache /var/www/html

# change kibana conf

sed -r 's@ elasticsearch:.*@ elasticsearch: "http://"+window.location.hostname+":9200",@g' /var/www/html/config.js > /var/www/html/config_new.js

mv /var/www/html/config_new.js /var/www/html/config.js

# add firewall rules and restart iptables

NUM = iptables -L INPUT --line-numbers | tail -n1 |cut -c 1

iptables -I INPUT $NUM -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT

iptables -I INPUT $NUM -m state --state NEW -m tcp -p tcp --dport 9200 -j ACCEPT

service iptables save

service iptables restart

# start httpd

chkconfig --level 2345 httpd on

service httpd start

# autostart elasticsearch

echo '/opt/elasticsearch/bin/elasticsearch -d' >> /etc/rc.local.orig

echo '/bin/bash /opt/logstash/bin/logstash agent -f /opt/logstash/logstash.conf -l /opt/logstash/logstash.log &' >> /etc/rc.local.orig

chmod +x /etc/rc.local.orig

_eof

text_template elk_result_template

Your Elasticsearch + Logstash + Kibana is ready at the following IP address:

http://{{ elk_server.ipaddress_public }}/

Go to your server address and read the Kibana dashboard for further instructions.

_eof