From 7430a34dd5516e9ca707c1d37effdcddc140ebe8 Mon Sep 17 00:00:00 2001
From: Andrew Siegman <blerrnk@gmail.com>
Date: Sat, 9 Feb 2019 02:40:14 -0600
Subject: [PATCH] Add logstash support (#1852)

* add logstash support
---
 .gitignore                   |  4 ++++
 docker-compose.yml           | 40 ++++++++++++++++++++++++++----------
 logstash/Dockerfile          | 10 +++++++++
 logstash/config/logstash.yml |  5 +++++
 logstash/pipeline/.gitkeep   |  0
 5 files changed, 48 insertions(+), 11 deletions(-)
 create mode 100644 logstash/Dockerfile
 create mode 100644 logstash/config/logstash.yml
 create mode 100644 logstash/pipeline/.gitkeep

diff --git a/.gitignore b/.gitignore
index 9888e50d..affb5c63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,10 @@
 /.project
 .docker-sync
 /jenkins/jenkins_home
+
+/logstash/pipeline/*.conf
+/logstash/config/pipelines.yml
+
 /nginx/ssl/*.crt
 /nginx/ssl/*.key
 /nginx/ssl/*.csr
diff --git a/docker-compose.yml b/docker-compose.yml
index bcc27631..7ed9f9f9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -602,6 +602,24 @@ services:
         - frontend
         - backend
 
+### Logstash ##############################################
+    logstash:
+      build: ./logstash
+      volumes:
+        - './logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml'
+        - './logstash/pipeline:/usr/share/logstash/pipeline'
+      ports:
+        - '5001:5001'
+      environment:
+        LS_JAVA_OPTS: '-Xmx1g -Xms1g'
+      env_file:
+        - .env
+      networks:
+        - frontend
+        - backend
+      depends_on:
+        - elasticsearch
+
 ### Kibana ##############################################
     kibana:
       build: ./kibana
@@ -742,7 +760,7 @@ services:
       networks:
         - frontend
         - backend
-        
+
 ### Solr ################################################
     solr:
       build:
@@ -904,15 +922,15 @@ services:
         context: ./gitlab
       environment:
         GITLAB_OMNIBUS_CONFIG: |
-          external_url '${GITLAB_DOMAIN_NAME}' 
-          redis['enable'] = false 
-          nginx['listen_https'] = false 
-          nginx['listen_port'] = 80 
-          postgresql['enable'] = false 
-          gitlab_rails['trusted_proxies'] = ['caddy','nginx','apache2'] 
-          gitlab_rails['redis_host'] = 'redis' 
+          external_url '${GITLAB_DOMAIN_NAME}'
+          redis['enable'] = false
+          nginx['listen_https'] = false
+          nginx['listen_port'] = 80
+          postgresql['enable'] = false
+          gitlab_rails['trusted_proxies'] = ['caddy','nginx','apache2']
+          gitlab_rails['redis_host'] = 'redis'
           gitlab_rails['redis_database'] = 8
-          gitlab_rails['db_host'] = 'postgres' 
+          gitlab_rails['db_host'] = 'postgres'
           gitlab_rails['db_username'] = 'laradock_gitlab'
           gitlab_rails['db_password'] = 'laradock_gitlab'
           gitlab_rails['db_database'] = 'laradock_gitlab'
@@ -939,7 +957,7 @@ services:
         - ${DATA_PATH_HOST}/gitlab/runner:/etc/gitlab-runner
         - /var/run/docker.sock:/var/run/docker.sock:rw
       restart: always
-      
+
 ### JupyterHub #########################################
     jupyterhub:
       build:
@@ -1050,7 +1068,7 @@ services:
         - backend
       depends_on:
         - mongo
-  
+
 ### Metabase #################################################
     metabase:
       image: metabase/metabase:latest
diff --git a/logstash/Dockerfile b/logstash/Dockerfile
new file mode 100644
index 00000000..a8c54527
--- /dev/null
+++ b/logstash/Dockerfile
@@ -0,0 +1,10 @@
+FROM docker.elastic.co/logstash/logstash:6.4.2
+
+USER root
+RUN rm -f /usr/share/logstash/pipeline/logstash.conf
+RUN curl -L -o /usr/share/logstash/lib/mysql-connector-java-5.1.47.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.47/mysql-connector-java-5.1.47.jar
+ADD ./pipeline/ /usr/share/logstash/pipeline/
+ADD ./config/ /usr/share/logstash/config/
+
+RUN logstash-plugin install logstash-input-jdbc
+
diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml
new file mode 100644
index 00000000..c3447173
--- /dev/null
+++ b/logstash/config/logstash.yml
@@ -0,0 +1,5 @@
+http.host: "0.0.0.0"
+
+xpack.monitoring.enabled: false
+config.reload.automatic: true
+path.config: "/usr/share/logstash/pipeline"
diff --git a/logstash/pipeline/.gitkeep b/logstash/pipeline/.gitkeep
new file mode 100644
index 00000000..e69de29b