Add logstash support (#1852)

* add logstash support

.gitignore

@@ -5,6 +5,10 @@
 /.project
 .docker-sync
 /jenkins/jenkins_home
+
+/logstash/pipeline/*.conf
+/logstash/config/pipelines.yml
+
 /nginx/ssl/*.crt
 /nginx/ssl/*.key
 /nginx/ssl/*.csr

docker-compose.yml

@@ -602,6 +602,24 @@ services:
         - frontend
         - backend
 
+### Logstash ##############################################
+    logstash:
+      build: ./logstash
+      volumes:
+        - './logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml'
+        - './logstash/pipeline:/usr/share/logstash/pipeline'
+      ports:
+        - '5001:5001'
+      environment:
+        LS_JAVA_OPTS: '-Xmx1g -Xms1g'
+      env_file:
+        - .env
+      networks:
+        - frontend
+        - backend
+      depends_on:
+        - elasticsearch
+
 ### Kibana ##############################################
     kibana:
       build: ./kibana
@@ -742,7 +760,7 @@ services:
       networks:
         - frontend
         - backend
-        
+
 ### Solr ################################################
     solr:
       build:
@@ -904,15 +922,15 @@ services:
         context: ./gitlab
       environment:
         GITLAB_OMNIBUS_CONFIG: |
-          external_url '${GITLAB_DOMAIN_NAME}' 
+          external_url '${GITLAB_DOMAIN_NAME}'
-          redis['enable'] = false 
+          redis['enable'] = false
-          nginx['listen_https'] = false 
+          nginx['listen_https'] = false
-          nginx['listen_port'] = 80 
+          nginx['listen_port'] = 80
-          postgresql['enable'] = false 
+          postgresql['enable'] = false
-          gitlab_rails['trusted_proxies'] = ['caddy','nginx','apache2'] 
+          gitlab_rails['trusted_proxies'] = ['caddy','nginx','apache2']
-          gitlab_rails['redis_host'] = 'redis' 
+          gitlab_rails['redis_host'] = 'redis'
           gitlab_rails['redis_database'] = 8
-          gitlab_rails['db_host'] = 'postgres' 
+          gitlab_rails['db_host'] = 'postgres'
           gitlab_rails['db_username'] = 'laradock_gitlab'
           gitlab_rails['db_password'] = 'laradock_gitlab'
           gitlab_rails['db_database'] = 'laradock_gitlab'
@@ -939,7 +957,7 @@ services:
         - ${DATA_PATH_HOST}/gitlab/runner:/etc/gitlab-runner
         - /var/run/docker.sock:/var/run/docker.sock:rw
       restart: always
-      
+
 ### JupyterHub #########################################
     jupyterhub:
       build:
@@ -1050,7 +1068,7 @@ services:
         - backend
       depends_on:
         - mongo
-  
+
 ### Metabase #################################################
     metabase:
       image: metabase/metabase:latest

logstash/Dockerfile

@@ -0,0 +1,10 @@
+FROM docker.elastic.co/logstash/logstash:6.4.2
+
+USER root
+RUN rm -f /usr/share/logstash/pipeline/logstash.conf
+RUN curl -L -o /usr/share/logstash/lib/mysql-connector-java-5.1.47.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.47/mysql-connector-java-5.1.47.jar
+ADD ./pipeline/ /usr/share/logstash/pipeline/
+ADD ./config/ /usr/share/logstash/config/
+
+RUN logstash-plugin install logstash-input-jdbc
+

logstash/config/logstash.yml

@@ -0,0 +1,5 @@
+http.host: "0.0.0.0"
+
+xpack.monitoring.enabled: false
+config.reload.automatic: true
+path.config: "/usr/share/logstash/pipeline"