From 4d803e617b9e06f328f4b62d8f2571921e4c3976 Mon Sep 17 00:00:00 2001 From: Andrey Bunyaev Date: Wed, 11 Nov 2020 09:47:25 +0400 Subject: [PATCH] Add ClickHouse --- clickhouse/Dockerfile | 53 ++ clickhouse/config.xml | 510 ++++++++++++++++++ .../docker-entrypoint-initdb.d/init-db.sh | 0 clickhouse/docker_related_config.xml | 12 + clickhouse/entrypoint.sh | 103 ++++ clickhouse/users.xml | 138 +++++ docker-compose.yml | 33 +- env-example | 16 +- 8 files changed, 863 insertions(+), 2 deletions(-) create mode 100644 clickhouse/Dockerfile create mode 100644 clickhouse/config.xml create mode 100644 clickhouse/docker-entrypoint-initdb.d/init-db.sh create mode 100644 clickhouse/docker_related_config.xml create mode 100644 clickhouse/entrypoint.sh create mode 100644 clickhouse/users.xml diff --git a/clickhouse/Dockerfile b/clickhouse/Dockerfile new file mode 100644 index 00000000..853c14df --- /dev/null +++ b/clickhouse/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:20.04 + +ARG CLICKHOUSE_VERSION=20.9.4.76 +ARG CLICKHOUSE_GOSU_VERSION=1.10 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + apt-transport-https \ + dirmngr \ + gnupg \ + && mkdir -p /etc/apt/sources.list.d \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 \ + && echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" > /etc/apt/sources.list.d/clickhouse.list \ + && apt-get update \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --allow-unauthenticated --yes --no-install-recommends \ + clickhouse-common-static=$CLICKHOUSE_VERSION \ + clickhouse-client=$CLICKHOUSE_VERSION \ + clickhouse-server=$CLICKHOUSE_VERSION \ + locales \ + tzdata \ + wget \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get clean + +ADD https://github.com/tianon/gosu/releases/download/$CLICKHOUSE_GOSU_VERSION/gosu-amd64 /bin/gosu + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +RUN mkdir /docker-entrypoint-initdb.d + +COPY docker_related_config.xml /etc/clickhouse-server/config.d/ +COPY config.xml /etc/clickhouse-server/config.xml +COPY entrypoint.sh /entrypoint.sh + +RUN chmod +x \ + /entrypoint.sh \ + /bin/gosu + +EXPOSE 9000 8123 9009 +VOLUME /var/lib/clickhouse + +ENV CLICKHOUSE_CONFIG /etc/clickhouse-server/config.xml +ENV CLICKHOUSE_USER ${CLICKHOUSE_USER} +ENV CLICKHOUSE_PASSWORD ${CLICKHOUSE_PASSWORD} + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/clickhouse/config.xml b/clickhouse/config.xml new file mode 100644 index 00000000..c1479eaa --- /dev/null +++ b/clickhouse/config.xml @@ -0,0 +1,510 @@ + + + + + + + + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + + + + diff --git a/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/clickhouse/docker-entrypoint-initdb.d/init-db.sh new file mode 100644 index 00000000..e69de29b diff --git a/clickhouse/docker_related_config.xml b/clickhouse/docker_related_config.xml new file mode 100644 index 00000000..a1563c88 --- /dev/null +++ b/clickhouse/docker_related_config.xml @@ -0,0 +1,12 @@ + + + :: + 0.0.0.0 + 1 + + + diff --git a/clickhouse/entrypoint.sh b/clickhouse/entrypoint.sh new file mode 100644 index 00000000..03bacb7a --- /dev/null +++ b/clickhouse/entrypoint.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" +if [ x"$UID" == x0 ]; then + USER="$(id -u clickhouse)" + GROUP="$(id -g clickhouse)" + gosu="gosu $USER:$GROUP" +else + USER="$(id -u)" + GROUP="$(id -g)" + gosu="" +fi + +# port is needed to check if clickhouse-server is ready for connections +HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" + +# get CH directories locations +DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" +LOG_DIR="$(dirname $LOG_PATH || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" + +for dir in "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +do + # check if variable not empty + [ -z "$dir" ] && continue + # ensure directories exist + if ! mkdir -p "$dir"; then + echo "Couldn't create necessary directory: $dir" + exit 1 + fi + + if [ x"$UID" == x0 ] && [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then + # ensure proper directories permissions + chown -R "$USER:$GROUP" "$dir" + elif [ "$(stat -c %u "$dir")" != "$USER" ]; then + echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + exit 1 + fi +done + + + +if [ -n "$(ls /docker-entrypoint-initdb.d/)" ]; then + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG & + pid="$!" + + # check if clickhouse is ready to accept connections + # will try to send ping clickhouse via http_port (max 12 retries, with 1 sec delay) + if ! wget --spider --quiet --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then + echo >&2 'ClickHouse init process failed.' + exit 1 + fi + + if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then + printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" + fi + + clickhouseclient=( clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD ) + + echo + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; cat "$f" | "${clickhouseclient[@]}" ; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${clickhouseclient[@]}"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'Finishing of ClickHouse init process failed.' + exit 1 + fi +fi + +# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments +if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then + exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" +fi + +# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image +exec "$@" diff --git a/clickhouse/users.xml b/clickhouse/users.xml new file mode 100644 index 00000000..fc5897c2 --- /dev/null +++ b/clickhouse/users.xml @@ -0,0 +1,138 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + HAHA + + + + ::/0 + + + + default + + + default + + + + + + + a = 1 + + + + + a + b < 1 or c - d > 5 + + + + + c = 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/docker-compose.yml b/docker-compose.yml index cb056aca..f6a5458f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -596,7 +596,38 @@ services: networks: - backend -### Redis ################################################ + ### ClickHouse ############################################# + clickhouse: + build: + context: ./clickhouse + args: + - CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} + - CLICKHOUSE_GOSU_VERSION=${CLICKHOUSE_GOSU_VERSION} + environment: + - CLICKHOUSE_USER=${CLICKHOUSE_USER} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD} + volumes: + - ${DATA_PATH_HOST}/clickhouse:/var/lib/clickhouse + - ${CLICKHOUSE_CUSTOM_CONFIG}:/etc/clickhouse-server/config.xml + - ${CLICKHOUSE_USERS_CUSTOM_CONFIG}:/etc/clickhouse-server/users.xml + - ${CLICKHOUSE_HOST_LOG_PATH}:/var/log/clickhouse + - ${CLICKHOUSE_ENTRYPOINT_INITDB}:/docker-entrypoint-initdb.d + links: + - workspace + ports: + - "${CLICKHOUSE_HTTP_PORT}:8123" + - "${CLICKHOUSE_CLIENT_PORT}:9000" + - "${CLICKHOUSE_NATIVE_PORT}:9009" + ulimits: + nproc: 65535 + nofile: + soft: 262144 + hard: 262144 + networks: +# - frontend + - backend + + ### Redis ################################################ redis: build: ./redis volumes: diff --git a/env-example b/env-example index 0d4ebe6c..336ed67b 100644 --- a/env-example +++ b/env-example @@ -39,7 +39,7 @@ COMPOSE_PROJECT_NAME=laradock # Select a PHP version of the Workspace and PHP-FPM containers (Does not apply to HHVM). # Accepted values: 7.4 - 7.3 - 7.2 - 7.1 - 7.0 - 5.6 -PHP_VERSION=7.3 +PHP_VERSION=7.4 ### Phalcon Version ########################################### @@ -325,6 +325,20 @@ MYSQL_PORT=3306 MYSQL_ROOT_PASSWORD=root MYSQL_ENTRYPOINT_INITDB=./mysql/docker-entrypoint-initdb.d +### CLICKHOUSE ################################################# + +CLICKHOUSE_VERSION=20.6 +CLICKHOUSE_GOSU_VERSION=1.10 +CLICKHOUSE_CUSTOM_CONFIG=./clickhouse/config.xml +CLICKHOUSE_USERS_CUSTOM_CONFIG=./clickhouse/users.xml +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD=HAHA +CLICKHOUSE_HTTP_PORT=8123 +CLICKHOUSE_CLIENT_PORT=9000 +CLICKHOUSE_NATIVE_PORT=9009 +CLICKHOUSE_ENTRYPOINT_INITDB=./clickhouse/docker-entrypoint-initdb.d +CLICKHOUSE_HOST_LOG_PATH=./logs/clickhouse + ### REDIS ################################################# REDIS_PORT=6379