Add Jupyterhub (#1686)

docker-compose.yml

@@ -722,6 +722,41 @@ services:
       ports:
         - 9010:9000
       
+### JupyterHub #########################################
+    jupyterhub:
+      build:
+        context: ./jupyterhub
+      depends_on:
+        - postgres
+        - jupyterhub-user
+      restart: always
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock:rw
+        - ${DATA_PATH_HOST}/jupyterhub/:/data
+        - ${JUPYTERHUB_CUSTOM_CONFIG}:/jupyterhub_config.py
+        - ${JUPYTERHUB_USER_DATA}:/user-data
+        - ${JUPYTERHUB_USER_LIST}:/userlist
+      networks:
+        - backend
+      ports:
+        - "${JUPYTERHUB_PORT}:80"
+      environment:
+        - TERM=xterm
+        - JUPYTERHUB_USER_DATA=${JUPYTERHUB_USER_DATA}
+        - JUPYTERHUB_POSTGRES_DB=${JUPYTERHUB_POSTGRES_DB}
+        - JUPYTERHUB_POSTGRES_USER=${JUPYTERHUB_POSTGRES_USER}
+        - JUPYTERHUB_POSTGRES_HOST=${JUPYTERHUB_POSTGRES_HOST}
+        - JUPYTERHUB_POSTGRES_PASSWORD=${JUPYTERHUB_POSTGRES_PASSWORD}
+        - JUPYTERHUB_OAUTH_CALLBACK_URL=${JUPYTERHUB_OAUTH_CALLBACK_URL}
+        - JUPYTERHUB_OAUTH_CLIENT_ID=${JUPYTERHUB_OAUTH_CLIENT_ID}
+        - JUPYTERHUB_OAUTH_CLIENT_SECRET=${JUPYTERHUB_OAUTH_CLIENT_SECRET}
+        - JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE=${JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE}
+    jupyterhub-user:
+      build:
+        context: ./jupyterhub
+        dockerfile: Dockerfile.user
+      command: ["sh", "-c", "echo \"build only\""]
+
 ### IPython #########################################
     ipython-controller:
       build:

env-example

@@ -360,6 +360,20 @@ SOLR_VERSION=5.5
 SOLR_PORT=8983
 SOLR_DATAIMPORTHANDLER_MYSQL=false
 
+### JUPYTERHUB ###############################################
+JUPYTERHUB_POSTGRES_HOST=postgres
+JUPYTERHUB_POSTGRES_USER=laradock_jupyterhub
+JUPYTERHUB_POSTGRES_PASSWORD=laradock_jupyterhub
+JUPYTERHUB_POSTGRES_DB=laradock_jupyterhub
+JUPYTERHUB_PORT=9991
+JUPYTERHUB_OAUTH_CALLBACK_URL=http://laradock:9991/hub/oauth_callback
+JUPYTERHUB_OAUTH_CLIENT_ID={GITHUB_CLIENT_ID}
+JUPYTERHUB_OAUTH_CLIENT_SECRET={GITHUB_CLIENT_SECRET}
+JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE=laradock_jupyterhub-user
+JUPYTERHUB_CUSTOM_CONFIG=./jupyterhub/jupyterhub_config.py
+JUPYTERHUB_USER_DATA=/jupyterhub
+JUPYTERHUB_USER_LIST=./jupyterhub/userlist
+
 ### IPYTHON ##################################################
 LARADOCK_IPYTHON_CONTROLLER_IP=127.0.0.1
 

jupyterhub/Dockerfile

@@ -0,0 +1,26 @@
+FROM python
+LABEL maintainer="ahkui <ahkui@outlook.com>"
+
+ENV JUPYTERHUB_USER_DATA ${JUPYTERHUB_USER_DATA}
+ENV JUPYTERHUB_POSTGRES_DB ${JUPYTERHUB_POSTGRES_DB}
+ENV JUPYTERHUB_POSTGRES_USER ${JUPYTERHUB_POSTGRES_USER}
+ENV JUPYTERHUB_POSTGRES_HOST ${JUPYTERHUB_POSTGRES_HOST}
+ENV JUPYTERHUB_POSTGRES_PASSWORD ${JUPYTERHUB_POSTGRES_PASSWORD}
+ENV JUPYTERHUB_OAUTH_CALLBACK_URL ${JUPYTERHUB_OAUTH_CALLBACK_URL}
+ENV JUPYTERHUB_OAUTH_CLIENT_ID ${JUPYTERHUB_OAUTH_CLIENT_ID}
+ENV JUPYTERHUB_OAUTH_CLIENT_SECRET ${JUPYTERHUB_OAUTH_CLIENT_SECRET}
+ENV JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE ${JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE}
+
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
+
+RUN apt update -yqq && \
+    apt-get install -y nodejs
+    
+RUN npm install -g configurable-http-proxy
+
+RUN pip install jupyterhub
+RUN pip install oauthenticator
+RUN pip install dockerspawner
+RUN pip install psycopg2 psycopg2-binary
+
+CMD ["sh", "-c", "jupyterhub upgrade-db && jupyterhub -f /jupyterhub_config.py"]

jupyterhub/Dockerfile.user

@@ -0,0 +1,72 @@
+FROM tensorflow/tensorflow:latest-gpu
+
+MAINTAINER ahkui <ahkui@outlook.com>
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python \
+        python-dev \
+        && \
+    apt-get autoremove -y && \
+    apt-get autoclean && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        wget \
+        git \
+        && \
+    apt-get autoremove -y && \
+    apt-get autoclean && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py && \
+    rm get-pip.py
+
+RUN python3 -m pip --quiet --no-cache-dir install \
+        Pillow \
+        h5py \
+        ipykernel \
+        jupyter \
+        notebook \
+        jupyterhub \
+        matplotlib \
+        numpy \
+        pandas \
+        scipy \
+        sklearn \
+        Flask \
+        gunicorn \
+        pymongo \
+        redis \
+        requests \
+        ipyparallel \
+        bs4 \
+        && \
+    python3 -m ipykernel.kernelspec
+
+RUN pip --no-cache-dir install \
+    https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.8.0-cp35-cp35m-linux_x86_64.whl
+
+RUN ln -s -f /usr/bin/python3 /usr/bin/python
+
+COPY start.sh /usr/local/bin/
+COPY start-notebook.sh /usr/local/bin/
+COPY start-singleuser.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/start.sh
+RUN chmod +x /usr/local/bin/start-notebook.sh
+RUN chmod +x /usr/local/bin/start-singleuser.sh
+
+RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \
+    mv tini /usr/local/bin/tini && \
+    chmod +x /usr/local/bin/tini
+
+# cleanup
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ENTRYPOINT ["tini", "--"]
+
+CMD ["start-notebook.sh"]
+
+

jupyterhub/jupyterhub_config.py

@@ -0,0 +1,121 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# Configuration file for JupyterHub
+import os
+
+c = get_config()
+
+def create_dir_hook(spawner):
+    username = spawner.user.name # get the username
+    volume_path = os.path.join('/user-data', username)
+    if not os.path.exists(volume_path):
+        # create a directory with umask 0755 
+        # hub and container user must have the same UID to be writeable
+        # still readable by other users on the system
+        os.mkdir(volume_path, 0o755)
+        os.chown(volume_path, 1000,100)
+        # now do whatever you think your user needs
+        # ...
+        pass
+
+# attach the hook function to the spawner
+c.Spawner.pre_spawn_hook = create_dir_hook
+
+# We rely on environment variables to configure JupyterHub so that we
+# avoid having to rebuild the JupyterHub container every time we change a
+# configuration parameter.
+
+# Spawn single-user servers as Docker containers
+c.JupyterHub.spawner_class = 'dockerspawner.DockerSpawner'
+
+# Spawn containers from this image
+c.DockerSpawner.image = os.environ['JUPYTERHUB_LOCAL_NOTEBOOK_IMAGE']
+
+# JupyterHub requires a single-user instance of the Notebook server, so we
+# default to using the `start-singleuser.sh` script included in the
+# jupyter/docker-stacks *-notebook images as the Docker run command when
+# spawning containers.  Optionally, you can override the Docker run command
+# using the DOCKER_SPAWN_CMD environment variable.
+spawn_cmd = os.environ.get('JUPYTERHUB_DOCKER_SPAWN_CMD', "start-singleuser.sh")
+c.DockerSpawner.extra_create_kwargs.update({ 'command': spawn_cmd })
+
+# Connect containers to this Docker network
+network_name = os.environ.get('JUPYTERHUB_NETWORK_NAME','laradock_backend')
+c.DockerSpawner.use_internal_ip = True
+c.DockerSpawner.network_name = network_name
+
+# Pass the network name as argument to spawned containers
+c.DockerSpawner.extra_host_config = { 'network_mode': network_name, 'runtime': 'nvidia' }
+# c.DockerSpawner.extra_host_config = { 'network_mode': network_name, "devices":["/dev/nvidiactl","/dev/nvidia-uvm","/dev/nvidia0"] }
+# Explicitly set notebook directory because we'll be mounting a host volume to
+# it.  Most jupyter/docker-stacks *-notebook images run the Notebook server as
+# user `jovyan`, and set the notebook directory to `/home/jovyan/work`.
+# We follow the same convention.
+# notebook_dir = os.environ.get('JUPYTERHUB_DOCKER_NOTEBOOK_DIR') or '/home/jovyan/work'
+notebook_dir = '/notebooks'
+c.DockerSpawner.notebook_dir = notebook_dir
+
+# Mount the real user's Docker volume on the host to the notebook user's
+# notebook directory in the container
+user_data = os.environ.get('JUPYTERHUB_USER_DATA','/jupyterhub')
+c.DockerSpawner.volumes = {
+    user_data+'/{username}': notebook_dir
+}
+
+c.DockerSpawner.extra_create_kwargs.update({ 'user': 'root'})
+
+# volume_driver is no longer a keyword argument to create_container()
+# c.DockerSpawner.extra_create_kwargs.update({ 'volume_driver': 'local' })
+# Remove containers once they are stopped
+c.DockerSpawner.remove_containers = True
+
+# For debugging arguments passed to spawned containers
+c.DockerSpawner.debug = True
+
+# User containers will access hub by container name on the Docker network
+c.JupyterHub.hub_ip = 'jupyterhub'
+c.JupyterHub.hub_port = 8000
+
+# TLS config
+c.JupyterHub.port = 80
+# c.JupyterHub.ssl_key = os.environ['SSL_KEY']
+# c.JupyterHub.ssl_cert = os.environ['SSL_CERT']
+
+# Authenticate users with GitHub OAuth
+c.JupyterHub.authenticator_class = 'oauthenticator.GitHubOAuthenticator'
+c.GitHubOAuthenticator.oauth_callback_url = os.environ['JUPYTERHUB_OAUTH_CALLBACK_URL']
+c.GitHubOAuthenticator.client_id = os.environ['JUPYTERHUB_OAUTH_CLIENT_ID']
+c.GitHubOAuthenticator.client_secret = os.environ['JUPYTERHUB_OAUTH_CLIENT_SECRET']
+
+# Persist hub data on volume mounted inside container
+data_dir = '/data'
+
+c.JupyterHub.cookie_secret_file = os.path.join(data_dir,
+    'jupyterhub_cookie_secret')
+
+print(os.environ)
+
+c.JupyterHub.db_url = 'postgresql://{user}:{password}@{host}/{db}'.format(
+    user=os.environ['JUPYTERHUB_POSTGRES_USER'],
+    host=os.environ['JUPYTERHUB_POSTGRES_HOST'],
+    password=os.environ['JUPYTERHUB_POSTGRES_PASSWORD'],
+    db=os.environ['JUPYTERHUB_POSTGRES_DB'],
+)
+
+# Whitlelist users and admins
+c.Authenticator.whitelist = whitelist = set()
+c.Authenticator.admin_users = admin = set()
+c.JupyterHub.admin_access = True
+pwd = os.path.dirname(__file__)
+with open(os.path.join(pwd, 'userlist')) as f:
+    for line in f:
+        if not line:
+            continue
+        parts = line.split()
+        name = parts[0]
+        print(name)
+        whitelist.add(name)
+        if len(parts) > 1 and parts[1] == 'admin':
+            admin.add(name)
+admin.add('laradock')

jupyterhub/start-notebook.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+set -e
+
+if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then
+  # launched by JupyterHub, use single-user entrypoint
+  exec /usr/local/bin/start-singleuser.sh $*
+else
+  . /usr/local/bin/start.sh jupyter notebook $*
+fi

jupyterhub/start-singleuser.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+set -e
+
+# set default ip to 0.0.0.0
+if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then
+  NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS"
+fi
+
+# handle some deprecated environment variables
+# from DockerSpawner < 0.8.
+# These won't be passed from DockerSpawner 0.9,
+# so avoid specifying --arg=empty-string
+# if [ ! -z "$NOTEBOOK_DIR" ]; then
+  # NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS"
+# fi
+if [ ! -z "$JPY_PORT" ]; then
+  NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS"
+fi
+if [ ! -z "$JPY_USER" ]; then
+  NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS"
+fi
+if [ ! -z "$JPY_COOKIE_NAME" ]; then
+  NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS"
+fi
+if [ ! -z "$JPY_BASE_URL" ]; then
+  NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS"
+fi
+if [ ! -z "$JPY_HUB_PREFIX" ]; then
+  NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS"
+fi
+if [ ! -z "$JPY_HUB_API_URL" ]; then
+  NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS"
+fi
+
+NOTEBOOK_ARGS=" --allow-root --notebook-dir='/notebooks' $NOTEBOOK_ARGS"
+
+. /usr/local/bin/start.sh jupyterhub-singleuser $NOTEBOOK_ARGS $@

jupyterhub/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+set -e
+
+exec sh -c "env PATH=$PATH $*"

jupyterhub/userlist

@@ -0,0 +1 @@
+laradock

postgres/docker-entrypoint-initdb.d/.gitignore

@@ -1 +1,2 @@
 *.sh
+!init_jupyterhub_db.sh

postgres/docker-entrypoint-initdb.d/init_jupyterhub_db.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy createdb.sh.example to createdb.sh
+# then uncomment then set database name and username to create you need databases
+#
+# example: .env POSTGRES_USER=appuser and need db name is myshop_db
+# 
+#    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+#        CREATE USER myuser WITH PASSWORD 'mypassword';
+#        CREATE DATABASE myshop_db;
+#        GRANT ALL PRIVILEGES ON DATABASE myshop_db TO myuser;
+#    EOSQL
+#
+# this sh script will auto run when the postgres container starts and the $DATA_PATH_HOST/postgres not found.
+#
+# 
+# psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+#     CREATE USER db1 WITH PASSWORD 'db1';
+#     CREATE DATABASE db1;
+#     GRANT ALL PRIVILEGES ON DATABASE db1 TO db1;
+# EOSQL
+# 
+# psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+#     CREATE USER db2 WITH PASSWORD 'db2';
+#     CREATE DATABASE db2;
+#     GRANT ALL PRIVILEGES ON DATABASE db2 TO db2;
+# EOSQL
+# 
+# psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+#     CREATE USER db3 WITH PASSWORD 'db3';
+#     CREATE DATABASE db3;
+#     GRANT ALL PRIVILEGES ON DATABASE db3 TO db3;
+# EOSQL
+# 
+### default database and user for jupyterhub ##############################################
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE USER laradock_jupyterhub WITH PASSWORD 'laradock_jupyterhub';
+    CREATE DATABASE laradock_jupyterhub;
+    GRANT ALL PRIVILEGES ON DATABASE laradock_jupyterhub TO laradock_jupyterhub;
+    ALTER ROLE laradock_jupyterhub CREATEROLE SUPERUSER;
+EOSQL