Automated Deployment with Ansible¶

---
- name: Setup Rootless Docker User
  hosts: all
  become: yes

  vars_prompt:
    - name: service_user_name
      prompt: "Enter the new service username (e.g., tao, mmu)"
      private: no
    - name: service_user_home_base
      prompt: "Enter the base directory for the user home"
      default: "/srv"
      private: no

  vars:
    service_user_home: "{{ service_user_home_base }}/{{ service_user_name }}"
    docker_packages:
      - docker-ce
      - docker-ce-cli
      - containerd.io
      - docker-buildx-plugin
      - docker-compose-plugin
      - docker-ce-rootless-extras
      - uidmap
      - dbus-user-session
      - slirp4netns

  tasks:
    - name: Ensure Docker and dependencies are installed
      ansible.builtin.apt:
        name: "{{ docker_packages }}"
        state: present
        update_cache: yes

    - name: Stop and disable rootful Docker service/socket
      ansible.builtin.systemd:
        name: "{{ item }}"
        state: stopped
        enabled: no
      loop:
        - docker.service
        - docker.socket
      ignore_errors: yes

    - name: Remove rootful Docker socket file
      ansible.builtin.file:
        path: /var/run/docker.sock
        state: absent

    - name: Create the dedicated service user
      ansible.builtin.user:
        name: "{{ service_user_name }}"
        home: "{{ service_user_home }}"
        shell: /bin/bash
        system: yes
        create_home: yes

    - name: Get User UID
      ansible.builtin.command: "id -u {{ service_user_name }}"
      register: user_uid
      changed_when: false

    - name: Check if user already has subuid mapping
      ansible.builtin.shell: "grep '^{{ service_user_name }}:' /etc/subuid"
      register: subuid_check
      failed_when: false
      changed_when: false

    - name: Calculate next start ID for subuid/subgid
      ansible.builtin.shell: |
        awk -F: '{print $2 + $3}' /etc/subuid /etc/subgid 2>/dev/null | sort -n | tail -1
      register: next_id_cmd
      when: subuid_check.rc != 0
      changed_when: false

    - name: Set next start ID fact (fallback to 100000)
      ansible.builtin.set_fact:
        next_start_id: "{{ next_id_cmd.stdout | default('100000', true) }}"
      when: subuid_check.rc != 0

    - name: Add subuid and subgid ranges
      ansible.builtin.command: >
        usermod --add-subuids {{ next_start_id }}-{{ next_start_id | int + 65535 }}
        --add-subgids {{ next_start_id }}-{{ next_start_id | int + 65535 }} {{ service_user_name }}
      when: subuid_check.rc != 0

    - name: Ensure ip_tables module is loaded
      ansible.builtin.command: modprobe ip_tables
      register: modprobe_res
      failed_when: false
      changed_when: false

    - name: Fallback to insmod if modprobe failed (Ubuntu 24.04 bug workaround)
      ansible.builtin.command: "insmod /lib/modules/{{ ansible_kernel }}/kernel/net/ipv4/netfilter/ip_tables.ko.zst"
      when: modprobe_res.rc != 0
      failed_when: false
      changed_when: false

    - name: Enable linger for the service user (required for systemd user session)
      ansible.builtin.command: "loginctl enable-linger {{ service_user_name }}"

    - name: Install Rootless Docker
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.shell: |
        export XDG_RUNTIME_DIR=/run/user/{{ user_uid.stdout }}
        export DBUS_SESSION_BUS_ADDRESS=unix:path=${XDG_RUNTIME_DIR}/bus
        export PATH=/usr/bin:$PATH
        dockerd-rootless-setuptool.sh install
      args:
        creates: "{{ service_user_home }}/.config/systemd/user/docker.service"

    - name: Enable and start Rootless Docker service
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.shell: |
        export XDG_RUNTIME_DIR=/run/user/{{ user_uid.stdout }}
        export DBUS_SESSION_BUS_ADDRESS=unix:path=${XDG_RUNTIME_DIR}/bus
        systemctl --user enable --now docker

    - name: Configure .bashrc with Docker variables
      ansible.builtin.blockinfile:
        path: "{{ service_user_home }}/.bashrc"
        marker: "# {mark} ANSIBLE MANAGED BLOCK - DOCKER ROOTLESS"
        block: |
          export PATH=/usr/bin:$PATH
          export DOCKER_HOST=unix:///run/user/{{ user_uid.stdout }}/docker.sock
          if [[ $- == *i* ]] && [ -d "$HOME/jupyterlab" ]; then
              cd ~/jupyterlab
          fi

---
- name: Setup Carto-Lab Jupyter Instance
  hosts: all
  become: yes

  vars_prompt:
    - name: service_user_name
      prompt: "Enter the service username (e.g., tao, dre, mmu)"
      private: no
    - name: jupyter_port
      prompt: "Enter local Jupyter web port (e.g., 9288)"
      private: no
    - name: jupyter_password
      prompt: "Enter the Jupyter password (leave blank for auto-generation)"
      private: yes
    - name: git_user_name
      prompt: "Enter Git user name for container"
      default: "Jupyter Container Bot"
      private: no
    - name: git_user_email
      prompt: "Enter Git user email for container"
      default: "bot@example.com"
      private: no
    - name: git_token
      prompt: "Enter Git access token (leave blank if public clone works)"
      private: yes

  vars:
    service_user_home: "/srv/{{ service_user_name }}"
    git_repo: "https://{{ 'oauth2:' + git_token + '@' if git_token != '' else '' }}github.com/ioer-dresden/carto-lab-docker.git"

    # Auto-generate a password if the prompt was left empty
    final_jupyter_password: "{{ jupyter_password | default(lookup('password', '/dev/null chars=ascii_letters,digits length=16'), true) }}"

  pre_tasks:
    - name: Prompt for Web URL dynamically
      ansible.builtin.pause:
        prompt: "Enter the public URL (leave blank to default to https://{{ service_user_name }}.example.com)"
      register: prompt_web_url
      run_once: true

    - name: Set final_web_url fact
      ansible.builtin.set_fact:
        final_web_url: "{{ prompt_web_url.user_input if prompt_web_url.user_input != '' else 'https://' + service_user_name + '.example.com' }}"

  tasks:
    - name: Get User UID
      ansible.builtin.command: "id -u {{ service_user_name }}"
      register: user_uid
      changed_when: false

    - name: Ensure target directories exist
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        mode: '0755'
      loop:
        - "{{ service_user_home }}/notebooks"
        - "{{ service_user_home }}/envs"

    - name: Clone Carto-Lab repository
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.git:
        repo: "{{ git_repo }}"
        dest: "{{ service_user_home }}/jupyterlab"
        recursive: yes
        version: master
        update: no

    - name: Create .env file with specific variables
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.copy:
        dest: "{{ service_user_home }}/jupyterlab/.env"
        mode: '0644'
        content: |
          TAG=v1.1.0

          GIT_USER_NAME="{{ git_user_name }}"
          GIT_USER_EMAIL="{{ git_user_email }}"

          JUPYTER_PASSWORD={{ final_jupyter_password }}

          JUPYTER_NOTEBOOKS={{ service_user_home }}/notebooks
          JUPYTER_WEBPORT={{ jupyter_port }}

          JUPYTER_WEBURL={{ final_web_url }}

          CONDA_ENVS={{ service_user_home }}/envs

          COLLABORATIVE=true
          GENERATE_TOKEN=true
          JUPYTER_EXTRA_ARGS=--ServerApp.allow_origin='{{ final_web_url }}'

    - name: Ensure /srv/base exists
      ansible.builtin.file:
        path: /srv/base
        state: directory
        mode: '0755'

    - name: Ensure login_v2.html exists (prevents Docker mounting it as a folder)
      ansible.builtin.copy:
        dest: /srv/base/login_v2.html
        content: ""
        force: no
        mode: '0644'

    - name: Check for user SSH deploy key
      ansible.builtin.stat:
        path: "{{ service_user_home }}/.ssh/jupyter_deploy_key"
      register: deploy_key_stat

    - name: Check for user SSH config
      ansible.builtin.stat:
        path: "{{ service_user_home }}/.ssh/jupyter_ssh_config"
      register: ssh_config_stat

    - name: Create docker-compose.override.yml
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.copy:
        dest: "{{ service_user_home }}/jupyterlab/docker-compose.override.yml"
        mode: '0644'
        content: |
          services:
            jupyterlab:
              restart: unless-stopped
              volumes:
                - /srv/base/login_v2.html:/etc/jupyter/templates/login.html:ro
          {% if deploy_key_stat.stat.exists %}
                - {{ service_user_home }}/.ssh/jupyter_deploy_key:/root/.ssh/id_ed25519:ro
          {% endif %}
          {% if ssh_config_stat.stat.exists %}
                - {{ service_user_home }}/.ssh/jupyter_ssh_config:/root/.ssh/config:ro
          {% endif %}
              environment:
                - JUPYTER_AUTOSHUTDOWN_TIMEOUT=0
                - GIT_USER_NAME=${GIT_USER_NAME:-Jupyter Container Bot}
                - GIT_USER_EMAIL=${GIT_USER_EMAIL:-bot@example.com}
            telegraf:
              image: telegraf:alpine
              container_name: telegraf-{{ service_user_name }}
              restart: always
              user: "0:0"
              entrypoint: ["telegraf"]
              volumes:
                - {{ service_user_home }}/telegraf.conf:/etc/telegraf/telegraf.conf:ro
                - /run/user/{{ user_uid.stdout }}/docker.sock:/var/run/docker.sock:ro
                - {{ service_user_home }}:/user_home:ro
              environment:
                - JUPYTER_USER={{ service_user_name }}
              networks:
                - lbsn-network

    - name: Create docker network
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.shell: docker network create lbsn-network || true
      environment:
        DOCKER_HOST: "unix:///run/user/{{ user_uid.stdout }}/docker.sock"

    - name: Create Telegraf configuration securely outside of the Git repo
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.copy:
        dest: "{{ service_user_home }}/telegraf.conf"
        mode: '0600' # Restricted permissions since it holds a token
        content: |
          [agent]
            skip_processors_after_aggregators = true

          [[outputs.influxdb_v2]]
            urls = ["{{ influxdb_url }}"]
            token = "{{ influxdb_token }}"
            organization = "{{ influxdb_org }}"
            bucket = "{{ influxdb_bucket }}"

          [[inputs.docker]]
            interval = "60s"
            endpoint = "unix:///var/run/docker.sock"
            [inputs.docker.tags]
              user = "${JUPYTER_USER}"

          [[inputs.exec]]
            interval = "60s"
            commands = ["/usr/bin/du -s /user_home/notebooks"]
            data_format = "grok"
            grok_patterns = ["%{NUMBER:size_bytes:int}\t%{PATH:directory}"]
            name_override = "home_dir_usage"
            [inputs.exec.tags]
              user = "${JUPYTER_USER}"

    - name: Pull and Start Docker compose
      become: yes
      become_user: "{{ service_user_name }}"
      ansible.builtin.shell: |
        docker compose pull
        docker compose up -d
      args:
        chdir: "{{ service_user_home }}/jupyterlab"
      environment:
        DOCKER_HOST: "unix:///run/user/{{ user_uid.stdout }}/docker.sock"

    - name: Create daily cron job for resetting Docker Compose
      become: yes
      ansible.builtin.copy:
        dest: "/etc/cron.daily/reset_jupyter_{{ service_user_name }}"
        owner: root
        group: root
        mode: '0755'
        content: |
          #!/bin/sh

          # --- CONFIGURATION ---
          USER="{{ service_user_name }}"
          WORK_DIR="/srv/$USER/jupyterlab"

          # --- CHECK IF ON HOLD ---
          if [ -f "/srv/$USER/.disabled" ]; then
              echo "User $USER is on hold (/srv/$USER/.disabled exists). Skipping reset."
              exit 0
          fi

          # --- RESTART AS TARGET USER IF RUNNING AS ROOT ---
          if [ "$(id -u)" -eq 0 ]; then
              exec sudo -H -u "$USER" "$0" "$@"
              echo "This line is never reached."
          fi

          echo "Running as user $(id -un)"

          # --- RESET CONTAINERS ---
          cd "$WORK_DIR" || exit 1
          docker compose down
          docker compose up -d

          # --- PRUNE DOCKER ONLY IF CONTAINERS ARE RUNNING ---
          if docker compose ps --filter "status=running" --quiet | grep -q .; then
              echo "Containers are running — pruning unused Docker objects."
              docker system prune -a -f
          else
              echo "No containers running — skipping prune."
          fi

    # --- Local Nginx Configuration (Jupyter VM) ---
    - name: Create Nginx Site configuration
      ansible.builtin.template:
        src: "nginx_jupyter.conf.j2"
        dest: "/etc/nginx/sites-available/{{ service_user_name }}.example.com.conf"

    - name: Enable Nginx configuration
      ansible.builtin.file:
        src: "/etc/nginx/sites-available/{{ service_user_name }}.example.com.conf"
        dest: "/etc/nginx/sites-enabled/{{ service_user_name }}.example.com.conf"
        state: link
      notify: Reload Nginx

  handlers:
    - name: Reload Nginx
      ansible.builtin.service:
        name: nginx
        state: reloaded

# Top-level HTTP config for WebSocket headers
map $http_upgrade $connection_upgrade {
    default upgrade;
    ''      close;
}

# HTTP server to handle JupyterLab
server {
    listen 80;
    listen [::]:80;

    # Adapt to your domain structure
    server_name {{ service_user_name }}.example.com;

    access_log /var/log/nginx/{{ service_user_name }}.example.com-access.log;
    error_log /var/log/nginx/{{ service_user_name }}.example.com-error.log;

    client_max_body_size 100M;

    location / {
        # Security: allow only downstream nginx / reverse proxy
        # allow 192.168.1.50;
        # deny all;

        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # websocket headers
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_set_header X-Scheme $scheme;

        proxy_buffering off;

        # send traffic to jupyter/docker
        proxy_pass http://127.0.0.1:{{ jupyter_port }};
    }
}

---
- name: Reconcile and fix all Carto-Lab user deployments
  hosts: all
  become: yes

  tasks:
    - name: Ensure /srv/base exists
      ansible.builtin.file:
        path: /srv/base
        state: directory
        mode: '0755'

    - name: Ensure login_v2.html template exists
      ansible.builtin.copy:
        dest: /srv/base/login_v2.html
        content: ""
        force: no
        mode: '0644'

    - name: Discover all user directories in /srv
      ansible.builtin.find:
        paths: /srv
        file_type: directory
        recurse: no
      register: srv_dirs

    - name: Filter directories containing a Carto-Lab deployment
      ansible.builtin.stat:
        path: "{{ item.path }}/jupyterlab"
      loop: "{{ srv_dirs.files }}"
      register: compose_checks

    - name: Build list of all Carto-Lab users
      ansible.builtin.set_fact:
        all_cartolab_users: "{{ compose_checks.results | selectattr('stat.exists') | map(attribute='item.path') | map('basename') | list }}"

    - name: Check for .disabled flag file in each user home
      ansible.builtin.stat:
        path: "/srv/{{ item }}/.disabled"
      loop: "{{ all_cartolab_users }}"
      register: disabled_checks

    - name: Separate active and disabled users
      ansible.builtin.set_fact:
        disabled_users: "{{ disabled_checks.results | selectattr('stat.exists') | map(attribute='item') | list }}"
        active_users: "{{ disabled_checks.results | rejectattr('stat.exists') | map(attribute='item') | list }}"

    - name: Gather user UIDs
      ansible.builtin.getent:
        database: passwd

    - name: Check for user SSH deploy keys
      ansible.builtin.stat:
        path: "/srv/{{ item }}/.ssh/jupyter_deploy_key"
      loop: "{{ all_cartolab_users }}"
      register: user_deploy_keys

    - name: Check for user SSH configs
      ansible.builtin.stat:
        path: "/srv/{{ item }}/.ssh/jupyter_ssh_config"
      loop: "{{ all_cartolab_users }}"
      register: user_ssh_configs

    - name: Map SSH key presence per user
      ansible.builtin.set_fact:
        user_deploy_key_map: "{{ user_deploy_key_map | default({}) | combine({item.item: item.stat.exists}) }}"
      loop: "{{ user_deploy_keys.results }}"

    - name: Map SSH config presence per user
      ansible.builtin.set_fact:
        user_ssh_config_map: "{{ user_ssh_config_map | default({}) | combine({item.item: item.stat.exists}) }}"
      loop: "{{ user_ssh_configs.results }}"

    - name: Check if telegraf.conf is mistakenly a directory
      ansible.builtin.stat:
        path: "/srv/{{ item }}/telegraf.conf"
      loop: "{{ all_cartolab_users }}"
      register: telegraf_conf_stats

    - name: Remove telegraf.conf if it was created as a directory
      ansible.builtin.file:
        path: "/srv/{{ item.item }}/telegraf.conf"
        state: absent
      loop: "{{ telegraf_conf_stats.results }}"
      when: item.stat.exists and item.stat.isdir

    - name: Ensure Telegraf configuration exists and is secured
      ansible.builtin.copy:
        dest: "/srv/{{ item }}/telegraf.conf"
        owner: "{{ item }}"
        group: "{{ item }}"
        mode: '0600'
        content: |
          [agent]
            skip_processors_after_aggregators = true

          [[outputs.influxdb_v2]]
            urls = ["{{ influxdb_url }}"]
            token = "{{ influxdb_token }}"
            organization = "{{ influxdb_org }}"
            bucket = "{{ influxdb_bucket }}"

          [[inputs.docker]]
            interval = "60s"
            endpoint = "unix:///var/run/docker.sock"
            [inputs.docker.tags]
              user = "${JUPYTER_USER}"

          [[inputs.exec]]
            interval = "60s"
            commands = ["/usr/bin/du -s /user_home/notebooks"]
            data_format = "grok"
            grok_patterns = ["%{NUMBER:size_bytes:int}\t%{PATH:directory}"]
            name_override = "home_dir_usage"
            [inputs.exec.tags]
              user = "${JUPYTER_USER}"
      loop: "{{ all_cartolab_users }}"

    - name: Update docker-compose.override.yml for all users
      ansible.builtin.copy:
        dest: "/srv/{{ item }}/jupyterlab/docker-compose.override.yml"
        owner: "{{ item }}"
        group: "{{ item }}"
        mode: '0644'
        content: |
          services:
            jupyterlab:
              restart: unless-stopped
              volumes:
                - /srv/base/login_v2.html:/etc/jupyter/templates/login.html:ro
          {% if user_deploy_key_map[item] %}
                - /srv/{{ item }}/.ssh/jupyter_deploy_key:/root/.ssh/id_ed25519:ro
          {% endif %}
          {% if user_ssh_config_map[item] %}
                - /srv/{{ item }}/.ssh/jupyter_ssh_config:/root/.ssh/config:ro
          {% endif %}
              environment:
                - JUPYTER_AUTOSHUTDOWN_TIMEOUT=0
                - GIT_USER_NAME=${GIT_USER_NAME:-Jupyter Container Bot}
                - GIT_USER_EMAIL=${GIT_USER_EMAIL:-bot@example.com}
            telegraf:
              image: telegraf:alpine
              container_name: telegraf-{{ item }}
              restart: always
              user: "0:0"
              entrypoint: ["telegraf"]
              volumes:
                - /srv/{{ item }}/telegraf.conf:/etc/telegraf/telegraf.conf:ro
                - /run/user/{{ ansible_facts.getent_passwd[item][1] }}/docker.sock:/var/run/docker.sock:ro
                - /srv/{{ item }}:/user_home:ro
              environment:
                - JUPYTER_USER={{ item }}
              networks:
                - lbsn-network
      loop: "{{ all_cartolab_users }}"

    - name: Ensure lbsn-network exists
      become: yes
      become_user: "{{ item }}"
      ansible.builtin.shell: docker network create lbsn-network || true
      environment:
        DOCKER_HOST: "unix:///run/user/{{ ansible_facts.getent_passwd[item][1] }}/docker.sock"
      loop: "{{ all_cartolab_users }}"

    - name: Recreate and start containers for ACTIVE users only
      become: yes
      become_user: "{{ item }}"
      ansible.builtin.shell: |
        docker compose up -d
      args:
        chdir: "/srv/{{ item }}/jupyterlab"
      environment:
        XDG_RUNTIME_DIR: "/run/user/{{ ansible_facts.getent_passwd[item][1] }}"
        DOCKER_HOST: "unix:///run/user/{{ ansible_facts.getent_passwd[item][1] }}/docker.sock"
      loop: "{{ active_users }}"

    - name: Ensure containers are STOPPED for DISABLED users
      become: yes
      become_user: "{{ item }}"
      ansible.builtin.shell: |
        docker compose down
      args:
        chdir: "/srv/{{ item }}/jupyterlab"
      environment:
        XDG_RUNTIME_DIR: "/run/user/{{ ansible_facts.getent_passwd[item][1] }}"
        DOCKER_HOST: "unix:///run/user/{{ ansible_facts.getent_passwd[item][1] }}/docker.sock"
      loop: "{{ disabled_users }}"