Fix(domserver): docker compose mount point

Reviewed-on: #1

.gitea/workflows/lint.yml

@@ -0,0 +1,17 @@
+name: Ansible Playbook lint
+on: [push]
+
+jobs:
+  ansible-lint:
+    runs-on: imgbuilder
+    container:
+      image: gitea.konchin.com/image/ansible
+      credentials:
+        username: ${{ secrets.REGISTRY_USERNAME }}
+        password: ${{ secrets.REGISTRY_PASSWORD }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Ansible Lint
+        run: |
+          ansible-lint playbooks/ roles/ 

README.md

@@ -1 +1,29 @@
 # Ansible / domjudge
+
+## Usage
+
+0. Install Arch Linux for domserver and **Debian 12** for judgehost.
+1. Fill in the vars in `group_vars`.
+2. Fill in `domserver` and `judgehost` groups in `hosts` file.
+3. Run `ansible-playbook playbooks/domserver`.
+4. Run `ansible-playbook playbooks/judgehost`.
+5. Put web cert and key to `/etc/haproxy/cert.pem` on domserver.
+6. Check if judgehost been registered.
+
+## Trouble shooting
+
+### I give up
+
+Just use Debian 12 + docker.
+
+### Judgehost cannot startup
+
+The cgroups v2 support had been patched since October, 2024. However the latest
+release of domjudge is 8.3.1, which was published on September, 2024.
+
+Therefore, before cgroups v2 patch came out as a stable release, we still have
+to use a older release, like Debian 12, to make things work.
+
+### PHP syntax error
+
+PHP version on Debian 11 is too old, use Debian 12 instead.

group_vars/all/main.yml

@@ -1,3 +1,3 @@
 ---
-domjudge_base_dir: /opt/domjudge/midterm
+domjudge_base_dir: /opt/domjudge
-domserver_url: https://cp1.konchin.com
+domserver_url: https://cp1.konchin.com/

group_vars/judgehost/main.yml

@@ -0,0 +1,5 @@
+---
+judgehost_version: 8.3.1
+
+# For legacy support
+domjudge_tarball_url: https://www.domjudge.org/releases/domjudge-8.3.1.tar.gz

hosts

@@ -8,6 +8,7 @@ localhost ansible_connection=local
 [judgehost]
 10.4.2.227
 10.4.2.228
+10.4.2.229
 
 [all:vars]
-ansible_python_interpreter=/usr/bin/python
+ansible_python_interpreter=/usr/bin/python3

playbooks/domserver.yml

@@ -4,3 +4,4 @@
   roles:
     - role: install_packages
     - role: configure_domserver
+    - role: configure_haproxy

playbooks/judgehost-rebuild-chroot.yml

@@ -0,0 +1,9 @@
+- name: Rebuild chroot environment for judgehosts
+  hosts: judgehost
+  tasks:
+    - name: Run misc-tools/dj_make_chroot
+      ansible.builtin.command: |
+        ./misc-tools/dj_make_chroot -y -i kotlin
+      args:
+        chdir: "{{ domjudge_base_dir }}"
+      changed_when: true

playbooks/judgehost.yml

@@ -2,5 +2,6 @@
 - name: Install judgehost
   hosts: judgehost
   roles:
-    - role: install_packages
+    - role: install_packages_debian
+    # - role: configure_judgehost_docker
     - role: configure_judgehost

roles/configure_domserver/templates/docker-compose.yml.jinja

@@ -21,8 +21,11 @@ services:
       CONTAINER_TIMEZONE: Asia/Taipei
     restart: always
     volumes:
+      - "domserver_etc:/opt/domjudge/domserver/etc/"
       - "./config/php/domjudge.conf:/etc/php/8.2/fpm/pool.d/domjudge.conf"
     depends_on:
       - mariadb
     ports: 
       - "127.0.0.1:8080:80"
+volumes:
+  domserver_etc: {}

roles/configure_haproxy/files/haproxy.cfg

@@ -0,0 +1,52 @@
+global
+    default-path config
+    #zero-warning
+    maxconn     20000
+    log         127.0.0.1 local0
+    user        haproxy
+    #pidfile     /run/haproxy.pid
+    hard-stop-after 5m
+    daemon
+    user haproxy
+    group haproxy
+
+    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+    ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+    ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
+
+# default settings common to all HTTP proxies below
+defaults
+    mode http
+    log global
+    timeout client 1m
+    timeout server 1m
+    timeout connect 10s
+    timeout http-keep-alive 2m
+    timeout queue 15s
+    timeout tunnel 4h  # for websocket
+
+frontend external
+    bind :443 name secure ssl crt /etc/haproxy/cert.pem
+
+.if feature(QUIC)
+    bind quic4@:443 name quic ssl crt /etc/haproxy/cert.pem
+    http-response add-header alt-svc 'h3=":443"; ma=90000'
+.endif
+    
+    http-request redirect scheme https code 308 unless { ssl_fc }
+    http-request del-header x-forwarded-for
+    option forwardfor
+    option httplog
+
+    http-request set-header X-Forwarded-Proto https if { ssl_fc }
+
+    # enable HTTP compression of text contents
+    compression algo deflate gzip
+    compression type text/ application/javascript application/xhtml+xml image/x-icon
+
+    default_backend domserver
+
+backend domserver
+    balance leastconn
+    option abortonclose
+    server gitea1 127.0.0.1:8080 maxconn 100 check inter 1s

roles/configure_haproxy/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart haproxy
+  ansible.builtin.systemd_service:
+    name: haproxy.service
+    state: restarted

roles/configure_haproxy/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Update package cache
+  community.general.pacman:
+    update_cache: true
+- name: Install haproxy
+  community.general.pacman:
+    pkg:
+      - haproxy
+  notify: Upgrade packages
+
+- name: Install haproxy.cfg
+  ansible.builtin.copy:
+    src: haproxy.cfg
+    dest: /etc/haproxy/haproxy.cfg
+    mode: '0644'
+    owner: haproxy
+    group: haproxy
+  notify: Restart haproxy
+- name: Prompt for manually install cert
+  ansible.builtin.pause:
+    prompt: "Make sure the cert and key pair are store in /etc/haproxy/cert.pem"
+
+- name: Enable and start haproxy
+  ansible.builtin.systemd_service:
+    name: haproxy.service
+    state: started
+    enabled: true

roles/configure_judgehost/handlers/main.yml

@@ -1,6 +1,14 @@
 ---
-- name: Restart judgehost docker compose
+- name: Update grub
-  community.docker.docker_compose_v2:
+  ansible.builtin.command: |
-    project_src: "{{ domjudge_base_dir }}"
+    update-grub
+  changed_when: true
+- name: Reboot
+  ansible.builtin.reboot:
+- name: Systemd daemon-reload
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+- name: Restart judgehost
+  ansible.builtin.systemd_service:
+    name: domjudge-judgehost.target
     state: restarted
-    remove_orphans: true

roles/configure_judgehost/tasks/configure.yml

@@ -0,0 +1,142 @@
+---
+- name: Run ./configure
+  tags: [make]
+  ansible.builtin.command: >-
+    ./configure
+    --with-baseurl={{ domserver_url }}
+    --prefix={{ domjudge_base_dir }}
+  args:
+    chdir: "{{ domjudge_base_dir }}"
+  become: true
+  become_user: domjudge
+  changed_when: true
+  register: debug
+- name: Debug
+  tags: [make]
+  ansible.builtin.debug:
+    var: debug.stdout_lines
+
+- name: Run make judgehost
+  tags: [make]
+  ansible.builtin.command: |
+    make judgehost
+  args:
+    chdir: "{{ domjudge_base_dir }}"
+  become: true
+  become_user: domjudge
+  changed_when: true
+  register: debug
+- name: Debug
+  tags: [make]
+  ansible.builtin.debug:
+    var: debug.stdout_lines
+
+- name: Run make install-judgehost
+  tags: [make]
+  ansible.builtin.command: |
+    make install-judgehost
+  args:
+    chdir: "{{ domjudge_base_dir }}"
+  notify:
+    - Systemd daemon-reload
+  changed_when: true
+  register: debug
+- name: Debug
+  tags: [make]
+  ansible.builtin.debug:
+    var: debug.stdout_lines
+
+- name: Create domjudge-run group
+  tags: [make]
+  ansible.builtin.group:
+    name: domjudge-run
+
+- name: Add domjudge-run user
+  tags: [make]
+  ansible.builtin.user:
+    name: domjudge-run
+    home: /nonexistent
+    group: domjudge-run
+    shell: /bin/false
+
+- name: Add domjudge-run-0 user
+  tags: [make]
+  ansible.builtin.user:
+    name: domjudge-run-0
+    home: /nonexistent
+    group: domjudge-run
+    shell: /bin/false
+
+- name: Copy sudoers-domjudge
+  tags: [make]
+  ansible.builtin.copy:
+    src: "{{ domjudge_base_dir }}/etc/sudoers-domjudge"
+    dest: /etc/sudoers.d/sudoers-domjudge
+    remote_src: true
+    mode: '0440'
+    owner: root
+    group: root
+
+- name: Run misc-tools/dj_make_chroot
+  tags: [chroot]
+  ansible.builtin.command: |
+    ./misc-tools/dj_make_chroot -i kotlin
+  args:
+    chdir: "{{ domjudge_base_dir }}"
+    creates: /chroot/domjudge
+
+- name: Modify boot options
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX_DEFAULT='
+    line: >-
+      GRUB_CMDLINE_LINUX_DEFAULT="quiet
+      cgroup_enable=memory swapaccount=1 isolcpus=0
+      systemd.unified_cgroup_hierarchy=0"
+  notify:
+    - Update grub
+    - Reboot
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Fetch judgehost password
+  tags: [make]
+  community.docker.docker_compose_v2_exec:
+    project_src: "{{ domjudge_base_dir }}"
+    service: domserver
+    command: >-
+      sed -nr 's/^.*\W+judgehost\W+(.+)$/\1/p'
+      /opt/domjudge/domserver/etc/restapi.secret
+  delegate_to: "{{ groups['domserver'] | first }}"
+  run_once: true
+  register: fetch_reg
+- name: Set judgehost facts
+  tags: [make]
+  ansible.builtin.set_fact:
+    domserver_url: "{{ domserver_url }}"
+    judgehost_password: "{{ fetch_reg['stdout'] }}"
+  run_once: true
+- name: Show judgehost password
+  tags: [make]
+  ansible.builtin.debug:
+    var: judgehost_password
+- name: Install restapi.secret
+  tags: [make]
+  ansible.builtin.copy:
+    content: >-
+      default {{ domserver_url }}/api judgehost {{ judgehost_password }}
+    dest: "{{ domjudge_base_dir }}/judgehost/etc/restapi.secret"
+    mode: '0640'
+    owner: domjudge
+    group: domjudge
+
+- name: Flush handlers
+  tags: [make]
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable and start domjudge-judgehost.target
+  tags: [make]
+  ansible.builtin.systemd_service:
+    name: domjudge-judgehost.target
+    state: started
+    enabled: true

roles/configure_judgehost/tasks/download.yml

@@ -0,0 +1,43 @@
+- name: Install domjudge directory
+  ansible.builtin.file:
+    path: "{{ domjudge_base_dir | dirname }}"
+    state: directory
+    recurse: true
+    mode: '0755'
+    owner: domjudge
+    group: domjudge
+- name: Install domjudge by tarball
+  block:
+    - name: Create tmp for domjudge tarball
+      ansible.builtin.file:
+        path: /run/domjudge
+        state: directory
+        mode: '0755'
+        owner: root
+        group: root
+    - name: Download domjudge tarball
+      ansible.builtin.get_url:
+        url: "{{ domjudge_tarball_url }}"
+        dest: /run/domjudge/
+        mode: '0644'
+        owner: domjudge
+        group: domjudge
+      register: downloaded_tarball
+    - name: Extract domjudge tarball
+      ansible.builtin.unarchive:
+        src: "{{ downloaded_tarball.dest }}"
+        dest: /run/domjudge/
+        remote_src: true
+        owner: domjudge
+        group: domjudge
+      register: domjudge_extracted
+    - name: Set new domjudge_base_dir
+      ansible.builtin.set_fact:
+        domjudge_extracted_dir: "{{ domjudge_extracted.dest }}/domjudge-{{ judgehost_version }}"
+    - name: Move content to domjudge base directory
+      ansible.builtin.shell: |
+        echo "{{ domjudge_extracted_dir }}"
+        echo "{{ domjudge_base_dir }}"
+        mv {{ domjudge_extracted_dir }} {{ domjudge_base_dir }}
+      args:
+        creates: "{{ domjudge_base_dir }}"

roles/configure_judgehost/tasks/main.yml

@@ -1,57 +1,10 @@
 ---
-- name: Add boot parameters
+- name: Add domjudge user
-  ansible.builtin.lineinfile:
+  ansible.builtin.user:
-    path: /boot/loader/entries/arch.conf
+    name: domjudge
-    line: >-
+    create_home: true
-      options
+- name: Import domjudge download
-      cgroup_enable=memory
+  ansible.builtin.import_tasks: download.yml
-      swapaccount=1
+- name: Import domjudge configure
-      systemd.unified_cgroup_hierarchy=0
+  ansible.builtin.import_tasks: configure.yml
-  notify: Reboot
+  notify: Restart judgehost
-- name: Flush handlers
-  ansible.builtin.meta: flush_handlers
-
-- name: Fetch judgehost password
-  community.docker.docker_compose_v2_exec:
-    project_src: "{{ domjudge_base_dir }}"
-    service: domserver
-    command: >-
-      sed -nr 's/^.*\W+judgehost\W+(.+)$/\1/p'
-      /opt/domjudge/domserver/etc/restapi.secret
-  register: fetch_reg
-- name: Set judgehost facts
-  ansible.builtin.set_fact:
-    domserver_url: "{{ domserver_url }}"
-    judgehost_password: "{{ fetch_reg['stdout'] }}"
-  delegate_to: domserver[0]
-  run_once: true
-- name: Show judgehost password
-  ansible.builtin.debug:
-    var: judgehost_password
-
-- name: Install judgehost directory
-  ansible.builtin.file:
-    path: "{{ domjudge_base_dir }}"
-    state: directory
-    mode: '0750'
-    owner: root
-    group: docker
-- name: Install judgehost docker compose
-  notify:
-    - Restart judgehost docker compose
-  block:
-    - name: Install docker-compose.yml
-      ansible.builtin.template:
-        src: docker-compose.yml.jinja
-        dest: "{{ domjudge_base_dir }}/docker-compose.yml"
-        mode: '0644'
-        owner: root
-        group: root
-- name: Enable and start docker
-  ansible.builtin.systemd_service:
-    name: docker.service
-    state: started
-    enabled: true
-- name: Run docker compose up
-  community.docker.docker_compose_v2:
-    project_src: "{{ domjudge_base_dir }}"

roles/configure_judgehost_docker/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Reboot
+  ansible.builtin.reboot: {}
+- name: Update grub
+  ansible.builtin.command: |
+    update-grub
+  changed_when: true
+- name: Restart judgehost docker compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ domjudge_base_dir }}"
+    state: restarted
+    remove_orphans: true

roles/configure_judgehost_docker/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+- name: Add boot parameters (Archlinux)
+  ansible.builtin.lineinfile:
+    path: /boot/loader/entries/arch.conf
+    line: >-
+      options cgroup_enable=memory
+  notify: Reboot
+  when: ansible_facts['distribution'] == "Archlinux"
+- name: Add boot parameters (Debian)
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX_DEFAULT='
+    line: >-
+      GRUB_CMDLINE_LINUX_DEFAULT="quiet
+      cgroup_enable=memory swapaccount=1 isolcpus=0
+      systemd.unified_cgroup_hierarchy=0"
+  notify:
+    - Update grub
+    - Reboot
+  when: ansible_facts['distribution'] == "Debian"
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Fetch judgehost password
+  community.docker.docker_compose_v2_exec:
+    project_src: "{{ domjudge_base_dir }}"
+    service: domserver
+    command: >-
+      sed -nr 's/^.*\W+judgehost\W+(.+)$/\1/p'
+      /opt/domjudge/domserver/etc/restapi.secret
+  delegate_to: "{{ groups['domserver'] | first }}"
+  run_once: true
+  register: fetch_reg
+- name: Set judgehost facts
+  ansible.builtin.set_fact:
+    domserver_url: "{{ domserver_url }}"
+    judgehost_password: "{{ fetch_reg['stdout'] }}"
+  run_once: true
+- name: Show judgehost password
+  ansible.builtin.debug:
+    var: judgehost_password
+
+- name: Install judgehost directory
+  ansible.builtin.file:
+    path: "{{ domjudge_base_dir }}"
+    state: directory
+    mode: '0750'
+    owner: root
+    group: docker
+- name: Install judgehost docker compose
+  notify:
+    - Restart judgehost docker compose
+  block:
+    - name: Install docker-compose.yml
+      ansible.builtin.template:
+        src: docker-compose.yml.jinja
+        dest: "{{ domjudge_base_dir }}/docker-compose.yml"
+        mode: '0644'
+        owner: root
+        group: root
+- name: Enable and start docker
+  ansible.builtin.systemd_service:
+    name: docker.service
+    state: started
+    enabled: true
+- name: Run docker compose up
+  community.docker.docker_compose_v2:
+    project_src: "{{ domjudge_base_dir }}"

roles/configure_judgehost_docker/templates/docker-compose.yml.jinja

@@ -10,3 +10,5 @@ services:
       DOMJUDGE_CREATE_WRITABLE_TEMP_DIR: 1
       CONTAINER_TIMEZONE: Asia/Taipei
     restart: always
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup

roles/install_packages_debian/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Install packages
+  ansible.builtin.apt:
+    pkg:
+      - make
+      - pkg-config
+      - sudo
+      - debootstrap
+      - libcgroup-dev
+      - php-cli
+      - php-curl
+      - php-json
+      - php-xml
+      - php-zip
+      - lsof
+      - procps
+      - gcc
+      - g++