Update to 2021-09-06 09:00

roles/ampache/defaults/main.yml

@@ -3,10 +3,10 @@
 ampache_id: "1"
 ampache_manage_upgrade: True
 
-ampache_version: '4.4.3'
+ampache_version: '5.0.0'
-ampache_config_version: 49
+ampache_config_version: 56
 ampache_zip_url: https://github.com/ampache/ampache/releases/download/{{ ampache_version }}/ampache-{{ ampache_version }}_all.zip
-ampache_zip_sha1: ac5daa06a3ca02f210cd91c2eaf9ae5e9daf457f
+ampache_zip_sha1: 3ae83e37667102e385607b4207c068bab89c654d
 
 ampache_root_dir: /opt/ampache_{{ ampache_id }}
 

roles/ampache/tasks/main.yml

@@ -71,13 +71,13 @@
       synchronize:
         src: "{{ ampache_root_dir }}/tmp/ampache/"
         dest: "{{ ampache_root_dir }}/web/"
-        recursive: True
         delete: True
+        compress: False
       delegate_to: "{{ inventory_hostname }}"
   tags: ampache
 
 - name: Check if htaccess files needs to be moved
-  stat: path={{ ampache_root_dir }}/web/{{ item }}/.htaccess.dist
+  stat: path={{ ampache_root_dir }}/web/public/{{ item }}/.htaccess.dist
   with_items:
     - channel
     - play
@@ -86,7 +86,7 @@
   tags: ampache
 
 - name: Rename htaccess files
-  command: mv -f {{ ampache_root_dir }}/web/{{ item.item }}/.htaccess.dist {{ ampache_root_dir }}/web/{{ item.item }}/.htaccess
+  command: mv -f {{ ampache_root_dir }}/web/public/{{ item.item }}/.htaccess.dist {{ ampache_root_dir }}/web/public/{{ item.item }}/.htaccess
   with_items: "{{ htaccess.results }}"
   when: item.stat.exists
   tags: ampache
@@ -126,12 +126,16 @@
   when: ampache_install_mode == 'install'
   tags: ampache
 
-- name: Upgrade SQL database
+- name: Deploy ampache configuration
-  command: php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/install/update_db.inc
+  template: src=ampache.cfg.php.j2 dest={{ ampache_root_dir }}/web/config/ampache.cfg.php group={{ ampache_php_user }} mode=640
-  become_user: "{{ ampache_php_user }}"
-  when: ampache_install_mode == 'upgrade'
   tags: ampache
 
+#- name: Upgrade SQL database
+#  command: php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/cli admin:updateDatabase
+#  become_user: "{{ ampache_php_user }}"
+#  when: ampache_install_mode == 'upgrade'
+#  tags: ampache
+
 - name: Grant admin privileges
   command: mysql --host={{ ampache_mysql_server }} --user=sqladmin --password={{ mysql_admin_pass }} {{ ampache_mysql_db }} -e "UPDATE `user` SET `access`='100' WHERE `username`='{{ item }}'"
   changed_when: False
@@ -146,10 +150,6 @@
     - php_fpm_pool: "{{ ampache_php_fpm_pool | default('') }}"
   tags: ampache
 
-- name: Deploy ampache configuration
-  template: src=ampache.cfg.php.j2 dest={{ ampache_root_dir }}/web/config/ampache.cfg.php group={{ ampache_php_user }} mode=640
-  tags: ampache
-
 - name: Deploy motd
   template: src=motd.php.j2 dest={{ ampache_root_dir }}/web/config/motd.php
   when: ampache_motd is defined

roles/ampache/templates/ampache.cfg.php.j2

@@ -130,4 +130,6 @@ proxy_port = "{{ system_proxy | urlsplit('port') }}"
 proxy_user = "{{ system_proxy | urlsplit('username') }}"
 proxy_pass = "{{ system_proxy | urlsplit('password') }}"
 {% endif %}
-
+metadata_order_video = "filename,getID3"
+registration_display_fields = "fullname,website"
+registration_mandatory_fields = "fullname"

roles/ampache/templates/cron.sh.j2

@@ -16,16 +16,16 @@ echo -n $NEW_HASH > {{ ampache_root_dir }}/tmp/data_hash.txt
 # If file list has changed since last time, then update the catalog
 if [ "$PREV_HASH" != "$NEW_HASH" ]; then
   # Clean (remove files which doesn't exists anymore)
-  /bin/php{{ (ampache_php_version == '54') | ternary('',ampache_php_version) }} {{ ampache_root_dir }}/web/bin/catalog_update.inc -c > /dev/null 2>&1
+  /bin/php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/cli run:updateCatalog -c > /dev/null 2>&1
   # Add (files added)
-  /bin/php{{ (ampache_php_version == '54') | ternary('',ampache_php_version) }} {{ ampache_root_dir }}/web/bin/catalog_update.inc -a > /dev/null 2>&1
+  /bin/php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/cli run:updateCatalog -a > /dev/null 2>&1
   # Update graphics
-  /bin/php{{ (ampache_php_version == '54') | ternary('',ampache_php_version) }} {{ ampache_root_dir }}/web/bin/catalog_update.inc -g > /dev/null 2>&1
+  /bin/php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/cli run:updateCatalog -g > /dev/null 2>&1
 fi
 
 # Now check if files have changed recently. We can have the same file list, but metadata updates
 NEW_FILES=$(find {{ ampache_root_dir }}/data/{music,video} -type f -mtime -1 | wc -l)
 if [ "$NEW_FILES" -gt "0" ]; then
   # Verify (update metadata)
-  /bin/php{{ (ampache_php_version == '54') | ternary('',ampache_php_version) }} {{ ampache_root_dir }}/web/bin/catalog_update.inc -v > /dev/null 2>&1
+  /bin/php{{ ampache_php_version }} {{ ampache_root_dir }}/web/bin/cli run:updateCatalog -e > /dev/null 2>&1
 fi

roles/ampache/templates/httpd.conf.j2

@@ -1,11 +1,11 @@
 {% if ampache_alias is defined %}
-Alias /{{ ampache_alias }} {{ ampache_root_dir }}/web
+Alias /{{ ampache_alias }} {{ ampache_root_dir }}/web/public
 {% else %}
 # No alias defined, create a vhost to access it
 {% endif %}
 
 RewriteEngine On
-<Directory {{ ampache_root_dir }}/web>
+<Directory {{ ampache_root_dir }}/web/public>
   AllowOverride All
   Options FollowSymLinks
 {% if ampache_allowed_ip is defined %}
@@ -16,7 +16,7 @@ RewriteEngine On
   <FilesMatch \.php$>
     SetHandler "proxy:unix:/run/php-fpm/{{ ampache_php_fpm_pool  | default('ampache_' + ampache_id | string) }}.sock|fcgi://localhost"
   </FilesMatch>
-  <FilesMatch "(.maintenance.*|.ansible.*|.t?git.*|.php_cs|.travis.*)">
+  <FilesMatch "(.maintenance.*|.ansible.*|.php_cs|.travis.*)">
     Require all denied
   </FilesMatch>
 </Directory>

roles/mailman/defaults/main.yml

@@ -2,16 +2,16 @@
 
 # Version to install
 mailman_version:
-  core: 3.3.2
+  core: 3.3.4
-  postorius: 1.3.3
+  postorius: 1.3.4
-  hyperkitty: 1.3.3
+  hyperkitty: 1.3.4
 mailman_root_dir: /opt/mailman
 mailman_user: mailman
 # Should ansible handle upgrades ? If False, only initale inistall
 mailman_manage_upgrade: True
 
 # Can be mysql or postgres
-mailman_db_engine: mysql
+mailman_db_engine: postgres
 mailman_db_server: "{{ (mailman_db_engine == 'postgres') | ternary(pg_server,mysql_server) | default('localhost') }}"
 mailman_db_port: "{{ (mailman_db_engine == 'postgres') | ternary('5432','3306') }}"
 mailman_db_user: mailman
@@ -25,7 +25,8 @@ mailman_db_name:
 # Email address of the admin
 mailman_site_owner: "{{ system_admin_email | default('admin' + ansible_domain) }}"
 mailman_public_url: https://listes.{{ ansible_domain }}/
-# Django secret key. A random one will be generated if not set
+# Django secret key. A random one will be generated and stored in
+# {{ mailman_root_dir }}/meta/ansible_secret_key if not set
 # mailman_secret_key: 'p@ssW0rd'
 
 # Port on which uwsgi will listen
@@ -45,6 +46,10 @@ mailman_rest_port: 8013
 # A random one will be created if not defined here
 # mailman_rest_pass: F00/b4r\B4Z
 
+# the API key for HyperKitty. A random one will be generated and stored in
+# {{ mailman_root_dir }}/meta/ansible_hyperkitty_api_key if not defined
+# mailman_hyperkitty_api_key
+
 # Default FROM email
 mailman_email_from: mailman-no-reply@{{ ansible_domain }}
 mailman_smtp_server: localhost

roles/mailman/handlers/main.yml

@@ -9,5 +9,5 @@
 - name: restart mailman timers
   systemd: name={{ item }}.timer state=restarted
   loop:
-    - mailman-digest
+    - mailman-digests
     - mailman-notify

roles/mailman/tasks/archive_pre.yml

@@ -10,8 +10,8 @@
   synchronize:
     src: "{{ mailman_root_dir }}/{{ item }}"
     dest: "{{ mailman_archive_dir }}/"
-    recursive: True
     delete: True
+    compress: False
   loop:
     - venv
     - data
@@ -25,7 +25,7 @@
     --create
     --host={{ mailman_db_server | quote }}
     --port={{ mailman_db_port | quote }}
-    --username=sqladmin {{ mailman_db_name | quote }}
+    --username=sqladmin {{ mailman_db_name[item] | quote }}
     --file="{{ mailman_archive_dir }}/{{ mailman_db_name[item] }}.sql"
   loop: "{{ mailman_db_name.keys() | list }}"
   environment:
@@ -36,7 +36,7 @@
 - name: Dump the database
   mysql_db:
     state: dump
-    name: "{{ item }}"
+    name: "{{ mailman_db_name[item] }}"
     target: "{{ mailman_archive_dir }}/{{ mailman_db_name[item] }}.sql.xz"
     login_host: "{{ mailman_db_server }}"
     login_port: "{{ mailman_db_port }}"
@@ -44,6 +44,7 @@
     login_password: "{{ mailman_db_pass }}"
     quick: True
     single_transaction: True
+  loop: "{{ mailman_db_name.keys() | list }}"
   environment:
     XZ_OPT: -T0
   when: mailman_db_engine == 'mysql'

roles/mailman/tasks/conf.yml

@@ -6,6 +6,8 @@
     - mailman.cfg
     - settings.py
     - uwsgi.ini
+    - urls.py
+    - hyperkitty.cfg
   notify:
     - restart mailman
   tags: mailman
@@ -33,8 +35,5 @@
   environment:
     DJANGO_SUPERUSER_PASSWORD: "{{ mailman_admin_pass }}"
   register: mailman_admin_user
-  failed_when:
+  when: mailman_install_mode == 'install'
-    - mailman_admin_user.rc != 0
-    - mailman_admin_user.stdout is not search('That username is already taken')
-  when: mailman_install_mode != 'none'
   tags: mailman

roles/mailman/tasks/directories.yml

@@ -9,7 +9,7 @@
     mode: "{{ item.mode | default(omit) }}"
   loop:
     - dir: venv
-    - dir: archive
+    - dir: archives
       mode: 700
     - dir: meta
       mode: 700

roles/mailman/tasks/facts.yml

@@ -8,7 +8,7 @@
   tags: mailman
 
 - block:
-    - set_fact: mailman_core_install_mode={{ (mailman_manage_upgrade and install_mode == 'upgrade') | ternary('none',install_mode) }}
+    - set_fact: mailman_core_install_mode={{ (install_mode == 'upgrade' and not mailman_manage_upgrade) | ternary('none', install_mode) }}
     - set_fact: mailman_core_current_version={{ current_version | default('') }}
   tags: mailman
 
@@ -20,7 +20,7 @@
   tags: mailman
 
 - block:
-    - set_fact: mailman_postorius_install_mode={{ (mailman_manage_upgrade and install_mode == 'upgrade') | ternary('none',install_mode) }}
+    - set_fact: mailman_postorius_install_mode={{ (install_mode == 'upgrade' and not mailman_manage_upgrade) | ternary('none', install_mode) }}
     - set_fact: mailman_postorius_current_version={{ current_version | default('') }}
   tags: mailman
 
@@ -32,16 +32,16 @@
   tags: mailman
 
 - block:
-    - set_fact: mailman_hyperkitty_install_mode={{ (mailman_manage_upgrade and install_mode == 'upgrade') | ternary('none',install_mode) }}
+    - set_fact: mailman_hyperkitty_install_mode={{ (install_mode == 'upgrade' and not mailman_manage_upgrade) | ternary('none', install_mode) }}
     - set_fact: mailman_hyperkitty_current_version={{ current_version | default('') }}
   tags: mailman
 
 - set_fact: mailman_install_mode='none'
   tags: mailman
 - set_fact: mailman_install_mode='upgrade'
-  when:
+  when: >
-    - "'upgrade' in [mailman_core_install_mode,mailman_postorius_install_mode,mailman_hyperkitty_install_mode]"
+    'upgrade' in [mailman_core_install_mode, mailman_postorius_install_mode, mailman_hyperkitty_install_mode] and
-    - "'install' not in [mailman_core_install_mode,mailman_postorius_install_mode,mailman_hyperkitty_install_mode]"
+    'install' not in [mailman_core_install_mode, mailman_postorius_install_mode, mailman_hyperkitty_install_mode]
   tags: mailman
 
 # Create a random pass for the DB if needed
@@ -72,6 +72,15 @@
   when: mailman_rest_pass is not defined
   tags: mailman
 
+# HyperKitty API Key
+- block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ mailman_root_dir }}/meta/ansible_hyperkitty_api_key"
+    - set_fact: mailman_hyperkitty_api_key={{ rand_pass }}
+  when: mailman_hyperkitty_api_key is not defined
+  tags: mailman
+
 # Random password for the admin account
 - block:
     - import_tasks: ../includes/get_rand_pass.yml

roles/mailman/tasks/install.yml

@@ -1,7 +1,7 @@
 ---
 
-- name: Enable python38 module
+- name: Enable python39 module
-  command: dnf -y module enable python38
+  command: dnf -y module enable python39
   args:
     warn: False
   changed_when: False
@@ -10,8 +10,8 @@
 - name: Install packages
   yum:
     name:
-      - python38-pip
+      - python39-pip
-      - python38-devel
+      - python39-devel
       - git
       - gcc
       - sassc
@@ -31,14 +31,17 @@
     name:
       - pip
       - wheel
+      - django<3.1
       - mailman=={{ mailman_version.core }}
       - postorius=={{ mailman_version.postorius }}
       - HyperKitty=={{ mailman_version.hyperkitty }}
+      - mailman_hyperkitty
       - mailman-web
       - uwsgi
       - whoosh
     virtualenv: "{{ mailman_root_dir }}/venv"
-    virtualenv_command: /usr/bin/python3.8 -m venv
+    virtualenv_command: /usr/bin/python3.9 -m venv
+  notify: restart mailman
   tags: mailman
 
 - when: mailman_db_engine == 'postgres'
@@ -51,9 +54,9 @@
     - name: Install postgres python support
       pip:
         name:
-          - psycopg2-binary
+          - psycopg2-binary==2.8.6 # There's a bug with TZ in 2.9
         virtualenv: "{{ mailman_root_dir }}/venv"
-        virtualenv_command: /usr/bin/python3.8 -m venv
+        virtualenv_command: /usr/bin/python3.9 -m venv
 
     - name: Create the PostgreSQL role
       postgresql_user:
@@ -91,7 +94,7 @@
           - mysqlclient
           - pymysql
         virtualenv: "{{ mailman_root_dir }}/venv"
-        virtualenv_command: /usr/bin/python3.8 -m venv
+        virtualenv_command: /usr/bin/python3.9 -m venv
 
     - include_tasks: ../includes/webapps_create_mysql_db.yml
       vars:
@@ -122,8 +125,8 @@
 - name: Deploy systemd timers
   template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }}
   loop:
-    - mailman-digest.service
+    - mailman-digests.service
-    - mailman-digest.timer
+    - mailman-digests.timer
     - mailman-notify.service
     - mailman-notify.timer
   notify: restart mailman timers

roles/mailman/tasks/services.yml

@@ -10,6 +10,6 @@
 - name: Start and enable timers
   systemd: name={{ item }}.timer state=started enabled=True
   loop:
-    - mailman-digest
+    - mailman-digests
     - mailman-notify
   tags: mailman

roles/mailman/templates/hyperkitty.cfg.j2

@@ -0,0 +1,3 @@
+[general]
+base_url: http://localhost:{{ mailman_web_port }}/hyperkitty/
+api_key: {{ mailman_hyperkitty_api_key }}

roles/mailman/templates/mailman-digests.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Mailman digests sender
+
+[Service]
+Type=oneshot
+PrivateTmp=yes
+User={{ mailman_user }}
+Group={{ mailman_user }}
+ExecStart={{ mailman_root_dir }}/venv/bin/mailman digests --periodic

roles/mailman/templates/mailman-digests.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Mailman digest sender
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=timers.target

roles/mailman/templates/mailman-web.service.j2

@@ -6,6 +6,7 @@ After=syslog.target network.target postgresql.service mailman-core.service
 Type=notify
 NotifyAccess=all
 Environment=PATH={{ mailman_root_dir }}/venv/bin:/bin:/sbin
+Environment=PYTHONPATH=/opt/mailman/etc
 RuntimeDirectory=uwsgi
 User={{ mailman_user }}
 Group={{ mailman_user }}

roles/mailman/templates/mailman.cfg.j2

@@ -24,8 +24,10 @@ class: mailman.database.mysql.MySQLDatabase
 url: mysql+pymysql://{{ mailman_db_user }}:{{ mailman_db_pass | urlencode | regex_replace('/','%2F') }}@{{ mailman_db_server }}:{{ mailman_db_port }}/{{ mailman_db_name.core }}?charset=utf8&use_unicode=1
 {% endif %}
 
-[archiver.prototype]
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
 enable: yes
+configuration: {{ mailman_root_dir }}/etc/hyperkitty.cfg
 
 [shell]
 history_file: $var_dir/history.py

roles/mailman/templates/settings.py.j2

@@ -23,8 +23,6 @@ DATABASES = {
 # Alias this location from your webserver to `/static`
 STATIC_ROOT = '{{ mailman_root_dir }}/web/static'
 
-# Make sure that this directory is created or Django will fail on start.
-#LOGGING['handlers']['file']['filename'] = '{{ mailman_root_dir }}/log/mailmanweb.log'
 LOGGING = {
   'version': 1,
   'disable_existing_loggers': False,
@@ -75,3 +73,34 @@ HAYSTACK_CONNECTIONS = {
 MAILMAN_REST_API_URL = 'http://localhost:{{ mailman_rest_port }}'
 MAILMAN_REST_API_USER = 'mailmanapi'
 MAILMAN_REST_API_PASS = '{{ mailman_rest_pass }}'
+MAILMAN_ARCHIVER_KEY = '{{ mailman_hyperkitty_api_key }}'
+MAILMAN_ARCHIVER_FROM = ('127.0.0.1')
+
+ROOT_URLCONF = 'urls'
+
+USE_X_FORWARDED_HOST = True
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+SESSION_COOKIE_SECURE = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_BROWSER_XSS_FILTER = True
+CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_HTTPONLY = True
+X_FRAME_OPTIONS = 'DENY'
+
+TIME_ZONE = '{{ (system_tz is defined) | ternary(system_tz, 'UTC') }}'
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+ACCOUNT_UNIQUE_EMAIL  = True
+
+Q_CLUSTER = {
+    'timeout': 300,
+    'save_limit': 100,
+    'orm': 'default',
+    'retry': 360,
+}
+

roles/mailman/templates/urls.py.j2

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Postorius.
+#
+# Postorius is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Postorius is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Postorius.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import include, url
+from django.contrib import admin
+from django.urls import reverse_lazy
+from django.views.generic import RedirectView
+
+urlpatterns = [
+    url(r'^$', RedirectView.as_view(
+        url=reverse_lazy('list_index'),
+        permanent=True)),
+    url(r'^postorius/', include('postorius.urls')),
+    url(r'^hyperkitty/', include('hyperkitty.urls')),
+    url(r'', include('django_mailman3.urls')),
+    url(r'^accounts/', include('allauth.urls')),
+    # Django admin
+    url(r'^admin/', admin.site.urls),
+]
+

roles/mailman/templates/uwsgi.ini.j2

@@ -3,7 +3,6 @@ http-socket = 0.0.0.0:{{ mailman_web_port }}
 virtualenv = {{ mailman_root_dir }}/venv/
 
 module=mailman_web.wsgi:application
-pythonpath = /etc/mailman3/
 env = DJANGO_SETTINGS_MODULE=settings
 
 # Setup default number of processes and threads per process.