Update to 2020-10-18 13:00

roles/papermerge/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+
+# Version of papermerge to deploy
+papermerge_version: 1.5.0
+# URL of the tarball
+papermerge_archive_url: https://github.com/ciur/papermerge/archive/v{{ papermerge_version }}.tar.gz
+# Expected sha1 of the archive, to check the download was OK
+papermerge_archive_sha1: d97d63f3102a48af9aaeb261e498533c54d22bc3
+
+# Papermerge uses 2 ports. for gunicorn and will only listen on the loopback
+# The other for nginx and is the one which will be accessible over the network
+# The port defined here is for nginx. Gunicorn will use this port +1
+papermerge_port: 8010
+# List of IP (or CIDR networks) for which access to the nginx port will be allowed
+papermerge_src_ip: []
+
+# Should ansible manage papermerge upgrades or just initial install
+papermerge_manage_upgrade: True
+
+  # Parameter for the postgres database
+papermerge_db_server: "{{ pg_server | default('localhost') }}"
+papermerge_db_port: 5432
+papermerge_db_user: papermerge
+papermerge_db_name: papermerge
+# If papermerge_db_pass is not defined, a random one will be created
+#papermerge_db_pass: S3cr3t.
+
+# Unix user under which papermerge will run
+papermerge_user: papermerge
+# Top dir where papermerge will be installed
+papermerge_root_dir: /opt/papermerge
+
+# Default lang for the OCR
+papermerge_ocr_default_lang: fra
+
+# Django secret key. A random one will be created if not set
+# papermerge_secret_key: abc123

roles/papermerge/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+
+- name: restart papermerge
+  service: name={{ item }} state=restarted
+  loop:
+    - papermerge-web
+    - papermerge-worker

roles/papermerge/meta/main.yml

@@ -0,0 +1,9 @@
+---
+
+dependencies:
+  - role: repo_scl         # For python 3.8
+  - role: repo_nux_dextop  # For pdftk
+  - role: nginx
+  - role: postgresql_server
+    when: papermerge_db_server == '127.0.0.1' or papermerge_db_server == 'localhost'
+

roles/papermerge/tasks/archive_post.yml

@@ -0,0 +1,8 @@
+---
+
+- import_tasks: ../includes/webapps_compress_archive.yml
+  vars:
+    - root_dir: "{{ papermerge_root_dir }}"
+    - version: "{{ papermerge_current_version }}"
+  tags: ged
+

roles/papermerge/tasks/archive_pre.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Create the archive dir
+  file: path={{ papermerge_root_dir }}/archives/{{ papermerge_current_version }} state=directory
+  tags: ged
+
+- name: Stop sevices during upgrade
+  service: name={{ item }} state=stopped
+  loop:
+    - papermerge-web
+    - papermerge-worker
+  tags: ged
+
+- name: Archive previous version
+  synchronize:
+    src: "{{ papermerge_root_dir }}/{{ item }}"
+    dest: "{{ papermerge_root_dir }}/archives/{{ papermerge_current_version }}/"
+    recursive: True
+    delete: True
+  loop:
+    - venv
+    - app
+  delegate_to: "{{ inventory_hostname }}"
+  tags: ged
+
+- name: Dump the database
+  command: >
+    /usr/pgsql-13/bin/pg_dump
+    --clean
+    --host={{ papermerge_db_server | quote }}
+    --port={{ papermerge_db_port | quote }}
+    --username=sqladmin {{ papermerge_db_name | quote }}
+    --file="{{ papermerge_root_dir }}/archives/{{ papermerge_current_version }}/{{ papermerge_db_name }}.sql"
+  environment:
+    - PGPASSWORD: "{{ pg_admin_pass }}"
+  tags: ged
+
+

roles/papermerge/tasks/cleanup.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Remove tmp and obsolete files
+  file: path={{ item }} state=absent
+  loop:
+    - "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}"
+    - "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}.tar.gz"
+  tags: ged

roles/papermerge/tasks/conf.yml

@@ -0,0 +1,49 @@
+---
+
+- name: Deploy configuration
+  template: src={{ item }}.j2 dest={{ papermerge_root_dir }}/app/{{ item }} group={{ papermerge_user }} mode=640
+  loop:
+    - papermerge.conf.py
+    - gunicorn.conf.py
+  notify: restart papermerge
+  tags: ged
+
+- name: Deploy production settings
+  template:
+    src: production.py.j2
+    dest: "{{ papermerge_root_dir }}/app/config/settings/production.py"
+    group: "{{ papermerge_user }}"
+    mode: 640
+  tags: ged
+
+- name: Iniialize or update the database
+  django_manage:
+    command: migrate
+    app_path: "{{ papermerge_root_dir }}/app"
+    virtualenv: "{{ papermerge_root_dir }}/venv"
+  when: papermerge_install_mode != 'none'
+  notify: restart papermerge
+  tags: ged
+
+- name: Collect staic files
+  django_manage:
+    command: collectstatic
+    app_path: "{{ papermerge_root_dir }}/app"
+    virtualenv: "{{ papermerge_root_dir }}/venv"
+  when: papermerge_install_mode != 'none'
+  tags: ged
+
+- name: Create an initial superuser
+  django_manage:
+    command: createsuperuser --noinput --username admin --email admin@example.org
+    app_path: "{{ papermerge_root_dir }}/app"
+    virtualenv: "{{ papermerge_root_dir }}/venv"
+  environment:
+    DJANGO_SUPERUSER_PASSWORD: admin
+  when: papermerge_install_mode == 'install'
+  tags: ged
+
+- name: Deploy nginx configuration
+  template: src=nginx.conf.j2 dest=/etc/nginx/ansible_conf.d/40-papermerge.conf
+  notify: reload nginx
+  tags: ged

roles/papermerge/tasks/directories.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Create directories
+  file: path={{ item.dir }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
+  loop:
+    - dir: "{{ papermerge_root_dir }}"
+      group: nginx
+      mode: 750
+    - dir: "{{ papermerge_root_dir }}/app"
+      owner: "{{ papermerge_user }}"
+      group: nginx
+      mode: 750
+    - dir: "{{ papermerge_root_dir }}/data"
+      owner: "{{ papermerge_user }}"
+      mode: 700
+    - dir: "{{ papermerge_root_dir }}/input"
+    - dir: "{{ papermerge_root_dir }}/tmp"
+      owner: "{{ papermerge_user }}"
+      mode: 700
+    - dir: "{{ papermerge_root_dir }}/meta"
+      mode: 700
+    - dir: "{{ papermerge_root_dir }}/archives"
+      mode: 700
+  tags: ged

roles/papermerge/tasks/facts.yml

@@ -0,0 +1,40 @@
+---
+
+- fail: msg="pg_admin_pass must be set"
+  when: pg_admin_pass is not defined
+  tags: ged
+
+- import_tasks: ../includes/webapps_set_install_mode.yml
+  vars:
+    - root_dir: "{{ papermerge_root_dir }}"
+    - version: "{{ papermerge_version }}"
+  tags: ged
+
+- import_tasks: ../includes/webapps_set_install_mode.yml
+  vars:
+    - root_dir: "{{ papermerge_root_dir }}"
+    - version: "{{ papermerge_version }}"
+  tags: ged
+
+- block:
+    - set_fact: papermerge_install_mode={{ (install_mode == 'upgrade' and not papermerge_manage_upgrade) | ternary('none',install_mode) }}
+    - set_fact: papermerge_current_version={{ current_version | default('') }}
+  tags: ged
+
+  # Create a random pass for the DB if needed
+- block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ papermerge_root_dir }}/meta/ansible_dbpass"
+    - set_fact: papermerge_db_pass={{ rand_pass }}
+  when: papermerge_db_pass is not defined
+  tags: ged
+
+  # Create a random secret key
+- block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ papermerge_root_dir }}/meta/ansible_secret_key"
+    - set_fact: papermerge_secret_key={{ rand_pass }}
+  when: papermerge_secret_key is not defined
+  tags: ged

roles/papermerge/tasks/install.yml

@@ -0,0 +1,105 @@
+---
+
+- name: Install needed tools
+  yum:
+    name:
+      - rh-python38-python-pip
+      - rh-python38-python-setuptools
+      - rh-python38-python-devel
+      - postgresql-devel
+      - tesseract
+      - tesseract-langpack-fra
+      - tesseract-langpack-deu
+      - tesseract-langpack-spa
+      - tesseract-langpack-ita
+      - pdftk
+      - glibc.i686       # pdftk from nux-dextop only exists for i686
+      - libstdc++.i686   # so install a few i686 libs to get it working
+      - zlib.i686
+      - poppler-utils
+      - ImageMagick
+      - git
+      - tar
+  tags: ged
+
+- when: papermerge_install_mode != 'none'
+  block:
+    - name: Download papermerge
+      get_url:
+        url: "{{ papermerge_archive_url }}"
+        dest: "{{ papermerge_root_dir }}/tmp/"
+        checksum: "sha1:{{ papermerge_archive_sha1 }}"
+
+    - name: Extract the archive
+      unarchive: src={{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}.tar.gz dest={{ papermerge_root_dir }}/tmp remote_src=True
+
+    - name: Move papermerge to the correct dir
+      synchronize:
+        src: "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}/"
+        dest: "{{ papermerge_root_dir }}/app/"
+        recursive: True
+        delete: True
+      delegate_to: "{{ inventory_hostname }}"
+
+    - name: Fix permissions on the app folder
+      file: path={{ papermerge_root_dir }}/app/ owner={{ papermerge_user }} group=nginx mode=750
+
+    - name: Wipe the venv on upgrades
+      file: path={{ papermerge_root_dir }}/venv state=absent
+    
+    - name: Create the venv dir
+      file: path={{ papermerge_root_dir }}/venv state=directory
+
+    - name: Create the venv
+      pip:
+        requirements: "{{ papermerge_root_dir }}/app/requirements/base.txt"
+        virtualenv: "{{ papermerge_root_dir }}/venv"
+        virtualenv_command: /opt/rh/rh-python38/root/usr/local/bin/virtualenv
+        virtualenv_python: /opt/rh/rh-python38/root/bin/python
+      notify: restart papermerge
+
+    - name: Install additional python modules
+      pip:
+        name:
+          - psycopg2-binary # building fails here, Python.h not found (??)
+          - gunicorn
+        virtualenv: "{{ papermerge_root_dir }}/venv"
+        virtualenv_command: /opt/rh/rh-python38/root/usr/local/bin/virtualenv
+        virtualenv_python: /opt/rh/rh-python38/root/bin/python
+      notify: restart papermerge
+  tags: ged
+
+- name: Create the PostgreSQL role
+  postgresql_user:
+    db: postgres
+    name: "{{ papermerge_db_user }}"
+    password: "{{ papermerge_db_pass }}"
+    login_host: "{{ papermerge_db_server }}"
+    login_user: sqladmin
+    login_password: "{{ pg_admin_pass }}"
+  tags: ged
+
+- name: Create the PostgreSQL database
+  postgresql_db:
+    name: "{{ papermerge_db_name }}"
+    encoding: UTF-8
+    template: template0
+    owner: "{{ papermerge_db_user }}"
+    login_host: "{{ papermerge_db_server }}"
+    login_user: sqladmin
+    login_password: "{{ pg_admin_pass }}"
+  tags: ged
+
+- name: Create systemd units
+  template: src={{ item }}.service.j2 dest=/etc/systemd/system/{{ item }}.service
+  loop:
+    - papermerge-web
+    - papermerge-worker
+  notify: restart papermerge
+  register: papermerge_units
+  tags: ged
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: papermerge_units.results | selectattr('changed','equalto',True) | list | length > 0
+  tags: ged

roles/papermerge/tasks/iptables.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Handle papermerge port in the firewall
+  iptables_raw:
+    name: papermerge_port
+    state: "{{ (papermerge_src_ip | length > 0) | ternary('present','absent') }}"
+    rules: "-A INPUT -m state --state NEW -p tcp --dport {{ papermerge_port }} -s {{ papermerge_src_ip | join(',') }} -j ACCEPT"
+  tags: firewall,ged
+

roles/papermerge/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+
+- include: user.yml
+- include: directories.yml
+- include: facts.yml
+- include: archive_pre.yml
+  when: papermerge_install_mode == 'upgrade'
+- include: install.yml
+- include: selinux.yml
+  when: ansible_selinux.status == 'enabled'
+- include: conf.yml
+- include: iptables.yml
+  when: iptables_manage | default(True)
+- include: services.yml
+- include: write_version.yml
+- include: archive_post.yml
+  when: papermerge_install_mode == 'upgrade'
+- include: cleanup.yml

roles/papermerge/tasks/selinux.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Allow nginx to bind on papermerge port
+  seport: ports={{ papermerge_port }} proto=tcp setype=http_port_t state=present
+  tags: ged

roles/papermerge/tasks/services.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Start and enable services
+  service: name={{ item }} state=started enabled=True
+  loop:
+    - papermerge-web
+    - papermerge-worker
+  tags: ged

roles/papermerge/tasks/user.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Create user account
+  user: name={{ papermerge_user }} home={{ papermerge_root_dir }} system=True
+  tags: ged

roles/papermerge/tasks/write_version.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Write installed version
+  copy: content={{ papermerge_version }} dest={{ papermerge_root_dir }}/meta/ansible_version
+  tags: ged

roles/papermerge/templates/gunicorn.conf.py.j2

@@ -0,0 +1,2 @@
+workers = 2
+bind = ["127.0.0.1:{{ papermerge_port | int + 1 }}"]

roles/papermerge/templates/nginx.conf.j2

@@ -0,0 +1,18 @@
+server {
+  server_name papermerge;
+  listen {{ papermerge_port }};
+
+  location /static/ {
+    alias {{ papermerge_root_dir }}/app/static/;
+  }
+
+  location /media/ {
+    alias {{ papermerge_root_dir }}/app/media/;
+  }
+
+  location / {
+    proxy_pass http://127.0.0.1:{{ papermerge_port | int + 1}};
+    # Don't restrict size here. You will probably put another front proxy anyway
+    client_max_body_size 200m;
+  }
+}

roles/papermerge/templates/papermerge-web.service.j2

@@ -0,0 +1,23 @@
+[Unit]
+Description=Paperemerge web service
+After=postgresql.service
+Requires=papermerge-worker.service
+
+[Service]
+WorkingDirectory={{ papermerge_root_dir }}/app
+Environment=DJANGO_SETTINGS_MODULE=config.settings.production
+ExecStart={{ papermerge_root_dir }}/venv/bin/gunicorn config.wsgi:application --config {{ papermerge_root_dir }}/app/gunicorn.conf.py
+User={{ papermerge_user }}
+Group={{ papermerge_user }}
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+MemoryLimit=1024M
+Restart=on-failure
+StartLimitInterval=0
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

roles/papermerge/templates/papermerge-worker.service.j2

@@ -0,0 +1,23 @@
+[Unit]
+Description=Papermerge Worker
+After=network.target
+
+[Service]
+Type=simple
+WorkingDirectory={{ papermerge_root_dir }}/app
+Environment=DJANGO_SETTINGS_MODULE=config.settings.production
+ExecStart={{ papermerge_root_dir }}/venv/bin/python manage.py worker
+User={{ papermerge_user }}
+Group={{ papermerge_user }}
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+MemoryLimit=1024M
+Restart=on-failure
+StartLimitInterval=0
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

roles/papermerge/templates/papermerge.conf.py.j2

@@ -0,0 +1,27 @@
+DBTYPE = "postgres"
+DBNAME = "{{ papermerge_db_name }}"
+DBUSER = "{{ papermerge_db_user }}"
+DBPASS = "{{ papermerge_db_pass }}"
+DBHOST = "{{ papermerge_db_server }}"
+DBPORT = "{{ papermerge_db_port }}"
+MEDIA_DIR = "{{ papermerge_root_dir }}/data"
+IMPORTER_DIR = "{{ papermerge_root_dir }}/input"
+FILES_MIN_UNMODIFIED_DURATION = 10
+OCR_DEFAULT_LANGUAGE = "{{ papermerge_ocr_default_lang }}"
+LANGUAGE_FROM_AGENT = True
+MIDDLEWARE.append(
+  'django.middleware.locale.LocaleMiddleware'
+)
+TASK_QUEUE_DIR = "{{ papermerge_root_dir }}/tmp/queue"
+OCR_LANGUAGES = {
+  "deu": "Deutsch",
+  "eng": "English",
+  "fra": "Français",
+  "spa": "Spanish",
+  "ita": "Italian"
+}
+
+METADATA_DATE_FORMATS = [
+    'yyyy-mm-dd',
+    'month'
+]

roles/papermerge/templates/production.py.j2

@@ -0,0 +1,5 @@
+from .base import *  # noqa
+DEBUG = False
+ALLOWED_HOSTS = ['127.0.0.1']
+SECRET_KEY = "{{ papermerge_secret_key }}"
+