Update to 2021-08-01 15:00

roles/paperless_ng/defaults/main.yml

@@ -0,0 +1,47 @@
+---
+
+# The version to deploy
+paperless_version: 1.4.5
+# URL of the paperless archive
+paperless_archive_url: https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperless_version }}/paperless-ng-{{ paperless_version }}.tar.xz
+# Expected checksum
+paperless_archive_sha1: 4c989458c59890b9bd1dcd97a18e8bcb68280250
+# Should ansible handle install and upgrades, or only initial install
+paperless_manage_upgrade: True
+# Root directory where paperless will be installed
+paperless_root_dir: /opt/paperless
+# Paperless will monitor this dir and import files from here
+paperless_consume_dir: "{{ paperless_root_dir }}/consume"
+
+# The user under which paperless will run
+paperless_user: paperless
+
+# Port used by paperless web interface
+# You should put it behind a reverse proxy
+paperless_port: 8027
+# List of IP/CIDR having access to {{ paperless_port }}
+paperless_src_ip: []
+
+# PostgreSQL database settings
+paperless_db_server: "{{ pg_server | default('localhost') }}"
+paperless_db_port: 5432
+paperless_db_name: paperless
+paperless_db_user: paperless
+# If the password is not defined, a random one will be generated and stored in {{ papermerge_root_dir }}/meta/ansible_dbpass
+# paperless_db_pass: S3cr3t.
+
+# Redis URL
+paperless_redis_url: redis://localhost:6379
+
+# Secret key to create session token. If not defined, a random one will be generated and stored in {{ papermerge_root_dir }}/meta/ansible_secret_key
+# paperless_secret_key: SecretKey
+
+# Public URL where paperless will be available
+paperless_public_url: http://{{ inventory_hostname }}
+
+# Language for the OCR process. Should be a 3 letter code for tesseract.
+# You can use several languages like fra+eng (but it will consume more CPU power)
+paperless_ocr_lang: fra
+
+# Password for the initial admin account. If not defined, a random one will be generated and stored in {{ papermerge_root_dir }}/meta/ansible_admin_pass
+# paperless_admin_pass: p@ssW0rd

roles/paperless_ng/files/paperless.te

@@ -0,0 +1,18 @@
+module paperless 1.3;
+
+require {
+        type gpg_exec_t;
+        type ldconfig_exec_t;
+        type init_t;
+        type ldconfig_t;
+        type postgresql_port_t;
+        class process2 nnp_transition;
+        class file { execute execute_no_trans map open read };
+        class tcp_socket name_connect;
+}
+
+#============= init_t ==============
+allow init_t gpg_exec_t:file { execute execute_no_trans map open read };
+allow init_t ldconfig_exec_t:file execute_no_trans;
+allow init_t ldconfig_t:process2 nnp_transition;
+allow init_t postgresql_port_t:tcp_socket name_connect;

roles/paperless_ng/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+
+- name: restart paperless
+  service: name={{ item }} state=restarted
+  loop:
+    - paperless-webserver
+    - paperless-scheduler
+    - paperless-consumer
+

roles/paperless_ng/meta/main.yml

@@ -0,0 +1,8 @@
+---
+
+dependencies:
+  - role: mkdir
+  - role: postgresql_server
+    when: paperless_db_server in ['localhost','127.0.0.1']
+  - role: redis_server
+    when: paperless_redis_url | urlsplit('hostname') in ['localhost','127.0.0.1']

roles/paperless_ng/tasks/archive_post.yml

@@ -0,0 +1,10 @@
+---
+
+- name: Compress previous version
+  command: tar cf {{ paperless_root_dir }}/archives/{{ paperless_current_version }}.tar.zst --use-compress-program=zstd ./
+  args:
+    chdir: "{{ paperless_root_dir }}/archives/{{ paperless_current_version }}"
+    warn: False
+  environment:
+    ZSTD_CLEVEL: 10
+  tags: ged

roles/paperless_ng/tasks/archive_pre.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Create the archive dir
+  file: path={{ paperless_root_dir }}/archives/{{ paperless_current_version }} state=directory
+  tags: ged
+
+- name: Install postgresql client
+  package:
+    name:
+      - postgresql13
+  tags: ged
+
+- name: Archive previous version
+  synchronize:
+    src: "{{ paperless_root_dir }}/{{ item }}"
+    dest: "{{ paperless_root_dir }}/archives/{{ paperless_current_version }}/"
+    recursive: True
+    delete: True
+    compress: False
+  loop:
+    - venv
+    - app
+  delegate_to: "{{ inventory_hostname }}"
+  tags: ged
+
+- name: Dump the database
+  command: >
+    /usr/pgsql-13/bin/pg_dump
+    --clean
+    --host={{ paperless_db_server | quote }}
+    --port={{ paperless_db_port | quote }}
+    --username=sqladmin {{ paperless_db_name | quote }}
+    --file="{{ paperless_root_dir }}/archives/{{ paperless_current_version }}/{{ paperless_db_name }}.sql"
+  environment:
+    - PGPASSWORD: "{{ pg_admin_pass }}"
+  tags: ged
+
+

roles/paperless_ng/tasks/cleanup.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Remove tmp and obsolete files
+  file: path={{ item }} state=absent
+  loop:
+    - "{{ paperless_root_dir }}/archives/{{ paperless_current_version }}"
+    - "{{ paperless_root_dir }}/tmp/paperless-ng-{{ paperless_version }}.tar.xz"
+    - "{{ paperless_root_dir }}/tmp/paperless-ng"
+  tags: ged

roles/paperless_ng/tasks/conf.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Deploy configuration
+  template: src={{ item }}.j2 dest={{ paperless_root_dir }}/app/{{ item }} group={{ paperless_user }} mode=640
+  loop:
+    - paperless.conf
+    - gunicorn.conf.py
+  notify: restart paperless
+  tags: ged
+
+- when: paperless_install_mode != 'none'
+  block:
+    - name: Migrate database
+      django_manage:
+        command: migrate
+        app_path: "{{ paperless_root_dir }}/app/src"
+        virtualenv: "{{ paperless_root_dir }}/venv"
+      notify: restart paperless
+
+    - name: Collect static files
+      django_manage:
+        command: collectstatic
+        app_path: "{{ paperless_root_dir }}/app/src"
+        virtualenv: "{{ paperless_root_dir }}/venv"
+
+  tags: ged
+
+- when: paperless_install_mode == 'install'
+  block:
+    - name: Create admin user
+      django_manage:
+        command: createsuperuser --noinput --username admin --email admin@{{ ansible_domain }}
+        app_path: "{{ paperless_root_dir }}/app/src"
+        virtualenv: "{{ paperless_root_dir }}/venv"
+      environment:
+        DJANGO_SUPERUSER_PASSWORD: '{{ paperless_admin_pass }}'
+
+  tags: ged

roles/paperless_ng/tasks/directories.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Create directories
+  file: path={{ paperless_root_dir }}/{{ item.dir }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
+  loop:
+    - dir: /
+      owner: "{{ paperless_user }}"
+      mode: 700
+    - dir: /meta
+      mode: 700
+    - dir: /tmp
+      owner: "{{ paperless_user }}"
+      mode: 700
+    - dir: archives
+      mode: 700
+    - dir: backup
+      mode: 700
+    - dir: venv
+    - dir: app
+    - dir: data
+      owner: "{{ paperless_user }}"
+      mode: 700
+    - dir: media
+      owner: "{{ paperless_user }}"
+    - dir: static
+    - dir: consume
+      owner: "{{ paperless_user }}"
+    - dir: log
+      owner: "{{ paperless_user }}"
+      mode: 700
+  tags: ged

roles/paperless_ng/tasks/facts.yml

@@ -0,0 +1,49 @@
+---
+
+- fail: msg="pg_admin_pass must be set"
+  when: pg_admin_pass is not defined
+  tags: ged
+
+- include_vars: "{{ item }}"
+  with_first_found:
+    - vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml
+    - vars/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml
+    - vars/{{ ansible_distribution }}.yml
+    - vars/{{ ansible_os_family }}.yml
+  tags: ged
+
+- block:
+    - import_tasks: ../includes/webapps_set_install_mode.yml
+      vars:
+        - root_dir: "{{ paperless_root_dir }}"
+        - version: "{{ paperless_version }}"
+    - set_fact: paperless_install_mode={{ (install_mode == 'upgrade' and not paperless_manage_upgrade) | ternary('none',install_mode) }}
+    - set_fact: paperless_current_version={{ current_version | default('') }}
+  tags: ged
+
+# Create a random pass for the DB if needed
+- when: paperless_db_pass is not defined
+  block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ paperless_root_dir }}/meta/ansible_dbpass"
+    - set_fact: paperless_db_pass={{ rand_pass }}
+  tags: ged
+
+# Create a random secret key
+- when: paperless_secret_key is not defined
+  block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ paperless_root_dir }}/meta/ansible_secret_key"
+    - set_fact: paperless_secret_key={{ rand_pass }}
+  tags: ged
+
+# Create a random secret key
+- when: paperless_admin_pass is not defined
+  block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "{{ paperless_root_dir }}/meta/ansible_admin_pass"
+    - set_fact: paperless_admin_pass={{ rand_pass }}
+  tags: ged

roles/paperless_ng/tasks/install.yml

@@ -0,0 +1,97 @@
+---
+
+- name: Install packages
+  package: name={{ paperless_packages }}
+  tags: ged
+
+- when: paperless_install_mode != 'none'
+  block:
+    - name: Download paperless
+      get_url:
+        url: "{{ paperless_archive_url }}"
+        dest: "{{ paperless_root_dir }}/tmp"
+        checksum: sha1:{{ paperless_archive_sha1 }}
+
+    - name: Extract archive
+      unarchive:
+        src: "{{ paperless_root_dir }}/tmp/paperless-ng-{{ paperless_version }}.tar.xz"
+        dest: "{{ paperless_root_dir }}/tmp"
+        remote_src: True
+
+    - name: Move paperless app
+      synchronize:
+        src: "{{ paperless_root_dir }}/tmp/paperless-ng/"
+        dest: "{{ paperless_root_dir }}/app/"
+        delete: True
+        compress: False
+      delegate_to: "{{ inventory_hostname }}"
+
+    - name: Fix permissions
+      file: path={{ paperless_root_dir }}/app/ owner=root group=root recurse=True
+
+    - name: Add exec perm on manage.py
+      file: path={{ paperless_root_dir }}/app/src/manage.py mode=755
+
+  tags: ged
+
+- name: Create the virtualenv
+  pip:
+    requirements: "{{ paperless_root_dir }}/app/requirements.txt"
+    state: "{{ (paperless_install_mode == 'upgrade') | ternary('latest', 'present') }}"
+    virtualenv: "{{ paperless_root_dir }}/venv"
+    virtualenv_command: /bin/python3.9 -m venv
+  tags: ged
+
+- block:
+    - name: Create the PostgreSQL role
+      postgresql_user:
+        db: postgres
+        name: "{{ paperless_db_user }}"
+        password: "{{ paperless_db_pass }}"
+        login_host: "{{ paperless_db_server }}"
+        login_user: sqladmin
+        login_password: "{{ pg_admin_pass }}"
+    
+    - name: Create the PostgreSQL database
+      postgresql_db:
+        name: "{{ paperless_db_name }}"
+        encoding: UTF-8
+        template: template0
+        owner: "{{ paperless_db_user }}"
+        login_host: "{{ paperless_db_server }}"
+        login_user: sqladmin
+        login_password: "{{ pg_admin_pass }}"
+  tags: ged
+
+- name: Install a wrapper for manage.py
+  copy:
+    content: |
+      #!/bin/bash
+      cd {{ paperless_root_dir }}/app/src
+      {{ paperless_root_dir }}/venv/bin/python ./manage.py $@
+    dest: /usr/local/bin/paperless
+    mode: 755
+  tags: ged
+
+- name: Install systemd units
+  template: src={{ item }}.service.j2 dest=/etc/systemd/system/{{ item }}.service
+  loop:
+    - paperless-webserver
+    - paperless-scheduler
+    - paperless-consumer
+  notify: restart paperless
+  register: paperless_units
+  tags: ged
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: paperless_units.results | selectattr('changed','equalto',True) | list | length > 0
+  tags: ged
+
+- name: Install backup hooks
+  template: src={{ item }}-backup.j2 dest=/etc/backup/{{ item }}.d/paperless mode=700
+  loop:
+    - pre
+    - post
+  tags: ged
+

roles/paperless_ng/tasks/iptables.yml

@@ -0,0 +1,9 @@
+---
+
+- name:  Handle paperless port in the firewall
+  iptables_raw:
+    name: paperless_port
+    state: "{{ (paperless_src_ip | length > 0) | ternary('present','absent') }}"
+    rules: "-A INPUT -m state --state NEW -p tcp --dport {{ paperless_port }} -s {{ paperless_src_ip | join(',') }} -j ACCEPT"
+  when: iptables_manage | default(True)
+  tags: firewall,ged

roles/paperless_ng/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+
+- include: user.yml
+- include: directories.yml
+- include: facts.yml
+- include: archive_pre.yml
+  when: paperless_install_mode == 'upgrade'
+- include: install.yml
+- include: conf.yml
+- include: iptables.yml
+  when: iptables_manage | default(True)
+- include: selinux.yml
+  when: ansible_selinux.status == 'enabled'
+- include: services.yml
+- include: write_version.yml
+- include: archive_post.yml
+  when: paperless_install_mode == 'upgrade'
+- include: cleanup.yml

roles/paperless_ng/tasks/selinux.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Copy SELinux policy
+  copy: src=paperless.te dest=/etc/selinux/targeted/local/
+  register: paperless_selinux_policy
+  tags: ged
+
+- name: Compile and load SELinux policy
+  shell: |
+    cd /etc/selinux/targeted/local/
+    checkmodule -M -m -o paperless.mod paperless.te
+    semodule_package -o paperless.pp -m paperless.mod
+    semodule -i /etc/selinux/targeted/local/paperless.pp
+  when: paperless_selinux_policy.changed
+  tags: ged

roles/paperless_ng/tasks/services.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Start and enable services
+  service: name={{ item }} state=started enabled=True
+  loop:
+    - paperless-webserver
+    - paperless-scheduler
+    - paperless-consumer
+  tags: ged

roles/paperless_ng/tasks/user.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Create system user
+  user:
+    name: "{{ paperless_user }}"
+    shell: /sbin/nologin
+    home: "{{ paperless_root_dir }}"
+    system: True
+  tags: ged

roles/paperless_ng/tasks/write_version.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Write installed version
+  copy: content={{ paperless_version }} dest={{ paperless_root_dir }}/meta/ansible_version
+  tags: ged

roles/paperless_ng/templates/gunicorn.conf.py.j2

@@ -0,0 +1,36 @@
+import os
+
+bind = '0.0.0.0:{{ paperless_port }}'
+workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 2))
+worker_class = 'paperless.workers.ConfigurableWorker'
+timeout = 120
+
+def pre_fork(server, worker):
+    pass
+
+def pre_exec(server):
+    server.log.info("Forked child, re-executing.")
+
+def when_ready(server):
+    server.log.info("Server is ready. Spawning workers")
+
+def worker_int(worker):
+    worker.log.info("worker received INT or QUIT signal")
+
+    ## get traceback info
+    import threading, sys, traceback
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    code = []
+    for threadId, stack in sys._current_frames().items():
+        code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""),
+            threadId))
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename,
+                lineno, name))
+            if line:
+                code.append("  %s" % (line.strip()))
+    worker.log.debug("\n".join(code))
+
+def worker_abort(worker):
+    worker.log.info("worker received SIGABRT signal")
+

roles/paperless_ng/templates/paperless-consumer.service.j2

@@ -0,0 +1,22 @@
+[Unit]
+Description=Paperless consumer
+After=redis.service postgresql.service
+
+[Service]
+User={{ paperless_user }}
+Group={{ paperless_user }}
+WorkingDirectory={{ paperless_root_dir }}/app/src
+ExecStart={{ paperless_root_dir }}/venv/bin/python3 manage.py document_consumer
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+MemoryLimit=200M
+SyslogIdentifier=paperless-scheduler
+Restart=on-failure
+StartLimitInterval=0
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

roles/paperless_ng/templates/paperless-scheduler.service.j2

@@ -0,0 +1,22 @@
+[Unit]
+Description=Paperless scheduler
+After=redis.service postgresql.service
+
+[Service]
+User={{ paperless_user }}
+Group={{ paperless_user }}
+WorkingDirectory={{ paperless_root_dir }}/app/src
+ExecStart={{ paperless_root_dir }}/venv/bin/python3 manage.py qcluster
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+MemoryLimit=1024M
+SyslogIdentifier=paperless-scheduler
+Restart=on-failure
+StartLimitInterval=0
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

roles/paperless_ng/templates/paperless-webserver.service.j2

@@ -0,0 +1,24 @@
+[Unit]
+Description=Paperless webserver
+After=network.target
+After=redis.service
+Wants=network.target
+
+[Service]
+User={{ paperless_user }}
+Group={{ paperless_user }}
+WorkingDirectory={{ paperless_root_dir }}/app/src
+ExecStart={{ paperless_root_dir }}/venv/bin/gunicorn -c {{ paperless_root_dir }}/app/gunicorn.conf.py paperless.asgi:application
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+MemoryLimit=1024M
+SyslogIdentifier=paperless-webserver
+Restart=on-failure
+StartLimitInterval=0
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

roles/paperless_ng/templates/paperless.conf.j2

@@ -0,0 +1,26 @@
+PAPERLESS_REDIS={{ paperless_redis_url }}
+PAPERLESS_DBHOST={{ paperless_db_server }}
+PAPERLESS_DBPORT={{ paperless_db_port }}
+PAPERLESS_DBNAME={{ paperless_db_name }}
+PAPERLESS_DBUSER={{ paperless_db_user }}
+PAPERLESS_DBPASS={{ paperless_db_pass }}
+PAPERLESS_CONSUMPTION_DIR={{ paperless_consume_dir }}
+PAPERLESS_DATA_DIR={{ paperless_root_dir }}/data
+PAPERLESS_MEDIA_ROOT={{ paperless_root_dir }}/media
+PAPERLESS_STATICDIR={{ paperless_root_dir }}/static
+PAPERLESS_FILENAME_FORMAT={created_year}/{created_month}/{title}
+PAPERLESS_LOGGING_DIR={{ paperless_root_dir }}/log
+PAPERLESS_SECRET_KEY={{ paperless_secret_key }}
+PAPERLESS_ALLOWED_HOSTS={{ paperless_public_url | urlsplit('hostname') }}
+PAPERLESS_CORS_ALLOWED_HOSTS={{ paperless_public_url }}
+PAPERLESS_FORCE_SCRIPT_NAME={{ paperless_public_url | urlsplit('path') | regex_replace('/$','') }}
+PAPERLESS_STATIC_URL={{ paperless_public_url | urlsplit('path') | regex_replace('/$','') }}/static/
+PAPERLESS_OCR_LANGUAGE={{ paperless_ocr_lang }}
+PAPERLESS_TASK_WORKERS=2
+{% if system_tz is defined %}
+PAPERLESS_TIME_ZONE={{ system_tz }}
+{% endif %}
+PAPERLESS_CONSUMER_DELETE_DUPLICATES=True
+PAPERLESS_CONSUMER_RECURSIVE=True
+PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=True
+PAPERLESS_CONVERT_TMPDIR={{ paperless_root_dir }}/tmp

roles/paperless_ng/templates/post-backup.j2

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+rm -f {{ paperless_root_dir }}/backup/*

roles/paperless_ng/templates/pre-backup.j2

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+PGPASSWORD='{{ paperless_db_pass }}' /usr/pgsql-13/bin/pg_dump \
+  --clean \
+  --username={{ paperless_db_user | quote }} \
+  --host={{ paperless_db_server | quote }} \
+  --port={{ paperless_db_port }} \
+  {{ paperless_db_name | quote }} | \
+  zstd -c > {{ paperless_root_dir }}/backup/{{ paperless_db_name | quote }}.sql.zst

roles/paperless_ng/vars/RedHat-8.yml

@@ -0,0 +1,20 @@
+---
+
+paperless_packages:
+  - python39-pip
+  - python39-setuptools
+  - python39-devel
+  - make
+  - gcc
+  - gcc-c++
+  - ImageMagick
+  - liberation-fonts
+  - optipng
+  - libpq-devel
+  - file-libs
+  - tesseract
+  - tesseract-langpack-fra
+  - tesseract-langpack-deu
+  - tesseract-langpack-spa
+  - tesseract-langpack-ita
+  - policycoreutils