From 50ead6075cadc7baf1b18f97bdd1b0b3b8ee2eda Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Sun, 18 Oct 2020 13:00:07 +0200 Subject: [PATCH] Update to 2020-10-18 13:00 --- roles/papermerge/defaults/main.yml | 37 ++++++++ roles/papermerge/handlers/main.yml | 7 ++ roles/papermerge/meta/main.yml | 9 ++ roles/papermerge/tasks/archive_post.yml | 8 ++ roles/papermerge/tasks/archive_pre.yml | 38 ++++++++ roles/papermerge/tasks/cleanup.yml | 8 ++ roles/papermerge/tasks/conf.yml | 49 ++++++++++ roles/papermerge/tasks/directories.yml | 24 +++++ roles/papermerge/tasks/facts.yml | 40 ++++++++ roles/papermerge/tasks/install.yml | 105 +++++++++++++++++++++ roles/papermerge/tasks/iptables.yml | 9 ++ roles/papermerge/tasks/main.yml | 18 ++++ roles/papermerge/tasks/selinux.yml | 5 + roles/papermerge/tasks/services.yml | 8 ++ roles/papermerge/tasks/user.yml | 5 + roles/papermerge/tasks/write_version.yml | 5 + roles/papermerge/templates/gunicorn.conf.py.j2 | 2 + roles/papermerge/templates/nginx.conf.j2 | 18 ++++ .../papermerge/templates/papermerge-web.service.j2 | 23 +++++ .../templates/papermerge-worker.service.j2 | 23 +++++ roles/papermerge/templates/papermerge.conf.py.j2 | 27 ++++++ roles/papermerge/templates/production.py.j2 | 5 + 22 files changed, 473 insertions(+) create mode 100644 roles/papermerge/defaults/main.yml create mode 100644 roles/papermerge/handlers/main.yml create mode 100644 roles/papermerge/meta/main.yml create mode 100644 roles/papermerge/tasks/archive_post.yml create mode 100644 roles/papermerge/tasks/archive_pre.yml create mode 100644 roles/papermerge/tasks/cleanup.yml create mode 100644 roles/papermerge/tasks/conf.yml create mode 100644 roles/papermerge/tasks/directories.yml create mode 100644 roles/papermerge/tasks/facts.yml create mode 100644 roles/papermerge/tasks/install.yml create mode 100644 roles/papermerge/tasks/iptables.yml create mode 100644 roles/papermerge/tasks/main.yml create mode 100644 roles/papermerge/tasks/selinux.yml create mode 100644 roles/papermerge/tasks/services.yml create mode 100644 roles/papermerge/tasks/user.yml create mode 100644 roles/papermerge/tasks/write_version.yml create mode 100644 roles/papermerge/templates/gunicorn.conf.py.j2 create mode 100644 roles/papermerge/templates/nginx.conf.j2 create mode 100644 roles/papermerge/templates/papermerge-web.service.j2 create mode 100644 roles/papermerge/templates/papermerge-worker.service.j2 create mode 100644 roles/papermerge/templates/papermerge.conf.py.j2 create mode 100644 roles/papermerge/templates/production.py.j2 diff --git a/roles/papermerge/defaults/main.yml b/roles/papermerge/defaults/main.yml new file mode 100644 index 0000000..e96f458 --- /dev/null +++ b/roles/papermerge/defaults/main.yml @@ -0,0 +1,37 @@ +--- + +# Version of papermerge to deploy +papermerge_version: 1.5.0 +# URL of the tarball +papermerge_archive_url: https://github.com/ciur/papermerge/archive/v{{ papermerge_version }}.tar.gz +# Expected sha1 of the archive, to check the download was OK +papermerge_archive_sha1: d97d63f3102a48af9aaeb261e498533c54d22bc3 + +# Papermerge uses 2 ports. for gunicorn and will only listen on the loopback +# The other for nginx and is the one which will be accessible over the network +# The port defined here is for nginx. Gunicorn will use this port +1 +papermerge_port: 8010 +# List of IP (or CIDR networks) for which access to the nginx port will be allowed +papermerge_src_ip: [] + +# Should ansible manage papermerge upgrades or just initial install +papermerge_manage_upgrade: True + + # Parameter for the postgres database +papermerge_db_server: "{{ pg_server | default('localhost') }}" +papermerge_db_port: 5432 +papermerge_db_user: papermerge +papermerge_db_name: papermerge +# If papermerge_db_pass is not defined, a random one will be created +#papermerge_db_pass: S3cr3t. + +# Unix user under which papermerge will run +papermerge_user: papermerge +# Top dir where papermerge will be installed +papermerge_root_dir: /opt/papermerge + +# Default lang for the OCR +papermerge_ocr_default_lang: fra + +# Django secret key. A random one will be created if not set +# papermerge_secret_key: abc123 diff --git a/roles/papermerge/handlers/main.yml b/roles/papermerge/handlers/main.yml new file mode 100644 index 0000000..4e947e7 --- /dev/null +++ b/roles/papermerge/handlers/main.yml @@ -0,0 +1,7 @@ +--- + +- name: restart papermerge + service: name={{ item }} state=restarted + loop: + - papermerge-web + - papermerge-worker diff --git a/roles/papermerge/meta/main.yml b/roles/papermerge/meta/main.yml new file mode 100644 index 0000000..c9b8ad1 --- /dev/null +++ b/roles/papermerge/meta/main.yml @@ -0,0 +1,9 @@ +--- + +dependencies: + - role: repo_scl # For python 3.8 + - role: repo_nux_dextop # For pdftk + - role: nginx + - role: postgresql_server + when: papermerge_db_server == '127.0.0.1' or papermerge_db_server == 'localhost' + diff --git a/roles/papermerge/tasks/archive_post.yml b/roles/papermerge/tasks/archive_post.yml new file mode 100644 index 0000000..56807c8 --- /dev/null +++ b/roles/papermerge/tasks/archive_post.yml @@ -0,0 +1,8 @@ +--- + +- import_tasks: ../includes/webapps_compress_archive.yml + vars: + - root_dir: "{{ papermerge_root_dir }}" + - version: "{{ papermerge_current_version }}" + tags: ged + diff --git a/roles/papermerge/tasks/archive_pre.yml b/roles/papermerge/tasks/archive_pre.yml new file mode 100644 index 0000000..8848d95 --- /dev/null +++ b/roles/papermerge/tasks/archive_pre.yml @@ -0,0 +1,38 @@ +--- + +- name: Create the archive dir + file: path={{ papermerge_root_dir }}/archives/{{ papermerge_current_version }} state=directory + tags: ged + +- name: Stop sevices during upgrade + service: name={{ item }} state=stopped + loop: + - papermerge-web + - papermerge-worker + tags: ged + +- name: Archive previous version + synchronize: + src: "{{ papermerge_root_dir }}/{{ item }}" + dest: "{{ papermerge_root_dir }}/archives/{{ papermerge_current_version }}/" + recursive: True + delete: True + loop: + - venv + - app + delegate_to: "{{ inventory_hostname }}" + tags: ged + +- name: Dump the database + command: > + /usr/pgsql-13/bin/pg_dump + --clean + --host={{ papermerge_db_server | quote }} + --port={{ papermerge_db_port | quote }} + --username=sqladmin {{ papermerge_db_name | quote }} + --file="{{ papermerge_root_dir }}/archives/{{ papermerge_current_version }}/{{ papermerge_db_name }}.sql" + environment: + - PGPASSWORD: "{{ pg_admin_pass }}" + tags: ged + + diff --git a/roles/papermerge/tasks/cleanup.yml b/roles/papermerge/tasks/cleanup.yml new file mode 100644 index 0000000..ab5bad8 --- /dev/null +++ b/roles/papermerge/tasks/cleanup.yml @@ -0,0 +1,8 @@ +--- + +- name: Remove tmp and obsolete files + file: path={{ item }} state=absent + loop: + - "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}" + - "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}.tar.gz" + tags: ged diff --git a/roles/papermerge/tasks/conf.yml b/roles/papermerge/tasks/conf.yml new file mode 100644 index 0000000..5224af5 --- /dev/null +++ b/roles/papermerge/tasks/conf.yml @@ -0,0 +1,49 @@ +--- + +- name: Deploy configuration + template: src={{ item }}.j2 dest={{ papermerge_root_dir }}/app/{{ item }} group={{ papermerge_user }} mode=640 + loop: + - papermerge.conf.py + - gunicorn.conf.py + notify: restart papermerge + tags: ged + +- name: Deploy production settings + template: + src: production.py.j2 + dest: "{{ papermerge_root_dir }}/app/config/settings/production.py" + group: "{{ papermerge_user }}" + mode: 640 + tags: ged + +- name: Iniialize or update the database + django_manage: + command: migrate + app_path: "{{ papermerge_root_dir }}/app" + virtualenv: "{{ papermerge_root_dir }}/venv" + when: papermerge_install_mode != 'none' + notify: restart papermerge + tags: ged + +- name: Collect staic files + django_manage: + command: collectstatic + app_path: "{{ papermerge_root_dir }}/app" + virtualenv: "{{ papermerge_root_dir }}/venv" + when: papermerge_install_mode != 'none' + tags: ged + +- name: Create an initial superuser + django_manage: + command: createsuperuser --noinput --username admin --email admin@example.org + app_path: "{{ papermerge_root_dir }}/app" + virtualenv: "{{ papermerge_root_dir }}/venv" + environment: + DJANGO_SUPERUSER_PASSWORD: admin + when: papermerge_install_mode == 'install' + tags: ged + +- name: Deploy nginx configuration + template: src=nginx.conf.j2 dest=/etc/nginx/ansible_conf.d/40-papermerge.conf + notify: reload nginx + tags: ged diff --git a/roles/papermerge/tasks/directories.yml b/roles/papermerge/tasks/directories.yml new file mode 100644 index 0000000..f1f25e6 --- /dev/null +++ b/roles/papermerge/tasks/directories.yml @@ -0,0 +1,24 @@ +--- + +- name: Create directories + file: path={{ item.dir }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }} + loop: + - dir: "{{ papermerge_root_dir }}" + group: nginx + mode: 750 + - dir: "{{ papermerge_root_dir }}/app" + owner: "{{ papermerge_user }}" + group: nginx + mode: 750 + - dir: "{{ papermerge_root_dir }}/data" + owner: "{{ papermerge_user }}" + mode: 700 + - dir: "{{ papermerge_root_dir }}/input" + - dir: "{{ papermerge_root_dir }}/tmp" + owner: "{{ papermerge_user }}" + mode: 700 + - dir: "{{ papermerge_root_dir }}/meta" + mode: 700 + - dir: "{{ papermerge_root_dir }}/archives" + mode: 700 + tags: ged diff --git a/roles/papermerge/tasks/facts.yml b/roles/papermerge/tasks/facts.yml new file mode 100644 index 0000000..c3ebfed --- /dev/null +++ b/roles/papermerge/tasks/facts.yml @@ -0,0 +1,40 @@ +--- + +- fail: msg="pg_admin_pass must be set" + when: pg_admin_pass is not defined + tags: ged + +- import_tasks: ../includes/webapps_set_install_mode.yml + vars: + - root_dir: "{{ papermerge_root_dir }}" + - version: "{{ papermerge_version }}" + tags: ged + +- import_tasks: ../includes/webapps_set_install_mode.yml + vars: + - root_dir: "{{ papermerge_root_dir }}" + - version: "{{ papermerge_version }}" + tags: ged + +- block: + - set_fact: papermerge_install_mode={{ (install_mode == 'upgrade' and not papermerge_manage_upgrade) | ternary('none',install_mode) }} + - set_fact: papermerge_current_version={{ current_version | default('') }} + tags: ged + + # Create a random pass for the DB if needed +- block: + - import_tasks: ../includes/get_rand_pass.yml + vars: + - pass_file: "{{ papermerge_root_dir }}/meta/ansible_dbpass" + - set_fact: papermerge_db_pass={{ rand_pass }} + when: papermerge_db_pass is not defined + tags: ged + + # Create a random secret key +- block: + - import_tasks: ../includes/get_rand_pass.yml + vars: + - pass_file: "{{ papermerge_root_dir }}/meta/ansible_secret_key" + - set_fact: papermerge_secret_key={{ rand_pass }} + when: papermerge_secret_key is not defined + tags: ged diff --git a/roles/papermerge/tasks/install.yml b/roles/papermerge/tasks/install.yml new file mode 100644 index 0000000..4a66198 --- /dev/null +++ b/roles/papermerge/tasks/install.yml @@ -0,0 +1,105 @@ +--- + +- name: Install needed tools + yum: + name: + - rh-python38-python-pip + - rh-python38-python-setuptools + - rh-python38-python-devel + - postgresql-devel + - tesseract + - tesseract-langpack-fra + - tesseract-langpack-deu + - tesseract-langpack-spa + - tesseract-langpack-ita + - pdftk + - glibc.i686 # pdftk from nux-dextop only exists for i686 + - libstdc++.i686 # so install a few i686 libs to get it working + - zlib.i686 + - poppler-utils + - ImageMagick + - git + - tar + tags: ged + +- when: papermerge_install_mode != 'none' + block: + - name: Download papermerge + get_url: + url: "{{ papermerge_archive_url }}" + dest: "{{ papermerge_root_dir }}/tmp/" + checksum: "sha1:{{ papermerge_archive_sha1 }}" + + - name: Extract the archive + unarchive: src={{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}.tar.gz dest={{ papermerge_root_dir }}/tmp remote_src=True + + - name: Move papermerge to the correct dir + synchronize: + src: "{{ papermerge_root_dir }}/tmp/papermerge-{{ papermerge_version }}/" + dest: "{{ papermerge_root_dir }}/app/" + recursive: True + delete: True + delegate_to: "{{ inventory_hostname }}" + + - name: Fix permissions on the app folder + file: path={{ papermerge_root_dir }}/app/ owner={{ papermerge_user }} group=nginx mode=750 + + - name: Wipe the venv on upgrades + file: path={{ papermerge_root_dir }}/venv state=absent + + - name: Create the venv dir + file: path={{ papermerge_root_dir }}/venv state=directory + + - name: Create the venv + pip: + requirements: "{{ papermerge_root_dir }}/app/requirements/base.txt" + virtualenv: "{{ papermerge_root_dir }}/venv" + virtualenv_command: /opt/rh/rh-python38/root/usr/local/bin/virtualenv + virtualenv_python: /opt/rh/rh-python38/root/bin/python + notify: restart papermerge + + - name: Install additional python modules + pip: + name: + - psycopg2-binary # building fails here, Python.h not found (??) + - gunicorn + virtualenv: "{{ papermerge_root_dir }}/venv" + virtualenv_command: /opt/rh/rh-python38/root/usr/local/bin/virtualenv + virtualenv_python: /opt/rh/rh-python38/root/bin/python + notify: restart papermerge + tags: ged + +- name: Create the PostgreSQL role + postgresql_user: + db: postgres + name: "{{ papermerge_db_user }}" + password: "{{ papermerge_db_pass }}" + login_host: "{{ papermerge_db_server }}" + login_user: sqladmin + login_password: "{{ pg_admin_pass }}" + tags: ged + +- name: Create the PostgreSQL database + postgresql_db: + name: "{{ papermerge_db_name }}" + encoding: UTF-8 + template: template0 + owner: "{{ papermerge_db_user }}" + login_host: "{{ papermerge_db_server }}" + login_user: sqladmin + login_password: "{{ pg_admin_pass }}" + tags: ged + +- name: Create systemd units + template: src={{ item }}.service.j2 dest=/etc/systemd/system/{{ item }}.service + loop: + - papermerge-web + - papermerge-worker + notify: restart papermerge + register: papermerge_units + tags: ged + +- name: Reload systemd + systemd: daemon_reload=True + when: papermerge_units.results | selectattr('changed','equalto',True) | list | length > 0 + tags: ged diff --git a/roles/papermerge/tasks/iptables.yml b/roles/papermerge/tasks/iptables.yml new file mode 100644 index 0000000..614034a --- /dev/null +++ b/roles/papermerge/tasks/iptables.yml @@ -0,0 +1,9 @@ +--- + +- name: Handle papermerge port in the firewall + iptables_raw: + name: papermerge_port + state: "{{ (papermerge_src_ip | length > 0) | ternary('present','absent') }}" + rules: "-A INPUT -m state --state NEW -p tcp --dport {{ papermerge_port }} -s {{ papermerge_src_ip | join(',') }} -j ACCEPT" + tags: firewall,ged + diff --git a/roles/papermerge/tasks/main.yml b/roles/papermerge/tasks/main.yml new file mode 100644 index 0000000..3c0591c --- /dev/null +++ b/roles/papermerge/tasks/main.yml @@ -0,0 +1,18 @@ +--- + +- include: user.yml +- include: directories.yml +- include: facts.yml +- include: archive_pre.yml + when: papermerge_install_mode == 'upgrade' +- include: install.yml +- include: selinux.yml + when: ansible_selinux.status == 'enabled' +- include: conf.yml +- include: iptables.yml + when: iptables_manage | default(True) +- include: services.yml +- include: write_version.yml +- include: archive_post.yml + when: papermerge_install_mode == 'upgrade' +- include: cleanup.yml diff --git a/roles/papermerge/tasks/selinux.yml b/roles/papermerge/tasks/selinux.yml new file mode 100644 index 0000000..a682016 --- /dev/null +++ b/roles/papermerge/tasks/selinux.yml @@ -0,0 +1,5 @@ +--- + +- name: Allow nginx to bind on papermerge port + seport: ports={{ papermerge_port }} proto=tcp setype=http_port_t state=present + tags: ged diff --git a/roles/papermerge/tasks/services.yml b/roles/papermerge/tasks/services.yml new file mode 100644 index 0000000..cf07c24 --- /dev/null +++ b/roles/papermerge/tasks/services.yml @@ -0,0 +1,8 @@ +--- + +- name: Start and enable services + service: name={{ item }} state=started enabled=True + loop: + - papermerge-web + - papermerge-worker + tags: ged diff --git a/roles/papermerge/tasks/user.yml b/roles/papermerge/tasks/user.yml new file mode 100644 index 0000000..1648e97 --- /dev/null +++ b/roles/papermerge/tasks/user.yml @@ -0,0 +1,5 @@ +--- + +- name: Create user account + user: name={{ papermerge_user }} home={{ papermerge_root_dir }} system=True + tags: ged diff --git a/roles/papermerge/tasks/write_version.yml b/roles/papermerge/tasks/write_version.yml new file mode 100644 index 0000000..8eb2a6a --- /dev/null +++ b/roles/papermerge/tasks/write_version.yml @@ -0,0 +1,5 @@ +--- + +- name: Write installed version + copy: content={{ papermerge_version }} dest={{ papermerge_root_dir }}/meta/ansible_version + tags: ged diff --git a/roles/papermerge/templates/gunicorn.conf.py.j2 b/roles/papermerge/templates/gunicorn.conf.py.j2 new file mode 100644 index 0000000..db6f781 --- /dev/null +++ b/roles/papermerge/templates/gunicorn.conf.py.j2 @@ -0,0 +1,2 @@ +workers = 2 +bind = ["127.0.0.1:{{ papermerge_port | int + 1 }}"] diff --git a/roles/papermerge/templates/nginx.conf.j2 b/roles/papermerge/templates/nginx.conf.j2 new file mode 100644 index 0000000..d5429c3 --- /dev/null +++ b/roles/papermerge/templates/nginx.conf.j2 @@ -0,0 +1,18 @@ +server { + server_name papermerge; + listen {{ papermerge_port }}; + + location /static/ { + alias {{ papermerge_root_dir }}/app/static/; + } + + location /media/ { + alias {{ papermerge_root_dir }}/app/media/; + } + + location / { + proxy_pass http://127.0.0.1:{{ papermerge_port | int + 1}}; + # Don't restrict size here. You will probably put another front proxy anyway + client_max_body_size 200m; + } +} diff --git a/roles/papermerge/templates/papermerge-web.service.j2 b/roles/papermerge/templates/papermerge-web.service.j2 new file mode 100644 index 0000000..d5130bd --- /dev/null +++ b/roles/papermerge/templates/papermerge-web.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Paperemerge web service +After=postgresql.service +Requires=papermerge-worker.service + +[Service] +WorkingDirectory={{ papermerge_root_dir }}/app +Environment=DJANGO_SETTINGS_MODULE=config.settings.production +ExecStart={{ papermerge_root_dir }}/venv/bin/gunicorn config.wsgi:application --config {{ papermerge_root_dir }}/app/gunicorn.conf.py +User={{ papermerge_user }} +Group={{ papermerge_user }} +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +MemoryLimit=1024M +Restart=on-failure +StartLimitInterval=0 +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/roles/papermerge/templates/papermerge-worker.service.j2 b/roles/papermerge/templates/papermerge-worker.service.j2 new file mode 100644 index 0000000..1f9c797 --- /dev/null +++ b/roles/papermerge/templates/papermerge-worker.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Papermerge Worker +After=network.target + +[Service] +Type=simple +WorkingDirectory={{ papermerge_root_dir }}/app +Environment=DJANGO_SETTINGS_MODULE=config.settings.production +ExecStart={{ papermerge_root_dir }}/venv/bin/python manage.py worker +User={{ papermerge_user }} +Group={{ papermerge_user }} +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +MemoryLimit=1024M +Restart=on-failure +StartLimitInterval=0 +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/roles/papermerge/templates/papermerge.conf.py.j2 b/roles/papermerge/templates/papermerge.conf.py.j2 new file mode 100644 index 0000000..1434ae2 --- /dev/null +++ b/roles/papermerge/templates/papermerge.conf.py.j2 @@ -0,0 +1,27 @@ +DBTYPE = "postgres" +DBNAME = "{{ papermerge_db_name }}" +DBUSER = "{{ papermerge_db_user }}" +DBPASS = "{{ papermerge_db_pass }}" +DBHOST = "{{ papermerge_db_server }}" +DBPORT = "{{ papermerge_db_port }}" +MEDIA_DIR = "{{ papermerge_root_dir }}/data" +IMPORTER_DIR = "{{ papermerge_root_dir }}/input" +FILES_MIN_UNMODIFIED_DURATION = 10 +OCR_DEFAULT_LANGUAGE = "{{ papermerge_ocr_default_lang }}" +LANGUAGE_FROM_AGENT = True +MIDDLEWARE.append( + 'django.middleware.locale.LocaleMiddleware' +) +TASK_QUEUE_DIR = "{{ papermerge_root_dir }}/tmp/queue" +OCR_LANGUAGES = { + "deu": "Deutsch", + "eng": "English", + "fra": "Français", + "spa": "Spanish", + "ita": "Italian" +} + +METADATA_DATE_FORMATS = [ + 'yyyy-mm-dd', + 'month' +] diff --git a/roles/papermerge/templates/production.py.j2 b/roles/papermerge/templates/production.py.j2 new file mode 100644 index 0000000..78404f5 --- /dev/null +++ b/roles/papermerge/templates/production.py.j2 @@ -0,0 +1,5 @@ +from .base import * # noqa +DEBUG = False +ALLOWED_HOSTS = ['127.0.0.1'] +SECRET_KEY = "{{ papermerge_secret_key }}" +