Playbook for installation of new certificates

also changed depricated hostfile option.

.gitignore

@@ -1,3 +1,13 @@
-secrets.yml
 *.retry
 *.pyc
+.vault_pass.txt
+# ---> Vim
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+*.swp
+.vault_pass.txt
+promtools/results

README.md

@@ -1,4 +1,4 @@
-# ssh keys repository
+# HPC playbooks
 
 The `users.yml` playbook contains users and public keys.
 The playbook uses `/etc/hosts` as a database for hosts to install the keys on.

ansible.cfg

@@ -1,2 +1,3 @@
 [defaults]
-hostfile = hosts.py
+inventory = hosts.py
+stdout_callback = debug

meta/main.yml

@@ -0,0 +1 @@
+---

molgenis

@@ -0,0 +1,81 @@
+molgenis[01:99]
+molgenis[100:110]
+
+[molgenis1-70]
+molgenis[01:70]
+
+[no-httpd]
+molgenis02
+molgenis07
+molgenis11
+molgenis15
+molgenis23
+molgenis24
+molgenis25
+molgenis28
+molgenis30
+molgenis32
+molgenis33
+molgenis36
+molgenis37
+molgenis38
+molgenis39
+molgenis40
+molgenis42
+molgenis43
+molgenis44
+molgenis45
+molgenis46
+molgenis47
+molgenis48
+molgenis49
+molgenis54
+molgenis57
+molgenis59
+molgenis61
+molgenis64
+molgenis65
+molgenis69
+molgenis70
+
+[localhost-certfile]
+molgenis03
+molgenis06
+molgenis04
+molgenis05
+molgenis09
+molgenis12
+molgenis13
+molgenis17
+molgenis16
+molgenis19
+molgenis20
+molgenis26
+molgenis21
+molgenis41
+molgenis51
+molgenis50
+molgenis52
+molgenis53
+molgenis56
+molgenis58
+molgenis68
+molgenis18
+molgenis55
+molgenis60
+molgenis66
+molgenis67
+
+[fqdn-certfile]
+molgenis01
+molgenis10
+molgenis14
+molgenis22
+molgenis08
+molgenis31
+molgenis27
+molgenis29
+molgenis34
+molgenis35
+molgenis62
+molgenis63

molgenis_cert.yml

@@ -0,0 +1,23 @@
+---
+- hosts: fqdn-certfile
+  become: false
+  tasks:
+    - copy:
+        src: newcertsmolgenis/{{ ansible_hostname }}_gcc_rug_nl.crt
+        dest: /etc/pki/tls/certs/{{ ansible_hostname }}_gcc_rug_nl/{{ ansible_hostname }}_gcc_rug_nl.crt
+        backup: yes
+    - copy:
+        src: newcertsmolgenis/rsa.{{ ansible_hostname }}.gcc.rug.nl.key
+        dest: /etc/pki/tls/private/{{ ansible_hostname }}_gcc_rug_nl/{{ ansible_hostname }}_gcc_rug_nl.key
+        backup: yes
+- hosts: localhost-certfile
+  become: true
+  tasks:
+    - copy:
+        src: newcertsmolgenis/{{ ansible_hostname }}_gcc_rug_nl.crt
+        dest: /etc/pki/tls/certs/localhost.crt
+        backup: yes
+    - copy:
+        src: newcertsmolgenis/rsa.{{ ansible_hostname }}.gcc.rug.nl.key
+        dest: /etc/pki/tls/private/localhost.key
+        backup: yes

nginx_proxy.yml

@@ -0,0 +1,6 @@
+---
+- hosts: all
+  become: True
+  roles:
+     - docker
+     - nginx-proxy

node_exporter.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+  become: True
+  roles:
+     - node_exporter

promtools/Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.9-stretch
+
+MAINTAINER Egon Rijpkema <e.m.a.rijpkema@rug.nl>
+
+RUN mkdir /results
+
+RUN go get github.com/prometheus/node_exporter && \
+    cd ${GOPATH-$HOME/go}/src/github.com/prometheus/node_exporter && \
+    make && \
+    cp node_exporter /results
+
+RUN go get github.com/robustperception/pushprox/proxy && \
+    cd ${GOPATH-$HOME/go}/src/github.com/robustperception/pushprox/proxy && \
+    go build && \
+    cp /go/bin/proxy /results
+
+RUN go get github.com/robustperception/pushprox/client && \
+    cd ${GOPATH-$HOME/go}/src/github.com/robustperception/pushprox/client && \
+    go build && \
+    cp /go/bin/client /results
+
+CMD /go/bin/proxy

promtools/addport.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+'''
+Pushprox:  does not include the port number in its targets json
+on the /clients endpoint. while Prometheus does seem to need it.
+
+for more info see: https://github.com/RobustPerception/PushProx
+'''
+
+import json
+from urllib import request
+
+url = 'http://knyft.hpc.rug.nl:6060/clients'
+outfile = 'targets.json'
+
+data = json.loads(request.urlopen(url).read().decode('utf-8'))
+
+targets = []
+
+for node in data:
+    for target in node['targets']:
+        if target[-5:] != '9100':
+            target = '{}:9100'.format(target)
+            targets.append(target)
+
+with open(outfile, 'w') as handle:
+    handle.write(json.dumps(
+        [{
+        "targets" : targets,
+        "labels": {
+            "env": "peregrine",
+            "job": "node"
+            }
+        }]
+        ,indent=4 ))

promtools/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+mkdir -p results
+docker build . -t promtools
+docker run -d --name promtools --rm promtools sleep 3
+docker cp promtools:/results .

roles/docker/main.yml

@@ -0,0 +1,25 @@
+---
+# Install Docker. Centos needs te be added.
+
+- apt_key:
+    id: 58118E89F3A912897C070ADBF76221572C52609D
+    keyserver: hkp://p80.pool.sks-keyservers.net:80
+    state: present
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'xenial'
+
+- apt_repository:
+    repo: deb https://apt.dockerproject.org/repo ubuntu-xenial main
+    update_cache: yes
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'xenial'
+
+- name: install docker
+  apt: pkg={{ item }} state=latest
+  with_items:
+     - docker-engine
+     - python-docker
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'xenial'
+
+- name: make sure service is started
+  systemd:
+    name: docker.service
+    state: started

roles/nginx-proxy/tasks/main.yml

@@ -0,0 +1,20 @@
+# Install a nginx reverse proxy with a systemd unit file.
+# See https://github.com/jwilder/nginx-proxy
+---
+- name: install service file.
+  template:
+    src: templates/nginx-proxy.service
+    dest: /etc/systemd/system/nginx-proxy.service
+    mode: 644
+    owner: root
+    group: root
+
+- command: systemctl daemon-reload
+
+- name: start service at boot.
+  command: systemctl reenable nginx-proxy.service
+
+- name: make sure service is started
+  systemd:
+    name: nginx-proxy.service
+    state: restarted

roles/nginx-proxy/templates/nginx-proxy.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=nginx reverse proxy for docker containers.
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker kill %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStart=/usr/bin/docker run --name %n \
+    --rm -d -p 80:80 -p 443:443 -v /srv/certs:/etc/nginx/certs \
+    -v /var/run/docker.sock:/tmp/docker.sock:ro jwilder/nginx-proxy
+
+[Install]
+WantedBy=multi-user.target

roles/node_exporter/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+- file:
+    path: /usr/local/prometheus
+    state: directory
+    mode: 0755
+
+- name: Install node exporter
+  copy:
+    src: "{{ playbook_dir }}/promtools/results/node_exporter"
+    dest: /usr/local/prometheus/node_exporter
+    mode: 0755
+
+- name: Install service files.
+  template:
+    src: templates/node-exporter.service
+    dest: /etc/systemd/system/node-exporter.service
+    mode: 644
+    owner: root
+    group: root
+  tags:
+      - service-files
+
+- name: install service files
+  command: systemctl daemon-reload
+
+- name: enable service at boot
+  systemd:
+    name: node-exporter
+    enabled: yes
+
+- name: make sure servcies are started.
+  systemd:
+    name: node-exporter.service
+    state: restarted
+  tags:
+      - start-service

roles/node_exporter/templates/node-exporter.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=prometheus node exporter
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStart=/usr/local/prometheus/node_exporter \
+ --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)" \
+{% if 'login' in role_names %}
+    --collector.filesystem.ignored-fs-types="^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue|overlay)(fs|)$$"
+{% else %}
+    --collector.filesystem.ignored-fs-types="^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue|overlay|lustre)(fs|)$$"
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

users.yml

@@ -21,8 +21,12 @@
 
     - authorized_key:
         user: wim
-        key: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPcJbucOFOFrPZwM1DKOvscYpDGYXKsgeh3/6skmZn/IhLWYHY6oanm4ifmY3kU0oNXpKgHR43x3JdkIRKmrEpYULspwdlj/ZKPYxFWhVaSTjJvmSJEgy7ET1xk+eVoKV1xRWm/BugWpbseFAOcI9ZwfH++S8JhfX6GgCIy06RUpM8EcFAWb/GO699ZnQ67qMxNdSWYHtK1zu+9svWgEzPk4zc2TihJsc7DxcfQCNfQ4vKH1Im3+QfG5bRtdyVl9yjbE+o4EWhPEWsTBgBosJfbqfywsuzibhTgyybR0Zzm4JN6Wh5wVazvNutAB291dIJt22XEx5tCyOAjLPybLy3 wim@wim-HP-Compaq-Elite-8300-MT'
+        key: '{{ item }}'
         state: present
+      with_items:
+          - 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAilJDjQ8CIdM+5w0Q9ORXheq+hYgfPbcpJ1BoWvMxZrz2ahbamWEeLanWeGcHeQ6rEqTIXv7B3i7erkPHFo+vWUt4b/e1N1OEpuJMueGAn2cDiWbTI9KU+yNCMO8UF6wK8LWqLkUBLm0lpnylwYJDW0NCoVkANU2NJ0JkdzT/bpuAWJp3rs4H7na/EV5vZT/gllMihtIBwWfJNh1BF048PhUBs+l0MSRG8rYe2YcUF66h8btghzYsSqiETGnroVW0XKOHKjxVWO2z2+OkcHOc19zSK6EQMe0+TZFp8Jg3jPZ+4wWnmBv+Zgxg4eEQ8FvfHS7/5lnGF6YATV2cG6Nh9w== rsa-key-20180502'
+          - 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPcJbucOFOFrPZwM1DKOvscYpDGYXKsgeh3/6skmZn/IhLWYHY6oanm4ifmY3kU0oNXpKgHR43x3JdkIRKmrEpYULspwdlj/ZKPYxFWhVaSTjJvmSJEgy7ET1xk+eVoKV1xRWm/BugWpbseFAOcI9ZwfH++S8JhfX6GgCIy06RUpM8EcFAWb/GO699ZnQ67qMxNdSWYHtK1zu+9svWgEzPk4zc2TihJsc7DxcfQCNfQ4vKH1Im3+QfG5bRtdyVl9yjbE+o4EWhPEWsTBgBosJfbqfywsuzibhTgyybR0Zzm4JN6Wh5wVazvNutAB291dIJt22XEx5tCyOAjLPybLy3 wim@wim-HP-Compaq-Elite-8300-MT'
+
 
     - user:
         name: egon
@@ -98,3 +102,14 @@
         user: ger
         key: 'ssh-dss AAAAB3NzaC1kc3MAAACBAMJfiOS0W95C1+r7IBgBR8CqEGpJZ8viv4bpzXWNtDTYLFbfb4rL/PgzlCQqhqJbKCkHluJPHPNAeaW8KalHvqUrtD5xciX8PovcMhkg9Dksp9P5WGKCVfJb5MKwfdtEM9tgq9OjNZFN0nF3R6oW42DvxDKu3mXWiH1PH1I4arQdAAAAFQDxHkrRaQ/t4wH2nO6WN9jWEUNiAwAAAIBR5zi9P3JudJu3dddweDaXlVXY51cGQjXvxFJtFv1d5/jI2gCxcah1dLqkJMwGgFowF4imqUXFit20kNQiG5bUnuGEJWfTg/BkaM7W3ujRxDK6wIQCvAnQ0+zJR/qMqqH7MFlutcEm+uVuACs5abvDOp0scHaOuvGfIyf+qegvLAAAAIB577xm9csmftKclreLmigUksY4zlWoIVYsjgB4ofDVemtHTGYWFBSxQsbhhUrUhB6+AcTRGJnvLyJSaEQdCghVJKEIrGl9YA9lgztd8YAHsG2iVve1mMiFI/8NYJHMWJLuFratq5eC5tpBaW+MTm21NqHKD5Ry88Ul04n+sv5lfw== ger@rc-514'
         state: present
+
+    - user:
+        name: robin
+        comment: "Robin Teeninga"
+        group: admin
+        state: present
+
+    - authorized_key:
+        user: robin
+        key: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXeVMbqjC0EKu8cmuxN+88l0TnzJUuRaFLufka2Mx9Adj8PtAZ4l9IP7f+O97ylbNQvci9DcC38NNe62b0ECutin3jUX9trvROYgxVMR/P89y139CSwWqBrHm29WLHdz9A0vO094HNzhp4xFVnblBUAFt3CCDIxvl59coV2bWgTykmVEoni9SSjqKgcC1hT0mIGcaDb428x9DsteJSakSNYwFbnbEbukA7Y5KQnbzaMl/h97C2FOsxiU5JZoiHgKNXCR5jkFsHzc3OEphXW1Ba4EnqsqUecpnfUr6OueFYR6a/q+AtIKVYT10lzCimXui/uf5zkntq1Kga/h3VtgmV root@robin-HP-Compaq-Elite-8300-MT'
+        state: present