Fixes made while testing the playbooks.

Updated url of docker registry. make a loop for more flexibility. Introducing listen_ip variable that overrides the default listen_ip. make a loop for more flexibility. Get a listen ip specifically for that host. see if components have listen_ip defined before using ansible_default_ipv4 Make service files look for listen_ip variable. implemented listen_ip variable here too map to different port to avoid clashes. Make PHYSICAL_INTERFACE_MAPPINGS variable... instead of just one provuider interface. it should contain something like: physnet2:eth1,physnet3:eth2 add openstack client on machine that is running keystone Added delay to check. enable all services are started at boot Inventory for gcc openstack03 all in one. added volume for glance images Added gcc all in one specific config. Prevent an error when there is no secrets.yml.. to back up removed reference to empty dir. added empty meta/main.yml And now with a list of roles ..and removed the list removed reference to empty dir. Added galera cluster support When at least three database nodes are installed, the playbook will install a galera cluster across them. The galera cnf can be the same template across... nodes. made environment file for the service. I am unable to reproduce systemctl set-environment to work as advertized. Reverted to updating init file by ansible. entrypoint.sh of the mariadb container seems unable to cope when a blank variable is passed by systemd. give the galera master node some time It seems to be nessecary to run in host mode.. for galera to work. I misunderstood pause. need sleep. Inventory for physical test cluster. Added CINDER_PASSWORD Make sure docker is started. If docker was already installed but not running it was not started. fixed refernce to neutron controller Added heat Added port for metadata service Passed metadata secret to be used in config. Listen ip should be the machine's ip... Added openstack client from repo. changed name of subnet added horizon Changed rabbitmq default user to openstack. This makes it no longer nesseccary to create a separate openstack user, which is lost on rabbitmq restart. Added sleep because hosts were usually.. not discovered. Removed unnessecary port mapping Making /dev/lvm available to container. fixed os-test inventory Make iscsi devices available (needed to attach cinder volumes to machines) command module no longer works with && add cinder to test setup
2017-08-31 15:02:47 +02:00
parent 95ef38a3ba
commit 30567679a2
40 changed files with 502 additions and 140 deletions
--- a/roles/cassandra/tasks/main.yml
+++ b/roles/cassandra/tasks/main.yml
@ -7,9 +7,14 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
 - name: make sure service is started
  systemd:
    name: cassandra.service
    state: started
+
+- name: start service at boot.
+  command: systemctl reenable cassandra.service
--- a/roles/cinder-controller/tasks/main.yml
+++ b/roles/cinder-controller/tasks/main.yml
@ -6,18 +6,18 @@
    name: secrets

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-cinder-controller:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-controller:latest
    env_vars: >
-        -e "MY_IP={{ ansible_default_ipv4.address }}"
-        -e "CINDER_HOST={{ hostvars[groups['cinder-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "CINDER_PASSWORD={{ secrets['CINDER_PASSWORD'] }}"
        -e "CINDER_USER=cinder"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"

@ -43,6 +43,9 @@
    owner: root
    group: root

+- name: start service at boot.
+  command: systemctl reenable cinder-controller.service
+
 - command: systemctl daemon-reload

 - name: Initialize database.
--- a/roles/cinder-storage/tasks/main.yml
+++ b/roles/cinder-storage/tasks/main.yml
@ -4,22 +4,24 @@
  include_vars:
    file: ../../secrets.yml
    name: secrets
+  tags: vars

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-cinder-storage:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage:latest
    env_vars: >
-        -e "MY_IP={{ ansible_default_ipv4.address }}"
-        -e "CINDER_HOST={{ hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address']) }}"
        -e "CINDER_PASSWORD={{ secrets['CINDER_PASSWORD'] }}"
        -e "CINDER_USER=cinder"
-        -e "GLANCE_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"
+  tags: vars

 - name: pull docker image
  docker_image:
@ -52,8 +54,13 @@
    mode: 644
    owner: root
    group: root
+  tags: systemd

 - command: systemctl daemon-reload
+  tags: systemd
+
+- name: start service at boot.
+  command: systemctl reenable cinder-storage.service

 - name: make sure service is started
  systemd:
--- a/roles/cinder-storage/templates/cinder-storage.service
+++ b/roles/cinder-storage/templates/cinder-storage.service
@ -1,5 +1,5 @@
 [Unit]
-Description=Openstack Glance Container
+Description=Openstack Cinder Storage container
 After=docker.service
 Requires=docker.service

@ -11,9 +11,13 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  --privileged \
  {{ env_vars | replace('\n', '') }} \
+  -v "/dev/cinder-volumes/":/dev/cinder-volumes \
  -v /srv/cinder-storage/root:/root \
  -v "{{ storage_volume }}":/dev/cinder_storage_volume \
-  -p 8776:8776 \
+  -v "/dev/lvm":/dev/lvm \
+  -v "/srv/cinder-storage/volumes/:/var/lib/cinder/volumes/" \
+  -p 8777:8776 \
+  -p 3260:3260 \
  {{ docker_image }}

 [Install]
--- a/roles/common/tasks/docker.yml
+++ b/roles/common/tasks/docker.yml
@ -13,3 +13,8 @@
  with_items:
     - docker-engine
     - python-docker
+
+- name: make sure service is started
+  systemd:
+    name: docker.service
+    state: started
--- a/roles/dockerregistry/tasks/main.yml
+++ b/roles/dockerregistry/tasks/main.yml
@ -7,13 +7,18 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
 - name: make sure service is started
  systemd:
    name: dockerregistry.service
    state: started

+- name: start service at boot.
+  command: systemctl reenable dockerregistry.service
+
 - name: Copy certificates and passwd file
  copy:
    src: "{{ item }}"
--- a/roles/glance-controller/tasks/main.yml
+++ b/roles/glance-controller/tasks/main.yml
@ -6,17 +6,17 @@
    name: secrets

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-glance:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance:latest
    env_vars: >
-        -e "GLANCE_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "GLANCE_PASSWORD={{ secrets['GLANCE_PASSWORD'] }}"
        -e "GLANCE_USER=glance"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"

@ -42,14 +42,18 @@
    owner: root
    group: root

+- name: start service at boot.
+  command: systemctl reenable glance.service
+
 - command: systemctl daemon-reload

 - name: Initialize database.
  command: >
             /usr/bin/docker run --rm
             {{ env_vars }}
-             --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+             --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
             -v /srv/glance/root:/root \
+             -v /var/lib/glance/images:/var/lib/glance/images \
             {{ docker_image }} /etc/bootstrap.sh
  tags: bootstrap

--- a/roles/glance-controller/templates/glance.service
+++ b/roles/glance-controller/templates/glance.service
@ -11,6 +11,7 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
  -v /srv/glance/root:/root \
+  -v /var/lib/glance/images:/var/lib/glance/images \
  -p 9292:9292 \
  {{ docker_image }}

--- a/roles/heat/tasks/main.yml
+++ b/roles/heat/tasks/main.yml
@ -0,0 +1,62 @@
+# Build and install a docker image for heat.
+---
+- name: include secrets
+  include_vars:
+    file: ../../secrets.yml
+    name: secrets
+
+- set_fact:
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-heat:latest
+    env_vars: >
+        -e "HEAT_HOST={{ listen_ip | default(hostvars[groups['heat'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "HEAT_PASSWORD={{ secrets['HEAT_PASSWORD'] }}"
+        -e "HEAT_USER=heat"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
+        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
+        -e "RABBIT_USER=openstack"
+
+- name: pull docker image
+  docker_image:
+    name: "{{ docker_image }}"
+  tags: pull
+
+- name: Make build and persistent directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0777
+  with_items:
+      - /srv/heat
+      - /srv/heat/root
+
+- name: install service file.
+  template:
+    src: templates/heat.service
+    dest: /etc/systemd/system/heat.service
+    mode: 644
+    owner: root
+    group: root
+
+- name: start service at boot.
+  command: systemctl reenable heat.service
+
+- command: systemctl daemon-reload
+
+- name: Initialize database.
+  command: >
+             /usr/bin/docker run --rm
+             {{ env_vars }}
+             --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
+             -v /srv/heat/root:/root \
+             {{ docker_image }} /etc/bootstrap.sh
+  tags: bootstrap
+
+- name: make sure service is started
+  systemd:
+    name: heat.service
+    state: restarted
--- a/roles/heat/templates/heat.service
+++ b/roles/heat/templates/heat.service
@ -0,0 +1,19 @@
+[Unit]
+Description=Openstack heat Container
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker kill %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStart=/usr/bin/docker run --name %n \
+  {{ env_vars | replace('\n', '') }} \
+  -v /srv/heat/root:/root \
+  -p 8000:8000 \
+  -p 8004:8004 \
+  {{ docker_image }}
+
+[Install]
+WantedBy=multi-user.target
--- a/roles/horizon/tasks/main.yml
+++ b/roles/horizon/tasks/main.yml
@ -1,7 +1,7 @@
 # Run hpc/horizon
 ---
 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-horizon:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-horizon:latest

 - name: pull docker image
  docker_image:
@ -19,6 +19,9 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable horizon.service
+
 - name: make sure service is started
  systemd:
    name: horizon.service
--- a/roles/horizon/templates/horizon.service
+++ b/roles/horizon/templates/horizon.service
@ -9,9 +9,9 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-  -e "MEMCACHED_SERVER={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}" \
-  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  -e "MEMCACHED_SERVER={{ hostvars[groups['memcached'][0]]['listen_ip'] | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}" \
+  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  -p 80:80 \
  {{ docker_image }}

--- a/roles/keystone/tasks/main.yml
+++ b/roles/keystone/tasks/main.yml
@ -17,7 +17,7 @@
    - /srv/keystone/root

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-keystone:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-keystone:latest

 - name: pull docker image
  docker_image:
@ -36,11 +36,14 @@
 - name: install service file
  command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable keystone.service
+
 - name: Initialize db
  script: scripts/initialize_db.sh
  environment:
    MYSQL_ROOT_PASSWORD: "{{ secrets['MYSQL_ROOT_PASSWORD'] }}"
-    DB_HOST: "{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+    DB_HOST: "{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
  register: result
  until: result|succeeded
  # sometimes the initial connect fails.
@ -52,9 +55,9 @@
 - name: keystone manage commands to setup db
  command: >
             /usr/bin/docker run --rm
-             --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
+             --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
             -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys
-             -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+             -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
             {{ docker_image }} keystone-manage {{ item }}
  with_items:
      - db_sync
@ -62,9 +65,9 @@
      - credential_setup --keystone-user keystone --keystone-group keystone
      - >
          bootstrap --bootstrap-password {{ secrets['OS_PASSWORD'] }}
-                    --bootstrap-admin-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:35357/v3/
-                    --bootstrap-internal-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:35357/v3/
-                    --bootstrap-public-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:5000/v3/
+                    --bootstrap-admin-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:35357/v3/
+                    --bootstrap-internal-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:35357/v3/
+                    --bootstrap-public-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:5000/v3/
                    --bootstrap-region-id RegionOne

 - name: make sure service is started
@ -75,10 +78,29 @@
 - name: Create a domain, projects users and roles
  command: >
             /usr/bin/docker run --rm
-             --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
+             --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
             -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys
             -v /srv/keystone/root:/root
-             -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+             -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
             -e "OS_AUTH_URL=http://${KEYSTONE_HOST}:35357/v3"
             -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
             {{ docker_image }} bash /etc/bootstrap.sh
+
+
+- name: install openstack repo on host.
+  command: >
+      echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main" > /etc/apt/sources.list.d/ocata.list &&
+      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+  tags: openstackclient
+
+- name: install openstack client for management
+  apt:
+    name: python-openstackclient
+    state: latest
+    update_cache: yes
+  tags: openstackclient
+
+- name: source admin-openrc.sh in root .bashrc
+  lineinfile:
+    path: /root/.bashrc
+    line: 'source /srv/keystone/root/admin-openrc.sh'
--- a/roles/keystone/templates/keystone.service
+++ b/roles/keystone/templates/keystone.service
@ -9,8 +9,8 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
  -p 5000:5000 -p 35357:35357 \
  -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys \
  -v /srv/keystone/root:/root \
--- a/roles/mariadb/files/galera.cnf
+++ b/roles/mariadb/files/galera.cnf
@ -0,0 +1,20 @@
+[mysqld]
+binlog_format=ROW
+default-storage-engine=innodb
+innodb_autoinc_lock_mode=2
+bind-address=0.0.0.0
+
+# Galera Provider Configuration
+wsrep_on=ON
+wsrep_provider=/usr/lib/galera/libgalera_smm.so
+
+# Galera Cluster Configuration
+wsrep_cluster_name="test_cluster"
+wsrep_cluster_address="gcomm://{{ ip_node0 }},{{ ip_node1 }},{{ ip_node2 }}"
+
+# Galera Synchronization Configuration
+wsrep_sst_method=rsync
+
+# Galera Node Configuration
+wsrep_node_address="{{ listen_ip | default(ansible_default_ipv4.address) }}"
+wsrep_node_name="{{ ansible_nodename }}"
--- a/roles/mariadb/tasks/main.yml
+++ b/roles/mariadb/tasks/main.yml
@ -5,14 +5,6 @@
    file: ../../secrets.yml
    name: secrets

- name: install service file.
-  template:
-    src: templates/mysql.service
-    dest: /etc/systemd/system/mysql.service
-    mode: 644
-    owner: root
-    group: root
-
 - name: make mariadb settings volume
  file:
    path: "{{ item }}"
@ -21,16 +13,60 @@
  with_items:
      - /srv/mariadb/lib/mysql
      - /srv/mariadb/etc/mysql
+      - /srv/mariadb/etc/mysql/conf.d

 - name: place settings file
  copy:
    src: files/my.cnf
-    dest: /srv/mariadb/etc/mysql
+    dest: /srv/mariadb/etc/mysql/conf.d/my.cnf
    mode: 660

- command: systemctl daemon-reload
+- name: Set galara.cnf on node if we have at least three nodes.
+  template:
+    src: files/galera.cnf
+    dest: /srv/mariadb/etc/mysql/conf.d/galera.cnf
+    mode: 660
+  when: groups['databases'] | length >= 3
+
+  # This mimics galera_new_cluster.sh
+- name: Initialize a new cluster.
+  block:
+    - set_fact:
+        mariadb_args: "--wsrep-new-cluster"
+
+    - template:
+        src: templates/mysql.service
+        dest: /etc/systemd/system/mysql.service
+        mode: 644
+        owner: root
+        group: root
+
+    - command: systemctl daemon-reload
+
+    - systemd:
+        name: mysql.service
+        state: started
+
+  when: groups['databases'] | length >= 3 and ansible_hostname == hostname_node0
+
+- name: install service file.
+  block:
+    - set_fact:
+        mariadb_args: ""
+    - template:
+        src: templates/mysql.service
+        dest: /etc/systemd/system/mysql.service
+        mode: 644
+        owner: root
+        group: root
+
+- name: Give the master node some time to initialize the cluster.
+  command: bash -c "sleep 60 && systemctl daemon-reload"

 - name: make sure service is started
  systemd:
    name: mysql.service
    state: started
+
+- name: start service at boot.
+  command: systemctl reenable mysql.service
--- a/roles/mariadb/templates/mysql.service
+++ b/roles/mariadb/templates/mysql.service
@ -9,10 +9,11 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n || /bin/true
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStartPre=/usr/bin/docker pull mariadb:10.2
-ExecStart=/usr/bin/docker run  -p 3306:3306 --name %n \
+ExecStart=/usr/bin/docker run --name %n \
+ --network host \
 -v /srv/mariadb/lib/mysql:/var/lib/mysql \
- -v /srv/mariadb/etc/mysql:/etc/mysql \
- -e MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }} mariadb:10.2
+ -v /srv/mariadb/etc/mysql/conf.d:/etc/mysql/conf.d \
+ -e MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }} mariadb:10.2 {{ mariadb_args }}

 [Install]
 WantedBy=multi-user.target
--- a/roles/memcached/tasks/main.yml
+++ b/roles/memcached/tasks/main.yml
@ -7,8 +7,13 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
+- name: start service at boot.
+  command: systemctl reenable memcached.service
+
 - name: make sure service is started
  systemd:
    name: memcached.service
--- a/roles/neutron-controller/tasks/main.yml
+++ b/roles/neutron-controller/tasks/main.yml
@ -6,7 +6,7 @@
    name: secrets

 - set_fact:
-    docker_image: "webhost12.service.rug.nl/hpc/openstack-neutron-controller:latest"
+    docker_image: "registry.webhosting.rug.nl/hpc/openstack-neutron-controller:latest"

 - name: pull docker image
  docker_image:
@ -16,21 +16,21 @@

 - set_fact:
    env_vars: >
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
        -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}"
-        -e "MY_IP={{ hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}"
        -e "NEUTRON_USER=neutron"
        -e "NOVA_USER=nova"
        -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}"
-        -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NOVA_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_USER=placement"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "PROVIDER_INTERFACE_NAME={{ provider_interface_name }}"
+        -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"
  tags: env
@ -45,12 +45,15 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable neutron-controller.service
+
 - name: Initialize neutron
  command: >
    /usr/bin/docker run --rm
    {{ env_vars }}
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}
+    --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
+    --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}
    --network host
    {{ docker_image }}
    /etc/bootstrap.sh
--- a/roles/neutron-controller/templates/neutron-controller.service
+++ b/roles/neutron-controller/templates/neutron-controller.service
@ -10,9 +10,9 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
-  --add-host=nova-controller:{{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  --add-host=nova-controller:{{ hostvars[groups['nova-controller'][0]]['listen_ip'] | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  --add-host={{ ansible_nodename }}:{{ ansible_default_ipv4.address }} \
  --privileged \
  --network host \
--- a/roles/nova-compute/tasks/main.yml
+++ b/roles/nova-compute/tasks/main.yml
@ -4,10 +4,11 @@
  include_vars:
    file: ../../secrets.yml
    name: secrets
+  tags: vars

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-nova-compute:latest
-  tags: facts
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-compute:latest
+  tags: vars

 - name: pull docker image
  docker_image:
@ -22,11 +23,13 @@
    mode: 644
    owner: root
    group: root
+  tags: systemd

 - command: systemctl daemon-reload
+  tags: systemd

 - apt:
-    name: '{{ item }}'
+    name: "{{ item }}"
  with_items:
      - kvm
      - libvirt0
@ -38,6 +41,9 @@
    name: nova-compute.service
    state: restarted

+- name: start service at boot.
+  command: systemctl reenable nova-compute.service
+
 - name: let nova controler discover new host
-  command: docker exec -i nova-controller.service nova-manage cell_v2 discover_hosts
+  shell: "sleep 10 && docker exec -i nova-controller.service nova-manage cell_v2 discover_hosts"
  delegate_to: "{{ hostvars[groups['nova-controller'][0]]['ansible_hostname'] }}"
--- a/roles/nova-compute/templates/nova-compute.service
+++ b/roles/nova-compute/templates/nova-compute.service
@ -9,30 +9,32 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-    -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MY_IP={{ hostvars[groups['nova-compute'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['listen_ip'] | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}" \
+    -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['listen_ip'] | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}" \
+    -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}" \
-    -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['listen_ip'] | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}" \
    -e "NEUTRON_USER=neutron" \
    -e "NOVA_COMPUTE_USER=nova_compute" \
-    -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['listen_ip'] | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}" \
    -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}" \
    -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}" \
    -e "NOVA_PLACEMENT_USER=placement" \
    -e "NOVA_USER=nova" \
    -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}" \
-    -e "PROVIDER_INTERFACE_NAME={{ provider_interface_name }}" \
-    -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}" \
+    -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['listen_ip'] | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}" \
    -e "RABBIT_USER=openstack" \
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
+    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
+    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
    --privileged \
+    -v /dev:/dev \
    -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
    -v /var/lib/nova/instances:/var/lib/nova/instances \
    -v /lib/modules:/lib/modules \
--- a/roles/nova-controller/tasks/main.yml
+++ b/roles/nova-controller/tasks/main.yml
@ -15,18 +15,19 @@
    - /srv/nova-controller/root

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-nova-service:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-service:latest
    env_vars: >
-        -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MY_IP={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
-        -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['listen_ip'] | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}"
        -e "NEUTRON_USER=neutron"
-        -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NOVA_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_USER=placement"
@ -52,12 +53,15 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable nova-controller.service
+
 - name: Initialize database.
  command: >
    /usr/bin/docker run --rm
    {{ env_vars }}
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}
+    --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
+    --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}
    -v /srv/nova-controller/root:/root
    {{ docker_image }}
    /etc/bootstrap.sh
--- a/roles/nova-controller/templates/nova-controller.service
+++ b/roles/nova-controller/templates/nova-controller.service
@ -10,11 +10,12 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  --privileged \
  -v /srv/nova-controller/root:/root \
  -p 8774:8774 \
+  -p 8775:8775 \
  -p 8778:8778 \
  -p 6080:6080 \
  {{ docker_image }} /etc/run.sh
--- a/roles/rabbitmq/files/rabbitmq.service
+++ b/roles/rabbitmq/files/rabbitmq.service
@ -10,11 +10,11 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStartPre=/usr/bin/docker pull rabbitmq:latest
 ExecStart=/usr/bin/docker run \
-          --add-host "{{ hostvars[groups['rabbitmq'][0]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}" \
-          --add-host "{{ hostvars[groups['rabbitmq'][1]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][1]]['ansible_default_ipv4']['address'] }}" \
-          --add-host "{{ hostvars[groups['rabbitmq'][2]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][2]]['ansible_default_ipv4']['address'] }}" \
+{% for host in groups['rabbitmq'] %}
+          --add-host "{{ host }}:{{ hostvars[host]['listen_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}" \
+{% endfor %}
          -p 4369:4369 -p 25679:25679 -p 25672:25672 -p 5671-5672:5671-5672 -p 8080:15672 \
-          -e "RABBITMQ_DEFAULT_USER=user" -e "RABBITMQ_DEFAULT_PASS={{ secrets['RABBIT_PASSWORD'] }}" \
+          -e "RABBITMQ_DEFAULT_USER=openstack" -e "RABBITMQ_DEFAULT_PASS={{ secrets['RABBIT_PASSWORD'] }}" \
          -e "RABBITMQ_ERLANG_COOKIE={{ secrets['RABBITMQ_ERLANG_COOKIE'] }}" \
          -e "RABBITMQ_NODENAME=rabbit_{{ ansible_nodename }}" \
          --hostname "{{ ansible_nodename }}" --name %n rabbitmq:3-management
--- a/roles/rabbitmq/tasks/main.yml
+++ b/roles/rabbitmq/tasks/main.yml
@ -5,9 +5,6 @@
    file: ../../secrets.yml
    name: secrets

- include_vars:
-    dir: 'vars'
-
 - name: install service file.
  template:
    src: files/rabbitmq.service
@ -19,6 +16,9 @@
 - name: install service file
  command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable rabbitmq.service
+
 - name: make sure service is started
  systemd:
    name: rabbitmq.service
@ -27,6 +27,7 @@
 - name: wait for container to be started
  wait_for:
      port: 5672
+      delay: 5

 - name: setup the cluster
  command: "docker exec -i rabbitmq.service {{ item }}"
@ -36,11 +37,3 @@
      - rabbitmqctl start_app
  when: ansible_nodename != hostname_node0

- name: create openstack user
-  command: "docker exec -i rabbitmq.service {{ item }}"
-  with_items:
-      - rabbitmqctl add_user openstack "{{ secrets['RABBIT_PASSWORD'] }}"
-      - rabbitmqctl set_permissions openstack ".*" ".*" ".*"
-  when: ansible_nodename == hostname_node0
-  register: command_result
-  failed_when: "command_result.rc not in (0, 70)"