Merge branch 'develop'

2018-01-19 09:18:10 +01:00 · 2018-01-19 09:18:10 +01:00 · c82d771d2e
commit c82d771d2e
parent 95ef38a3ba 30567679a2
40 changed files with 502 additions and 140 deletions
--- a/gcc-post-install.yml
+++ b/gcc-post-install.yml
@ -0,0 +1,35 @@
+---
+- hosts: all
+  name: Dummy to gather facts
+  tasks: []
+
+- hosts: keystone
+  become: True
+  vars_files:
+    - settings.yml
+  tasks:
+    - name: copy public key
+      copy:
+        content: "{{ rsa_pub }}"
+        dest: /srv/keystone/root/id_rsa.pub
+    - name: post install configuration
+      command: docker exec -i keystone.service bash -c "source /root/admin-openrc.sh && {{ item }}"
+      with_items:
+        - openstack network create --share --external --provider-physical-network provider --provider-network-type vlan --provider-segment 985 vlan985
+        - >
+            openstack subnet create --subnet-range 172.23.34.0/24 --gateway 172.23.34.1
+            --network vlan985 --allocation-pool start=172.23.34.50,end=172.23.34.60
+            --dns-nameserver 8.8.4.4 vlan985_subnet
+        - openstack network create --share --external --provider-physical-network provider --provider-network-type vlan --provider-segment 16 vlan16
+        - >
+            openstack subnet create --subnet-range 195.169.22.0/23 --gateway 195.169.23.251
+            --network vlan16 --allocation-pool start=195.169.22.237,end=195.169.22.237
+            --dns-nameserver 8.8.4.4 vlan16_subnet
+
+        - openstack flavor create --ram 4096 --disk 40 --vcpus 2  "Molgenis Dual"
+        - openstack flavor create --ram 16384 --disk 40 --vcpus 4  "Molgenis Quad 16GB"
+        - openstack flavor create --ram 8192 --disk 40 --vcpus 4  "Molgenis Quad 8GB"
+
+        - openstack keypair create --public-key /root/id_rsa.pub adminkey
+
+
--- a/gcc-site.yml
+++ b/gcc-site.yml
@ -0,0 +1,14 @@
+---
+- include: common.yml
+- include: rabbitmq.yml
+- include: memcached.yml
+- include: mariadb.yml
+- include: keystone.yml
+- include: glance-controller.yml
+- include: nova-controller.yml
+- include: neutron-controller.yml
+- include: cinder-controller.yml
+- include: cinder-storage.yml
+- include: nova-compute.yml
+- include: horizon.yml
+- include: gcc-post-install.yml
--- a/generate_secrets.py
+++ b/generate_secrets.py
@ -5,6 +5,7 @@ Open the secrets.yml and replace all passwords.
 Original is backed up.
 """

+from os import path
 import random
 import string
 from subprocess import call
@ -27,7 +28,8 @@ for key, value in data.iteritems():
        for _ in range(pass_length))

 # Make numbered backups of the secrets file.
-call(['cp', '--backup=numbered', 'secrets.yml', 'secrets.yml.bak'])
+if path.isfile('secrets.yml'):
+    call(['cp', '--backup=numbered', 'secrets.yml', 'secrets.yml.bak'])

 with open('secrets.yml', 'w') as f:
    dump(data, f, Dumper=Dumper, default_flow_style=False)
--- a/heat.yml
+++ b/heat.yml
@ -0,0 +1,9 @@
+---
+- hosts: all
+  name: Dummy to gather facts
+  tasks: []
+
+- hosts: heat
+  become: True
+  roles:
+     - heat
--- a/4
+++ b/4
@ -34,7 +34,7 @@ openstack01-node03
 #run_options="-e CASSANDRA_SEEDS=172.23.41.1"

 [neutron-controller]
-openstack01-node01 provider_interface_name=ens192
+openstack01-node01 physical_interface_mappings=provider:ens192

 [nova-controller]
 openstack01-node03
@ -46,4 +46,4 @@ openstack01-node03
 openstack01-node01 storage_volume=/dev/loop0

 [nova-compute]
-openstack01-node04 provider_interface_name=dummy0
+openstack01-node04 physical_interface_mappings=provider:dummy0
--- a/mariadb.yml
+++ b/mariadb.yml
@ -4,3 +4,10 @@
  become: True
  roles:
      - mariadb
+  vars:
+      hostname_node0: "{{ hostvars[groups['databases'][0]]['ansible_hostname'] }}"
+      hostname_node1: "{{ hostvars[groups['databases'][1]]['ansible_hostname'] }}"
+      hostname_node2: "{{ hostvars[groups['databases'][2]]['ansible_hostname'] }}"
+      ip_node0: "{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
+      ip_node1: "{{ hostvars[groups['databases'][1]]['listen_ip'] | default(hostvars[groups['databases'][1]]['ansible_default_ipv4']['address']) }}"
+      ip_node2: "{{ hostvars[groups['databases'][2]]['listen_ip'] | default(hostvars[groups['databases'][2]]['ansible_default_ipv4']['address']) }}"
--- a/meta/main.yml
+++ b/meta/main.yml
@ -0,0 +1 @@
+---
--- a/35
+++ b/35
@ -0,0 +1,35 @@
+[databases]
+openstack03
+
+[keystone]
+openstack03
+
+[glance-controller]
+openstack03
+
+[horizon]
+openstack03
+
+[rabbitmq]
+openstack03
+
+[memcached]
+openstack03
+
+[neutron-controller]
+openstack03 physical_interface_mappings=provider:enp4s0f0
+
+[nova-controller]
+openstack03
+
+[cinder-controller]
+openstack03
+
+[cinder-storage]
+openstack03 storage_volume=/dev/sdb1
+
+[nova-compute]
+openstack03 physical_interface_mappings=provider:enp4s0f0
+
+[all:vars]
+listen_ip=172.23.40.243
--- a/35
+++ b/35
@ -0,0 +1,35 @@
+[databases]
+os-test
+
+[keystone]
+os-test
+
+[glance-controller]
+os-test
+
+[horizon]
+os-test
+
+[rabbitmq]
+os-test
+
+[memcached]
+os-test
+
+[neutron-controller]
+os-test physical_interface_mappings=provider:enp4s0f0
+
+[nova-controller]
+os-test
+
+[cinder-controller]
+os-test
+
+[cinder-storage]
+os-test storage_volume=/dev/sdb
+
+[nova-compute]
+os-test physical_interface_mappings=provider:enp4s0f0
+
+[all:vars]
+listen_ip=129.125.60.194
--- a/post-install.yml
+++ b/post-install.yml
@ -19,7 +19,7 @@
        - >
            openstack subnet create --network provider
            --allocation-pool start={{ allocation_pool['start'] }},end={{ allocation_pool['end'] }}
-            --dns-nameserver {{ dns_nameserver }} --gateway {{ gateway }} --subnet-range {{ subnet_range }} provider
+            --dns-nameserver {{ dns_nameserver }} --gateway {{ gateway }} --subnet-range {{ subnet_range }} providersub
        - openstack flavor create --id 0 --vcpus 1 --ram 64 --disk 1 m1.nano
        - openstack keypair create --public-key /root/id_rsa.pub adminkey

--- a/roles/cassandra/tasks/main.yml
+++ b/roles/cassandra/tasks/main.yml
@ -7,9 +7,14 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
 - name: make sure service is started
  systemd:
    name: cassandra.service
    state: started
+
+- name: start service at boot.
+  command: systemctl reenable cassandra.service
--- a/roles/cinder-controller/tasks/main.yml
+++ b/roles/cinder-controller/tasks/main.yml
@ -6,18 +6,18 @@
    name: secrets

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-cinder-controller:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-controller:latest
    env_vars: >
-        -e "MY_IP={{ ansible_default_ipv4.address }}"
-        -e "CINDER_HOST={{ hostvars[groups['cinder-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "CINDER_PASSWORD={{ secrets['CINDER_PASSWORD'] }}"
        -e "CINDER_USER=cinder"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"

@ -43,6 +43,9 @@
    owner: root
    group: root

+- name: start service at boot.
+  command: systemctl reenable cinder-controller.service
+
 - command: systemctl daemon-reload

 - name: Initialize database.
--- a/roles/cinder-storage/tasks/main.yml
+++ b/roles/cinder-storage/tasks/main.yml
@ -4,22 +4,24 @@
  include_vars:
    file: ../../secrets.yml
    name: secrets
+  tags: vars

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-cinder-storage:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage:latest
    env_vars: >
-        -e "MY_IP={{ ansible_default_ipv4.address }}"
-        -e "CINDER_HOST={{ hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address']) }}"
        -e "CINDER_PASSWORD={{ secrets['CINDER_PASSWORD'] }}"
        -e "CINDER_USER=cinder"
-        -e "GLANCE_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"
+  tags: vars

 - name: pull docker image
  docker_image:
@ -52,8 +54,13 @@
    mode: 644
    owner: root
    group: root
+  tags: systemd

 - command: systemctl daemon-reload
+  tags: systemd
+
+- name: start service at boot.
+  command: systemctl reenable cinder-storage.service

 - name: make sure service is started
  systemd:
--- a/roles/cinder-storage/templates/cinder-storage.service
+++ b/roles/cinder-storage/templates/cinder-storage.service
@ -1,5 +1,5 @@
 [Unit]
-Description=Openstack Glance Container
+Description=Openstack Cinder Storage container
 After=docker.service
 Requires=docker.service

@ -11,9 +11,13 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  --privileged \
  {{ env_vars | replace('\n', '') }} \
+  -v "/dev/cinder-volumes/":/dev/cinder-volumes \
  -v /srv/cinder-storage/root:/root \
  -v "{{ storage_volume }}":/dev/cinder_storage_volume \
-  -p 8776:8776 \
+  -v "/dev/lvm":/dev/lvm \
+  -v "/srv/cinder-storage/volumes/:/var/lib/cinder/volumes/" \
+  -p 8777:8776 \
+  -p 3260:3260 \
  {{ docker_image }}

 [Install]
--- a/roles/common/tasks/docker.yml
+++ b/roles/common/tasks/docker.yml
@ -13,3 +13,8 @@
  with_items:
     - docker-engine
     - python-docker
+
+- name: make sure service is started
+  systemd:
+    name: docker.service
+    state: started
--- a/roles/dockerregistry/tasks/main.yml
+++ b/roles/dockerregistry/tasks/main.yml
@ -7,13 +7,18 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
 - name: make sure service is started
  systemd:
    name: dockerregistry.service
    state: started

+- name: start service at boot.
+  command: systemctl reenable dockerregistry.service
+
 - name: Copy certificates and passwd file
  copy:
    src: "{{ item }}"
--- a/roles/glance-controller/tasks/main.yml
+++ b/roles/glance-controller/tasks/main.yml
@ -6,17 +6,17 @@
    name: secrets

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-glance:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance:latest
    env_vars: >
-        -e "GLANCE_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "GLANCE_PASSWORD={{ secrets['GLANCE_PASSWORD'] }}"
        -e "GLANCE_USER=glance"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"

@ -42,14 +42,18 @@
    owner: root
    group: root

+- name: start service at boot.
+  command: systemctl reenable glance.service
+
 - command: systemctl daemon-reload

 - name: Initialize database.
  command: >
             /usr/bin/docker run --rm
             {{ env_vars }}
-             --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+             --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
             -v /srv/glance/root:/root \
+             -v /var/lib/glance/images:/var/lib/glance/images \
             {{ docker_image }} /etc/bootstrap.sh
  tags: bootstrap

--- a/roles/glance-controller/templates/glance.service
+++ b/roles/glance-controller/templates/glance.service
@ -11,6 +11,7 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
  -v /srv/glance/root:/root \
+  -v /var/lib/glance/images:/var/lib/glance/images \
  -p 9292:9292 \
  {{ docker_image }}

--- a/roles/heat/tasks/main.yml
+++ b/roles/heat/tasks/main.yml
@ -0,0 +1,62 @@
+# Build and install a docker image for heat.
+---
+- name: include secrets
+  include_vars:
+    file: ../../secrets.yml
+    name: secrets
+
+- set_fact:
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-heat:latest
+    env_vars: >
+        -e "HEAT_HOST={{ listen_ip | default(hostvars[groups['heat'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "HEAT_PASSWORD={{ secrets['HEAT_PASSWORD'] }}"
+        -e "HEAT_USER=heat"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
+        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
+        -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
+        -e "RABBIT_USER=openstack"
+
+- name: pull docker image
+  docker_image:
+    name: "{{ docker_image }}"
+  tags: pull
+
+- name: Make build and persistent directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0777
+  with_items:
+      - /srv/heat
+      - /srv/heat/root
+
+- name: install service file.
+  template:
+    src: templates/heat.service
+    dest: /etc/systemd/system/heat.service
+    mode: 644
+    owner: root
+    group: root
+
+- name: start service at boot.
+  command: systemctl reenable heat.service
+
+- command: systemctl daemon-reload
+
+- name: Initialize database.
+  command: >
+             /usr/bin/docker run --rm
+             {{ env_vars }}
+             --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
+             -v /srv/heat/root:/root \
+             {{ docker_image }} /etc/bootstrap.sh
+  tags: bootstrap
+
+- name: make sure service is started
+  systemd:
+    name: heat.service
+    state: restarted
--- a/roles/heat/templates/heat.service
+++ b/roles/heat/templates/heat.service
@ -0,0 +1,19 @@
+[Unit]
+Description=Openstack heat Container
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker kill %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStart=/usr/bin/docker run --name %n \
+  {{ env_vars | replace('\n', '') }} \
+  -v /srv/heat/root:/root \
+  -p 8000:8000 \
+  -p 8004:8004 \
+  {{ docker_image }}
+
+[Install]
+WantedBy=multi-user.target
--- a/roles/horizon/tasks/main.yml
+++ b/roles/horizon/tasks/main.yml
@ -1,7 +1,7 @@
 # Run hpc/horizon
 ---
 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-horizon:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-horizon:latest

 - name: pull docker image
  docker_image:
@ -19,6 +19,9 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable horizon.service
+
 - name: make sure service is started
  systemd:
    name: horizon.service
--- a/roles/horizon/templates/horizon.service
+++ b/roles/horizon/templates/horizon.service
@ -9,9 +9,9 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-  -e "MEMCACHED_SERVER={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}" \
-  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  -e "MEMCACHED_SERVER={{ hostvars[groups['memcached'][0]]['listen_ip'] | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}" \
+  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  -p 80:80 \
  {{ docker_image }}

--- a/roles/keystone/tasks/main.yml
+++ b/roles/keystone/tasks/main.yml
@ -17,7 +17,7 @@
    - /srv/keystone/root

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-keystone:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-keystone:latest

 - name: pull docker image
  docker_image:
@ -36,11 +36,14 @@
 - name: install service file
  command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable keystone.service
+
 - name: Initialize db
  script: scripts/initialize_db.sh
  environment:
    MYSQL_ROOT_PASSWORD: "{{ secrets['MYSQL_ROOT_PASSWORD'] }}"
-    DB_HOST: "{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+    DB_HOST: "{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
  register: result
  until: result|succeeded
  # sometimes the initial connect fails.
@ -52,9 +55,9 @@
 - name: keystone manage commands to setup db
  command: >
             /usr/bin/docker run --rm
-             --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
+             --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
             -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys
-             -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+             -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
             {{ docker_image }} keystone-manage {{ item }}
  with_items:
      - db_sync
@ -62,9 +65,9 @@
      - credential_setup --keystone-user keystone --keystone-group keystone
      - >
          bootstrap --bootstrap-password {{ secrets['OS_PASSWORD'] }}
-                    --bootstrap-admin-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:35357/v3/
-                    --bootstrap-internal-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:35357/v3/
-                    --bootstrap-public-url http://{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}:5000/v3/
+                    --bootstrap-admin-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:35357/v3/
+                    --bootstrap-internal-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:35357/v3/
+                    --bootstrap-public-url http://{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}:5000/v3/
                    --bootstrap-region-id RegionOne

 - name: make sure service is started
@ -75,10 +78,29 @@
 - name: Create a domain, projects users and roles
  command: >
             /usr/bin/docker run --rm
-             --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
+             --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
             -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys
             -v /srv/keystone/root:/root
-             -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+             -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
             -e "OS_AUTH_URL=http://${KEYSTONE_HOST}:35357/v3"
             -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
             {{ docker_image }} bash /etc/bootstrap.sh
+
+
+- name: install openstack repo on host.
+  command: >
+      echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main" > /etc/apt/sources.list.d/ocata.list &&
+      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+  tags: openstackclient
+
+- name: install openstack client for management
+  apt:
+    name: python-openstackclient
+    state: latest
+    update_cache: yes
+  tags: openstackclient
+
+- name: source admin-openrc.sh in root .bashrc
+  lineinfile:
+    path: /root/.bashrc
+    line: 'source /srv/keystone/root/admin-openrc.sh'
--- a/roles/keystone/templates/keystone.service
+++ b/roles/keystone/templates/keystone.service
@ -9,8 +9,8 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
  -p 5000:5000 -p 35357:35357 \
  -v /srv/keystone/fernet-keys:/etc/keystone/fernet-keys \
  -v /srv/keystone/root:/root \
--- a/roles/mariadb/files/galera.cnf
+++ b/roles/mariadb/files/galera.cnf
@ -0,0 +1,20 @@
+[mysqld]
+binlog_format=ROW
+default-storage-engine=innodb
+innodb_autoinc_lock_mode=2
+bind-address=0.0.0.0
+
+# Galera Provider Configuration
+wsrep_on=ON
+wsrep_provider=/usr/lib/galera/libgalera_smm.so
+
+# Galera Cluster Configuration
+wsrep_cluster_name="test_cluster"
+wsrep_cluster_address="gcomm://{{ ip_node0 }},{{ ip_node1 }},{{ ip_node2 }}"
+
+# Galera Synchronization Configuration
+wsrep_sst_method=rsync
+
+# Galera Node Configuration
+wsrep_node_address="{{ listen_ip | default(ansible_default_ipv4.address) }}"
+wsrep_node_name="{{ ansible_nodename }}"
--- a/roles/mariadb/tasks/main.yml
+++ b/roles/mariadb/tasks/main.yml
@ -5,14 +5,6 @@
    file: ../../secrets.yml
    name: secrets

- name: install service file.
-  template:
-    src: templates/mysql.service
-    dest: /etc/systemd/system/mysql.service
-    mode: 644
-    owner: root
-    group: root
-
 - name: make mariadb settings volume
  file:
    path: "{{ item }}"
@ -21,16 +13,60 @@
  with_items:
      - /srv/mariadb/lib/mysql
      - /srv/mariadb/etc/mysql
+      - /srv/mariadb/etc/mysql/conf.d

 - name: place settings file
  copy:
    src: files/my.cnf
-    dest: /srv/mariadb/etc/mysql
+    dest: /srv/mariadb/etc/mysql/conf.d/my.cnf
    mode: 660

- command: systemctl daemon-reload
+- name: Set galara.cnf on node if we have at least three nodes.
+  template:
+    src: files/galera.cnf
+    dest: /srv/mariadb/etc/mysql/conf.d/galera.cnf
+    mode: 660
+  when: groups['databases'] | length >= 3
+
+  # This mimics galera_new_cluster.sh
+- name: Initialize a new cluster.
+  block:
+    - set_fact:
+        mariadb_args: "--wsrep-new-cluster"
+
+    - template:
+        src: templates/mysql.service
+        dest: /etc/systemd/system/mysql.service
+        mode: 644
+        owner: root
+        group: root
+
+    - command: systemctl daemon-reload
+
+    - systemd:
+        name: mysql.service
+        state: started
+
+  when: groups['databases'] | length >= 3 and ansible_hostname == hostname_node0
+
+- name: install service file.
+  block:
+    - set_fact:
+        mariadb_args: ""
+    - template:
+        src: templates/mysql.service
+        dest: /etc/systemd/system/mysql.service
+        mode: 644
+        owner: root
+        group: root
+
+- name: Give the master node some time to initialize the cluster.
+  command: bash -c "sleep 60 && systemctl daemon-reload"

 - name: make sure service is started
  systemd:
    name: mysql.service
    state: started
+
+- name: start service at boot.
+  command: systemctl reenable mysql.service
--- a/roles/mariadb/templates/mysql.service
+++ b/roles/mariadb/templates/mysql.service
@ -9,10 +9,11 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n || /bin/true
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStartPre=/usr/bin/docker pull mariadb:10.2
-ExecStart=/usr/bin/docker run  -p 3306:3306 --name %n \
+ExecStart=/usr/bin/docker run --name %n \
+ --network host \
 -v /srv/mariadb/lib/mysql:/var/lib/mysql \
- -v /srv/mariadb/etc/mysql:/etc/mysql \
- -e MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }} mariadb:10.2
+ -v /srv/mariadb/etc/mysql/conf.d:/etc/mysql/conf.d \
+ -e MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }} mariadb:10.2 {{ mariadb_args }}

 [Install]
 WantedBy=multi-user.target
--- a/roles/memcached/tasks/main.yml
+++ b/roles/memcached/tasks/main.yml
@ -7,8 +7,13 @@
    mode: 644
    owner: root
    group: root
+
 - name: install service file
  command: systemctl daemon-reload
+
+- name: start service at boot.
+  command: systemctl reenable memcached.service
+
 - name: make sure service is started
  systemd:
    name: memcached.service
--- a/roles/neutron-controller/tasks/main.yml
+++ b/roles/neutron-controller/tasks/main.yml
@ -6,7 +6,7 @@
    name: secrets

 - set_fact:
-    docker_image: "webhost12.service.rug.nl/hpc/openstack-neutron-controller:latest"
+    docker_image: "registry.webhosting.rug.nl/hpc/openstack-neutron-controller:latest"

 - name: pull docker image
  docker_image:
@ -16,21 +16,21 @@

 - set_fact:
    env_vars: >
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
        -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}"
-        -e "MY_IP={{ hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "MY_IP={{ listen_ip | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
        -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}"
        -e "NEUTRON_USER=neutron"
        -e "NOVA_USER=nova"
        -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}"
-        -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NOVA_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_USER=placement"
        -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
-        -e "PROVIDER_INTERFACE_NAME={{ provider_interface_name }}"
+        -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}"
        -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
        -e "RABBIT_USER=openstack"
  tags: env
@ -45,12 +45,15 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable neutron-controller.service
+
 - name: Initialize neutron
  command: >
    /usr/bin/docker run --rm
    {{ env_vars }}
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}
+    --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
+    --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}
    --network host
    {{ docker_image }}
    /etc/bootstrap.sh
--- a/roles/neutron-controller/templates/neutron-controller.service
+++ b/roles/neutron-controller/templates/neutron-controller.service
@ -10,9 +10,9 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
-  --add-host=nova-controller:{{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  --add-host=nova-controller:{{ hostvars[groups['nova-controller'][0]]['listen_ip'] | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  --add-host={{ ansible_nodename }}:{{ ansible_default_ipv4.address }} \
  --privileged \
  --network host \
--- a/roles/nova-compute/tasks/main.yml
+++ b/roles/nova-compute/tasks/main.yml
@ -4,10 +4,11 @@
  include_vars:
    file: ../../secrets.yml
    name: secrets
+  tags: vars

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-nova-compute:latest
-  tags: facts
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-compute:latest
+  tags: vars

 - name: pull docker image
  docker_image:
@ -22,11 +23,13 @@
    mode: 644
    owner: root
    group: root
+  tags: systemd

 - command: systemctl daemon-reload
+  tags: systemd

 - apt:
-    name: '{{ item }}'
+    name: "{{ item }}"
  with_items:
      - kvm
      - libvirt0
@ -38,6 +41,9 @@
    name: nova-compute.service
    state: restarted

+- name: start service at boot.
+  command: systemctl reenable nova-compute.service
+
 - name: let nova controler discover new host
-  command: docker exec -i nova-controller.service nova-manage cell_v2 discover_hosts
+  shell: "sleep 10 && docker exec -i nova-controller.service nova-manage cell_v2 discover_hosts"
  delegate_to: "{{ hostvars[groups['nova-controller'][0]]['ansible_hostname'] }}"
--- a/roles/nova-compute/templates/nova-compute.service
+++ b/roles/nova-compute/templates/nova-compute.service
@ -9,30 +9,32 @@ Restart=always
 ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
-    -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MY_IP={{ hostvars[groups['nova-compute'][0]]['ansible_default_ipv4']['address'] }}" \
-    -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['listen_ip'] | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}" \
+    -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['listen_ip'] | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}" \
+    -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}" \
+    -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}" \
-    -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['listen_ip'] | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}" \
    -e "NEUTRON_USER=neutron" \
    -e "NOVA_COMPUTE_USER=nova_compute" \
-    -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['listen_ip'] | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}" \
    -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}" \
    -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}" \
    -e "NOVA_PLACEMENT_USER=placement" \
    -e "NOVA_USER=nova" \
    -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}" \
-    -e "PROVIDER_INTERFACE_NAME={{ provider_interface_name }}" \
-    -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}" \
+    -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}" \
+    -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['listen_ip'] | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}" \
    -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}" \
    -e "RABBIT_USER=openstack" \
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
+    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
+    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
    --privileged \
+    -v /dev:/dev \
    -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
    -v /var/lib/nova/instances:/var/lib/nova/instances \
    -v /lib/modules:/lib/modules \
--- a/roles/nova-controller/tasks/main.yml
+++ b/roles/nova-controller/tasks/main.yml
@ -15,18 +15,19 @@
    - /srv/nova-controller/root

 - set_fact:
-    docker_image: webhost12.service.rug.nl/hpc/openstack-nova-service:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-service:latest
    env_vars: >
-        -e "GLANCE_CONTROLLER_HOST={{ hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "KEYSTONE_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MEMCACHED_HOST={{ hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MY_IP={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
-        -e "MYSQL_HOST={{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "GLANCE_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "KEYSTONE_HOST={{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "METADATA_SECRET={{ secrets['METADATA_SECRET'] }}"
+        -e "MEMCACHED_HOST={{ listen_ip | default(hostvars[groups['memcached'][0]]['ansible_default_ipv4']['address']) }}"
+        -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
+        -e "MYSQL_HOST={{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
        -e "MYSQL_ROOT_PASSWORD={{ secrets['MYSQL_ROOT_PASSWORD'] }}"
-        -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NEUTRON_CONTROLLER_HOST={{ hostvars[groups['neutron-controller'][0]]['listen_ip'] | default(hostvars[groups['neutron-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NEUTRON_PASSWORD={{ secrets['NEUTRON_PASSWORD'] }}"
        -e "NEUTRON_USER=neutron"
-        -e "NOVA_CONTROLLER_HOST={{ hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address'] }}"
+        -e "NOVA_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}"
        -e "NOVA_PASSWORD={{ secrets['NOVA_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}"
        -e "NOVA_PLACEMENT_USER=placement"
@ -52,12 +53,15 @@

 - command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable nova-controller.service
+
 - name: Initialize database.
  command: >
    /usr/bin/docker run --rm
    {{ env_vars }}
-    --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }}
-    --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }}
+    --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
+    --add-host=keystone:{{ listen_ip | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }}
    -v /srv/nova-controller/root:/root
    {{ docker_image }}
    /etc/bootstrap.sh
--- a/roles/nova-controller/templates/nova-controller.service
+++ b/roles/nova-controller/templates/nova-controller.service
@ -10,11 +10,12 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
  {{ env_vars | replace('\n', '') }} \
-  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['ansible_default_ipv4']['address'] }} \
-  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address'] }} \
+  --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
+  --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
  --privileged \
  -v /srv/nova-controller/root:/root \
  -p 8774:8774 \
+  -p 8775:8775 \
  -p 8778:8778 \
  -p 6080:6080 \
  {{ docker_image }} /etc/run.sh
--- a/roles/rabbitmq/files/rabbitmq.service
+++ b/roles/rabbitmq/files/rabbitmq.service
@ -10,11 +10,11 @@ ExecStartPre=-/usr/bin/docker kill %n
 ExecStartPre=-/usr/bin/docker rm %n
 ExecStartPre=/usr/bin/docker pull rabbitmq:latest
 ExecStart=/usr/bin/docker run \
-          --add-host "{{ hostvars[groups['rabbitmq'][0]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address'] }}" \
-          --add-host "{{ hostvars[groups['rabbitmq'][1]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][1]]['ansible_default_ipv4']['address'] }}" \
-          --add-host "{{ hostvars[groups['rabbitmq'][2]]['ansible_hostname'] }}:{{ hostvars[groups['rabbitmq'][2]]['ansible_default_ipv4']['address'] }}" \
+{% for host in groups['rabbitmq'] %}
+          --add-host "{{ host }}:{{ hostvars[host]['listen_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}" \
+{% endfor %}
          -p 4369:4369 -p 25679:25679 -p 25672:25672 -p 5671-5672:5671-5672 -p 8080:15672 \
-          -e "RABBITMQ_DEFAULT_USER=user" -e "RABBITMQ_DEFAULT_PASS={{ secrets['RABBIT_PASSWORD'] }}" \
+          -e "RABBITMQ_DEFAULT_USER=openstack" -e "RABBITMQ_DEFAULT_PASS={{ secrets['RABBIT_PASSWORD'] }}" \
          -e "RABBITMQ_ERLANG_COOKIE={{ secrets['RABBITMQ_ERLANG_COOKIE'] }}" \
          -e "RABBITMQ_NODENAME=rabbit_{{ ansible_nodename }}" \
          --hostname "{{ ansible_nodename }}" --name %n rabbitmq:3-management
--- a/roles/rabbitmq/tasks/main.yml
+++ b/roles/rabbitmq/tasks/main.yml
@ -5,9 +5,6 @@
    file: ../../secrets.yml
    name: secrets

- include_vars:
-    dir: 'vars'
-
 - name: install service file.
  template:
    src: files/rabbitmq.service
@ -19,6 +16,9 @@
 - name: install service file
  command: systemctl daemon-reload

+- name: start service at boot.
+  command: systemctl reenable rabbitmq.service
+
 - name: make sure service is started
  systemd:
    name: rabbitmq.service
@ -27,6 +27,7 @@
 - name: wait for container to be started
  wait_for:
      port: 5672
+      delay: 5

 - name: setup the cluster
  command: "docker exec -i rabbitmq.service {{ item }}"
@ -36,11 +37,3 @@
      - rabbitmqctl start_app
  when: ansible_nodename != hostname_node0

- name: create openstack user
-  command: "docker exec -i rabbitmq.service {{ item }}"
-  with_items:
-      - rabbitmqctl add_user openstack "{{ secrets['RABBIT_PASSWORD'] }}"
-      - rabbitmqctl set_permissions openstack ".*" ".*" ".*"
-  when: ansible_nodename == hostname_node0
-  register: command_result
-  failed_when: "command_result.rc not in (0, 70)"
--- a/secrets.yml
+++ b/secrets.yml
@ -1,30 +1,30 @@
 $ANSIBLE_VAULT;1.1;AES256
-62633134346438356462333363626164393762356139653666323461333037393536373631653565
-6631306631333538353534663738313062636232633339610a303161323131373739393735666463
-65353135626430353737373239623361306137326334333761626235353463393465383830666666
-6138616530346563310a306263316331346263356139383435316239346230313266636363313564
-36633130393062373936363765636361343939313639326237633337353665666338633338343837
-34613534333063303537323738396436333964613362636664366264313334663365336132623464
-64656131373261376466356638636338643135393139386534626132323262393064626666323462
-64323664373262356632393465653932303939313338656665336639613966626234636666373163
-35633231666338643863623737396435626364333365656536613130666435323837323136663339
-61363936336434656530313538643463663737613831646265313731363734356635356438353062
-34323063346265393737343834343065616139656234666230323131366138396265393737666236
-39353766643239323339623534393962666432656331323462656439306365613539366230643133
-36316138303361313134336431343137343433383430616137376563383233303432383664333930
-61613531313638303531643232343066376565663032326533313461363839383664366338356439
-37363233666663653736376538386536653262653633323065363830623032363063393635653762
-32636365656362323362303962306538336234626533323830656230386432666461343063663832
-62373133343933353563653762333836333862376232353339313662363865616439623635393839
-37346433346264633036343761613230396434366132653261643137386466326235613030306235
-34333065623232303939623233373762393939653639333734336336303762326662386530356563
-65303165623564303635356337353662363433626466653939323438633938386166386262623435
-64376431396631623034386434393431616631363663393835343035313639663538643565616330
-65353365303131326335646164333231306564383936396139643935646331393235326666336230
-38326165663865343966356335326438303133663239656235313935626332323332376665343132
-62336139643262333938303537313533623535333736643163373137343035393034613939663061
-36323063643734343865333138356434643266663436653435353132386330636238343637653434
-65616361333263336332643262623034343439383737366663373166643433653466313237613930
-32373162646461323266353662326134343839613264313339306430366165633838663831666565
-65333337623962313561306333616232393334353934316565666331336561633934623339353138
-62656339386530333036383831613762353234643461656436623033613930353531
+35643437313834633532373265366630663035336231306639623561613765386332663334343237
+3339363162303463353437326331656532336138373066620a623137643762383532376361353364
+37646236386466353636396535376463333133323664316634663466663164303063383830653039
+3535666361303562630a316137376531636537383138663662373865383431343035646539356137
+38323866643831353537366630363333663865383261633938346664633362343661343839383766
+66363733356333303334323136376136353738376362376231353338343763663131363731343639
+61383138626235633663666430383964616239363035663663646133636434363032626633663865
+30663732646630393163653461626435333463396463333236313930346461626364626166386365
+66323736316230376165666366363136666533376335316132343361393532616536383965363339
+30376362356665633630393561653532613139366236663961643864383738353430666562623730
+34663166393665653265663836623731386235633062306562373935633737363639383336303539
+37663763623664623038316438356138363134646230643261646262353163333430616462393866
+31666233636233356464633436626637313633623736343264613037353432386131393964386663
+36353236613662633764366437306461316138366461653731373436613039346663663536653362
+38656636303935626563303732666261373665303035333661643865393166653330646336393961
+31646539396131626464313733383638656438613530663166393035343630353764313232323432
+34386334666231323261343765623636313032373835396332623037613866613636393038653266
+36336531356534633933383432646663663364376130386239613836336263623161326563346661
+33636232313866613662353661373533383138393434396338343934326333326238336638396462
+65376133343038313437343934373265333632663133653133656130636533663237623839623634
+35363764363763363465363437623964363362616261663166633066373033633864336532633031
+32323733616562663031303230383561373637326436336462363461313532623262653866323862
+34643631333533626537373538353564306261313035303530666462326534633638363932363037
+65336230373034643966656561303164373463353638316632613431643535303930373334383134
+38323731363535313065326330653666323934636466386238616664316635303333653631396639
+39303737613361653862343964303231393164346134633366633262326230643137303331373231
+31323832363937663935333737613133323265323863623933633962633230386339636432643937
+66653763376663666637353738646565343835333937343765356539383734316231623466343634
+30663135663938393561333133663737653635393432333534306466366332333338
--- a/secrets.yml.topol
+++ b/secrets.yml.topol
@ -9,3 +9,5 @@ OS_PASSWORD:   # Keystone admin password
 OS_DEMO_PASSWORD: # Keystone demo user password
 RABBIT_PASSWORD:
 RABBITMQ_ERLANG_COOKIE:
+CINDER_PASSWORD:
+HEAT_PASSWORD:
--- a/site.yml
+++ b/site.yml
@ -7,6 +7,9 @@
 - include: glance-controller.yml
 - include: nova-controller.yml
 - include: neutron-controller.yml
+- include: cinder-controller.yml
+- include: cinder-storage.yml
 - include: nova-compute.yml
 - include: horizon.yml
+- include: heat.yml
 - include: post-install.yml
--- a/12
+++ b/12
@ -1,5 +1,7 @@
 [databases]
+ansible-test
 ansible-test-2
+ansible-test-3

 [keystone]
 ansible-test-3
@ -15,14 +17,20 @@ ansible-test
 ansible-test-2
 ansible-test-3

+[cinder-storage]
+ansible-test
+
 [memcached]
 ansible-test-3

 [neutron-controller]
-ansible-test provider_interface_name=ens10
+ansible-test physical_interface_mappings=provider:ens10

 [nova-controller]
 ansible-test

 [nova-compute]
-ansible-test-2 provider_interface_name=ens10
+ansible-test-2 physical_interface_mappings=provider:ens10
+
+[heat]
+ansible-test