Fixed syntax error.

.. without running other tasks

ansible.cfg

@@ -1,2 +1,5 @@
 [defaults]
 inventory = merlin
+stdout_callback = debug
+vault_password_file = .vault_pass.txt
+forks = 20

ceph.xml

@@ -0,0 +1,6 @@
+<secret ephemeral="no" private="no">
+<uuid>d0db6ba7-a0c9-4da6-b0bc-aa7846325333</uuid>
+<usage type="ceph">
+<name>client.volumes secret</name>
+</usage>
+</secret>

cinder-controller.yml

@@ -2,6 +2,8 @@
 - hosts: all
   name: Dummy to gather facts
   tasks: []
+  tags:
+     - facts
 
 - hosts: cinder-controller
   become: True

create-non-administrative-user

@@ -0,0 +1,5 @@
+openstack project create --domain default --description "GCC testproject" gcc
+openstack user create --domain default --password-prompt gcc-user
+openstack role add --project gcc --user gcc-user user
+openstack user create --domain default --password-prompt gcc-admin
+openstack role add --project gcc --user gcc-admin admin

group_vars/all.yml

@@ -0,0 +1,9 @@
+---
+use_ceph: true
+ceph_mon_initial_members:
+ceph_mon_host:
+ceph_public_network:
+ceph_osd_pool_default_size:
+ceph_compute_client_keyring:
+ceph_cinder_client_keyring:
+ceph_images_client_keyring:

heat_templates/example_cluster.yml

@@ -0,0 +1,137 @@
+---
+heat_template_version: 2015-04-30
+
+description: Simple Example template to deploy a virtual compute cluster.
+
+parameters:
+  image_name:
+    type: string
+    label: Image Name
+    description: Name of image to be used for compute instance
+  public_net:
+    type: string
+    label: Public Net Name
+    description: Public network used for router.
+  ssh_key:
+    type: string
+    label: ssh key name.
+    description: ssh public key name. (Must be uploaded to openstack first)
+  compute_flavor:
+    type: string
+    label: Flavor for compute nodes,
+    description: Flavor with which to start compute nodes.
+  aux_flavor:
+    type: string
+    label: Flavor for auxiliary nodes.
+    description: Flavor with which to start auxiliary nodes. (for now only the interface machine)
+  cidr:
+    type: string
+    label: Cidr for internal subnet
+    description: Cidr for the subnet of the internal user network.
+  internal_net_name:
+    type: string
+    label: Name for the internal network
+    description: Name for the internal network of this cluster.
+  volume_size:
+    type: string
+    label: Size (GB)
+    description: Size (GB) of the volume for each compute node
+
+
+resources:
+  internal_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {get_param: internal_net_name}
+
+  internal_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: {get_resource: internal_net}
+      cidr: {get_param: cidr}
+      dns_nameservers: ["129.125.4.6", "129.125.36.10"]
+      ip_version: 4
+
+  internal_router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info: {network: {get_param: public_net}}
+
+  internal_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: {get_resource: internal_router}
+      subnet: {get_resource: internal_subnet}
+
+  public_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: {get_resource: internal_net}
+      fixed_ips:
+        - subnet_id: {get_resource: internal_subnet}
+      security_groups:
+        - default
+
+  floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: {get_param: public_net}
+      port_id: {get_resource: public_port}
+
+  interface:  # User-interface for cluster-operation
+    type: OS::Nova::Server
+    properties:
+      key_name: {get_param: ssh_key}
+      image: {get_param: image_name}
+      flavor: {get_param: aux_flavor}
+      networks:
+        - port: {get_resource: public_port}
+
+  admin:  # Machine to run slurm and other admin tools on.
+    type: OS::Nova::Server
+    properties:
+      key_name: {get_param: ssh_key}
+      image: {get_param: image_name}
+      flavor: {get_param: aux_flavor}
+      networks:
+        - network: {get_resource: internal_net}
+
+  vcompute01-volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: {get_param: volume_size}
+
+  vcompute01:
+    type: OS::Nova::Server
+    properties:
+      key_name: adminkey
+      image: {get_param: image_name}
+      flavor: {get_param: compute_flavor}
+      networks:
+        - network: {get_resource: internal_net}
+
+  volume_attachment:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      volume_id: {get_resource: vcompute01-volume}
+      instance_uuid: {get_resource: vcompute01}
+
+  vcompute02-volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: {get_param: volume_size}
+
+  vcompute02:
+    type: OS::Nova::Server
+    properties:
+      key_name: adminkey
+      image: {get_param: image_name}
+      flavor: {get_param: compute_flavor}
+      networks:
+        - network: {get_resource: internal_net}
+
+  volume_attachment:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      volume_id: {get_resource: vcompute02-volume}
+      instance_uuid: {get_resource: vcompute02}

hosts-openstack03

@@ -0,0 +1,26 @@
+[databases]
+openstack03.gcc.rug.nl
+
+[keystone]
+openstack03.gcc.rug.nl
+
+[glance-controller]
+openstack03.gcc.rug.nl
+
+[horizon]
+openstack03.gcc.rug.nl
+
+[rabbitmq]
+openstack03.gcc.rug.nl
+
+[memcached]
+openstack03.gcc.rug.nl
+
+[neutron-controller]
+openstack03.gcc.rug.nl
+
+[nova-controller]
+openstack03.gcc.rug.nl
+
+[nova-compute]
+openstack03.gcc.rug.nl

merlin

@@ -1,9 +1,9 @@
 [nova-compute]
-merlin-managementnode002 physical_interface_mappings=provider:eno3
+merlin-node001 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.1
-merlin-managementnode003 physical_interface_mappings=provider:eno3
+merlin-node002 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.2
-merlin-node001 physical_interface_mappings=provider:eno3
+merlin-node003 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.3
-merlin-node003 physical_interface_mappings=provider:eno3
+merlin-node004 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.4
-merlin-node004 physical_interface_mappings=provider:eno3
+merlin-node005 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.5
 
 [databases]
 merlin-managementnode001
@@ -28,10 +28,23 @@ merlin-managementnode001
 merlin-managementnode001
 
 [neutron-controller]
-merlin-managementnode001 physical_interface_mappings=provider:eno3
+merlin-managementnode001 physical_interface_mappings=provider:enp5s0f1 overlay_ip=172.23.43.101
 
 [heat]
 merlin-managementnode001
 
 [glance-controller]
-merlin-managementnode001
+merlin-managementnode002
+
+[cinder-controller]
+merlin-managementnode003
+
+[cinder-storage]
+merlin-node001
+merlin-node002
+merlin-node003
+merlin-node004
+merlin-node005
+
+[stor]
+merlin-stor00[1:8]

merlin2

@@ -0,0 +1,50 @@
+[nova-compute]
+merlin-node008 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.8
+merlin-node009 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.9
+merlin-node010 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.10
+merlin-node011 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.11
+merlin-node012 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.12
+merlin-node013 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.13
+
+[databases]
+merlin-node007
+merlin-node008
+merlin-node009
+
+[rabbitmq]
+merlin-node007
+merlin-node008
+merlin-node009
+
+[horizon]
+merlin-node007
+
+[memcached]
+merlin-node007
+
+[nova-controller]
+merlin-node007
+
+[keystone]
+merlin-node007
+
+[neutron-controller]
+merlin-node007 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.6
+
+[heat]
+merlin-node007
+
+[glance-controller]
+merlin-node008
+
+[cinder-controller]
+merlin-node009
+
+[cinder-storage]
+merlin-node008
+merlin-node009
+merlin-node010
+merlin-node011
+merlin-node012
+merlin-node013
+

merlinsdn

@@ -0,0 +1,37 @@
+[nova-compute]
+merlin-managementnode002 physical_interface_mappings=provider:eno3
+merlin-managementnode003 physical_interface_mappings=provider:eno3
+merlin-node001 physical_interface_mappings=provider:eno3
+merlin-node003 physical_interface_mappings=provider:eno3
+merlin-node004 physical_interface_mappings=provider:eno3
+
+[databases]
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
+
+[rabbitmq]
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
+
+[horizon]
+merlin-managementnode001
+
+[memcached]
+merlin-managementnode001
+
+[nova-controller]
+merlin-managementnode001
+
+[keystone]
+merlin-managementnode001
+
+[neutron-controller]
+merlin-managementnode001 physical_interface_mappings=provider:eno3
+
+[heat]
+merlin-managementnode001
+
+[glance-controller]
+merlin-managementnode001

nuke.yml

@@ -2,7 +2,7 @@
 # This playbook will reset the instalation to facilitate a new installation.
 # All data is lost!
 - hosts: all
-  become: True
+  become: true
   name: Dummy to gather facts
   tasks:
     - name: Stop docker service
@@ -13,9 +13,19 @@
         state: stopped
     - name: remove volumes
       shell: "rm -rf /srv"
+    - name: remove network namespaces
+      shell: "rm /var/run/netns/*"
+      ignore_errors: true
+    - name: Remove stale vxlan interfaces
+      shell: "for interface in $(ip link | grep DOWN | grep -Po 'vxlan-\\d{1,2}'); do ip link del $interface ; done"
+      ignore_errors: true
 
-- hosts: cinder-storage
+
-  become: True
+- hosts: nova-compute
+  gather_facts: false
+  become: true
   tasks:
-    - name: wipe cinder storage.
+    - name: kill all vm's
-      shell: "dd if=/dev/zero of={{ storage_volume }} bs=1M count=1 && sync"
+      shell: "for machine in  $(virsh list --uuid ); do virsh destroy $machine ; done"
+    - name: wipe all vm's
+      shell: "for machine in  $(virsh list --uuid --all); do virsh undefine $machine ; done"

post-install.yml

@@ -15,12 +15,19 @@
     - name: post install configuration
       command: docker exec -i keystone.service bash -c "source /root/admin-openrc.sh && {{ item }}"
       with_items:
-        - openstack network create  --share --external  --provider-physical-network provider  --provider-network-type flat provider
+        - openstack network create --share --external --provider-physical-network provider --provider-network-type vlan --provider-segment 983 vlan983
         - >
-            openstack subnet create --network provider
+            openstack subnet create --subnet-range 172.23.41.0/24 --gateway 172.23.41.101
-            --allocation-pool start={{ allocation_pool['start'] }},end={{ allocation_pool['end'] }}
+            --network vlan983 --allocation-pool start=172.23.41.75,end=172.23.41.100
-            --dns-nameserver {{ dns_nameserver }} --gateway {{ gateway }} --subnet-range {{ subnet_range }} providersub
+            --dns-nameserver 8.8.8.8 vlan983_subnet
         - openstack flavor create --id 0 --vcpus 1 --ram 64 --disk 1 m1.nano
         - openstack keypair create --public-key /root/id_rsa.pub adminkey
-
+    - name: Install cirros image
-
+      get_url:
+        url: http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img
+        dest: /tmp/cirros-0.4.0-x86_64-disk.img
+        checksum: sha256:a8dd75ecffd4cdd96072d60c2237b448e0c8b2bc94d57f10fdbc8c481d9005b8
+    - shell: >
+          bash -c "source /srv/keystone/root/admin-openrc.sh &&
+          openstack image create --disk-format qcow2 cirros
+          < /tmp/cirros-0.4.0-x86_64-disk.img"

roles/cinder-controller/tasks/main.yml

@@ -4,9 +4,11 @@
   include_vars:
     file: ../../secrets.yml
     name: secrets
+  tags:
+    - facts
 
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-controller:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-controller-merlin:latest
     env_vars: >
         -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
         -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-controller'][0]]['ansible_default_ipv4']['address']) }}"
@@ -20,6 +22,9 @@
         -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
         -e "RABBIT_USER=openstack"
+        -e "RBD_SECRET_UUID={{ secrets['CINDER_RBD_SECRET_UUID']}}"
+  tags:
+    - facts
 
 - name: pull docker image
   docker_image:
@@ -42,11 +47,17 @@
     mode: 644
     owner: root
     group: root
+  tags:
+    - systemd
 
 - name: start service at boot.
   command: systemctl reenable cinder-controller.service
+  tags:
+    - systemd
 
 - command: systemctl daemon-reload
+  tags:
+    - systemd
 
 - name: Initialize database.
   command: >

roles/cinder-storage/files/ceph.client.volumes.keyring

@@ -0,0 +1,9 @@
+$ANSIBLE_VAULT;1.1;AES256
+39313161646365373665646331613930316437363735326262376531636166346138303139613138
+3361353633616136303365646165643339333130393031380a373934636436626336326436306666
+34316532333165346139633239313930326238333134633365666138326338386632373937343335
+3262383863653136300a393464646365623763663063303936646462313764633736613562633661
+62313961626165363761656363393538396461653936353932303137626435626161316239623338
+65656132353136656430613462663466616432643761303366396461653066616162366666356533
+39386261623861323861633739343237386266306264356436666430313531303238636235393665
+31396533306261393835

roles/cinder-storage/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.volumes]
+keyring = /etc/ceph/ceph.client.volumes.keyring

roles/cinder-storage/files/uuid

@@ -0,0 +1 @@
+d0db6ba7-a0c9-4da6-b0bc-aa7846325333

roles/cinder-storage/tasks/main.yml

@@ -6,8 +6,11 @@
     name: secrets
   tags: vars
 
+#- command: uuidgen
+#  register: uuid
+
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage-merlin:latest
     env_vars: >
         -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
         -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address']) }}"
@@ -21,6 +24,12 @@
         -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
         -e "RABBIT_USER=openstack"
+        -e "USE_CEPH={{ use_ceph }}"
+        -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}"
+        -e "MON_HOST={{ ceph_mon_host }}"
+        -e "PUBLIC_NETWORK={{ ceph_public_network }}"
+        -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}"
+        -e "RBD_SECRET_UUID={{ secrets['CINDER_RBD_SECRET_UUID']}}"
   tags: vars
 
 - name: pull docker image
@@ -34,8 +43,9 @@
     state: directory
     mode: 0777
   with_items:
-      - /srv/cinder-storage
+    - /srv/cinder-storage
-      - /srv/cinder-storage/root
+    - /srv/cinder-storage/root
+    - /srv/cinder-storage/etc/ceph
 
 - name: initial setup
   command: >
@@ -43,10 +53,21 @@
              --privileged
              {{ env_vars }}
              -v /srv/cinder-storage/root:/root \
-             -v "{{ storage_volume }}":/dev/cinder_storage_volume \
              {{ docker_image }} /etc/bootstrap.sh
   tags: bootstrap
 
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/cinder-storage/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.volumes.keyring
+    dest: /srv/cinder-storage/etc/ceph/ceph.client.volumes.keyring
+    mode: 0644
+
 - name: install service file.
   template:
     src: templates/cinder-storage.service
@@ -56,6 +77,12 @@
     group: root
   tags: systemd
 
+#- name: set ceph client keyring
+#  copy:
+#    content: "{{ceph_cinder_client_keyring}}"
+#    dest: /srv/cinder-storage/etc/ceph
+#  when: use_ceph
+
 - command: systemctl daemon-reload
   tags: systemd
 

roles/cinder-storage/templates/cinder-storage.service

@@ -11,11 +11,8 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
   --privileged \
   {{ env_vars | replace('\n', '') }} \
-  -v "/dev/cinder-volumes/":/dev/cinder-volumes \
   -v /srv/cinder-storage/root:/root \
-  -v "{{ storage_volume }}":/dev/cinder_storage_volume \
+  -v /etc/ceph:/etc/ceph \
-  -v "/dev/lvm":/dev/lvm \
-  -v "/srv/cinder-storage/volumes/:/var/lib/cinder/volumes/" \
   -p 8777:8776 \
   -p 3260:3260 \
   {{ docker_image }}

roles/common/tasks/main.yml

@@ -10,13 +10,13 @@
 - name: common | install packages
   apt: pkg={{ item }} state=latest update_cache=yes
   with_items:
-     - curl
+    - curl
-     - htop
+    - htop
-     - molly-guard
+    - molly-guard
-     - sudo
+    - sudo
-     - tree
+    - tree
-     - vim
+    - vim
-     - python-simplejson
+    - python-simplejson
 
 - name: sshd_config
   file:
@@ -26,4 +26,7 @@
     owner: root
     group: root
 
+- name: disable apparmor
+  apt: pkg=apparmor state=absent
+
 - include: docker.yml

roles/glance-controller/files/ceph.client.images.keyring

@@ -0,0 +1,2 @@
+[client.images]
+	key = AQDCpDNbJ3DqDBAAvUOUcxEoZNvQUfoaU5i8iQ==

roles/glance-controller/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.images]
+keyring = /etc/ceph/ceph.client.images.keyring

roles/glance-controller/tasks/main.yml

@@ -6,7 +6,7 @@
     name: secrets
 
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance-merlin:latest
     env_vars: >
         -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
         -e "GLANCE_PASSWORD={{ secrets['GLANCE_PASSWORD'] }}"
@@ -19,11 +19,16 @@
         -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
         -e "RABBIT_USER=openstack"
+        -e "USE_CEPH={{ use_ceph }}"
+        -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}"
+        -e "MON_HOST={{ ceph_mon_host }}"
+        -e "PUBLIC_NETWORK={{ ceph_public_network }}"
+        -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}"
 
-- name: pull docker image
+#- name: pull docker image
-  docker_image:
+#  docker_image:
-    name: "{{ docker_image }}"
+#    name: "{{ docker_image }}"
-  tags: pull
+#  tags: pull
 
 - name: Make build and persistent directories
   file:
@@ -33,6 +38,25 @@
   with_items:
       - /srv/glance
       - /srv/glance/root
+      - /srv/glance/etc/ceph
+
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/glance/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.images.keyring
+    dest: /srv/glance/etc/ceph/ceph.client.images.keyring
+    mode: 0644
+
+#- name: set ceph client keyring
+#  copy:
+#    content: "{{ceph_images_client_keyring}}"
+#    dest: /srv/cinder-storage/etc/ceph/ceph.client.images.keyring
+#  when: use_ceph
 
 - name: install service file.
   template:

roles/glance-controller/templates/glance.service

@@ -11,7 +11,7 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
   {{ env_vars | replace('\n', '') }} \
   -v /srv/glance/root:/root \
-  -v /var/lib/glance/images:/var/lib/glance/images \
+  -v /etc/ceph:/etc/ceph \
   -p 9292:9292 \
   {{ docker_image }}
 

roles/keystone/tasks/main.yml

@@ -45,14 +45,14 @@
     MYSQL_ROOT_PASSWORD: "{{ secrets['MYSQL_ROOT_PASSWORD'] }}"
     DB_HOST: "{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}"
   register: result
-  until: result|succeeded
+  until: result is succeeded
   # sometimes the initial connect fails.
   # Retry until it succeeds.
   retries: 7
   delay: 3
   ignore_errors: yes
 
-- name: keystone manage commands to setup db
+- name: keystone manage commands to setup db_sync
   command: >
              /usr/bin/docker run --rm
              --add-host=mariadb:{{ listen_ip | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }}
@@ -90,12 +90,19 @@
              -e "OS_AUTH_URL=http://${KEYSTONE_HOST}:35357/v3"
              -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
              {{ docker_image }} bash /etc/bootstrap.sh
+  register: result
+  retries: 7
+  delay: 3
 
 
+- name: install openstack repo key host.
+  command: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+  tags: openstackclient
+
 - name: install openstack repo on host.
-  command: >
+  apt_repository:
-      echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main" > /etc/apt/sources.list.d/ocata.list &&
+      repo: "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main"
-      apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+      filename: ocata
   tags: openstackclient
 
 - name: install openstack client for management

roles/keystone/templates/admin-openrc.sh

@@ -1,3 +1,5 @@
+export OS_PROJECT_DOMAIN_NAME=Default
+export OS_USER_DOMAIN_NAME=Default
 export OS_TENANT_NAME=admin
 export OS_USERNAME=admin
 export OS_PASSWORD={{ hostvars[groups['keystone'][0]]['OS_PASSWORD'] }}

roles/mariadb/tasks/main.yml

@@ -70,3 +70,8 @@
 
 - name: start service at boot.
   command: systemctl reenable mysql.service
+
+- name: Give the cluster some time to initialize replication.
+  command: bash -c "sleep 60 && systemctl daemon-reload"
+  when: groups['databases'] | length >= 3
+

roles/neutron-controller/tasks/main.yml

@@ -6,7 +6,7 @@
     name: secrets
 
 - set_fact:
-    docker_image: "registry.webhosting.rug.nl/hpc/openstack-neutron-controller:latest"
+    docker_image: "registry.webhosting.rug.nl/hpc/openstack-neutron-controller-merlin:latest"
 
 - name: pull docker image
   docker_image:
@@ -29,6 +29,7 @@
         -e "NOVA_CONTROLLER_HOST={{ listen_ip | default(hostvars[groups['nova-controller'][0]]['ansible_default_ipv4']['address']) }}"
         -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}"
         -e "NOVA_PLACEMENT_USER=placement"
+        -e "OVERLAY_IP={{ overlay_ip }}"
         -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}"
         -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"

roles/neutron-controller/templates/neutron-controller.service

@@ -17,6 +17,7 @@ ExecStart=/usr/bin/docker run --name %n \
   --privileged \
   --network host \
   -v /lib/modules:/lib/modules \
+  -v /var/run/netns:/var/run/netns \
   {{ docker_image }} /etc/run.sh
 
 [Install]

roles/nova-compute/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.compute]
+keyring = /etc/ceph/ceph.client.compute.keyring

roles/nova-compute/files/uuid

@@ -0,0 +1 @@
+b5044271-1918-4070-822c-f19ed14d7494

roles/nova-compute/tasks/main.yml

@@ -7,7 +7,7 @@
   tags: vars
 
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-compute:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-nova-compute-merlin:latest
   tags: vars
 
 - name: pull docker image
@@ -16,6 +16,27 @@
     force: True
   tags: pull
 
+- name: Make build and persistent directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0777
+  with_items:
+    - /srv/nova-compute
+    - /srv/nova-compute/etc/ceph
+
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/nova-compute/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.compute.keyring
+    dest: /srv/nova-compute/etc/ceph/ceph.client.compute.keyring
+    mode: 0644
+
 - name: install service file.
   template:
     src: templates/nova-compute.service
@@ -25,16 +46,22 @@
     group: root
   tags: systemd
 
+#- name: set ceph client keyring
+#  copy:
+#    content: "{{ceph_compute_client_keyring}}"
+#    dest: /srv/nova-compute/etc/ceph
+#  when: use_ceph
+
 - command: systemctl daemon-reload
   tags: systemd
 
 - apt:
     name: "{{ item }}"
   with_items:
-      - kvm
+    - kvm
-      - libvirt0
+    - libvirt0
-      - libvirt-bin
+    - libvirt-bin
-      - qemu
+    - qemu
 
 - name: make sure service is started
   systemd:
@@ -47,3 +74,8 @@
 - name: let nova controler discover new host
   shell: "sleep 10 && docker exec -i nova-controller.service nova-manage cell_v2 discover_hosts"
   delegate_to: "{{ hostvars[groups['nova-controller'][0]]['ansible_hostname'] }}"
+  register: result
+  until: result is succeeded
+  retries: 7
+  delay: 3
+  ignore_errors: yes

roles/nova-compute/templates/nova-compute.service

@@ -26,19 +26,28 @@ ExecStart=/usr/bin/docker run --name %n \
     -e "NOVA_PLACEMENT_PASSWORD={{ secrets['NOVA_PLACEMENT_PASSWORD'] }}" \
     -e "NOVA_PLACEMENT_USER=placement" \
     -e "NOVA_USER=nova" \
+    -e "OVERLAY_IP={{ overlay_ip }}" \
     -e "OS_PASSWORD={{ secrets['OS_PASSWORD'] }}" \
     -e "PHYSICAL_INTERFACE_MAPPINGS={{ physical_interface_mappings }}" \
     -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['listen_ip'] | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}" \
     -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}" \
     -e "RABBIT_USER=openstack" \
+    -e "USE_CEPH={{ use_ceph }}" \
+    -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}" \
+    -e "MON_HOST={{ ceph_mon_host }}" \
+    -e "PUBLIC_NETWORK={{ ceph_public_network }}" \
+    -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}" \
+    -e "RBD_SECRET_UUID={{ secrets['NOVA_RBD_SECRET_UUID'] }}" \
     --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
     --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
     --privileged \
     -v /dev:/dev \
     -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
     -v /var/lib/nova/instances:/var/lib/nova/instances \
+    -v /var/run/netns:/var/run/netns \
     -v /lib/modules:/lib/modules \
     -v /etc/machine-id:/etc/machine-id \
+    -v /etc/ceph:/etc/ceph \
     --network host \
     {{ docker_image }} /etc/run.sh
 

secrets.yml

@@ -1,32 +1,42 @@
 $ANSIBLE_VAULT;1.1;AES256
-32313865346461323861313234623330633830663561353636396161643566353733393931303232
+65633261656530663035316431306465633266376462653564613237663833333630663333643764
-3134353364393034626564333866383065633162383561390a656463303464616434303435303037
+6434623237626630356632313933323637316535636235330a323266636338326361343938343931
-63313232373630616136626464373464646535353030396136656361343162666165366566383737
+63356362343538393030663864663363373633303231643233616563616537376239663337306464
-6133326539393432390a386162393639636137326532393939633665326637373461663766656565
+3164666366623639630a646633636134316561376137646632336139323265636366343938613062
-36636661653939373134313165383261353832373362613533366431626330313831643836363932
+32663934633366623664636364396130333463366535333336303962633663666432623365356537
-38623937663335646134323130636539333335636265313564323032623065393031343235633832
+65616339633433623761626537666131646365373334316237663839613264393564353230666134
-33336138653336633932383133366566656464356331306265663662356166613135663234326434
+63386439323966343065666138636436643433363931373766363632653661363031303138646632
-64343765366439303766613062363830346238376435366138323662366463353634623937376663
+61646437316265376539333661356239386533663533643864376263653237313533616263666563
-39313263613764623533323562333932656564346462333961663538353366313065343263383431
+65306465313362396235393366363532353932383633623832393161323265373065326432656338
-39643734353632336134663965343062373933656461393264323938303261316563313962343837
+34613761373230396332393239323733383937363339373438326434393030646231376531663963
-66623562333031646633643734383439386130653033343962633930613031313433633033383561
+32623664303935623334326532383334343466613133623532363062396363626262396135626663
-65366230663263346661343339636537336332323266623230646534323563373934356332663136
+35636636623833623165386137383664633561646630613930333061333466343831376332366266
-30626231623534616562323033333437353239306538653835623931363164383536336562336136
+65353030383461623665653362613863646331633036616637643838666231653438636332376132
-30666265366536313436646535383632626137613831633132666666653830383566323532306332
+30356433623662616430353265386632306564326633616538306632386465343636633538623263
-34353534336331653330663137323936303337306134333036633932363664643864333534316438
+30366139366638613564333532333733383364323063376638613063346665663965356439636636
-31623062303137376637353061643838383831663561663436663130663064323665316261316531
+32613035653134663733633731356530303338353030333532323762653864616230643931363032
-32616533333165636130623334373130316339326538343330646366383933353137623631323530
+35653962373030663164383666316636616639666431656638653064303433613431636263333636
-39653437343432383161323661643931346664663265326664336461373033646563666333353661
+65666138626563653538626164646265373766643131646162343366353835643031663866666137
-61633865373764346131623131346266373331626336663735303439376230333537386562303939
+39363232616632323035643432626639323233333930646230613732386163383133383964623133
-65363139346564653464663633326639323930333464376136353064393039373535613231623138
+33623663663130323737646133353139353833653138636338636336656562313639626162646531
-61373434323065646238356436373730333939613965666237616265653033333230353466316266
+32353331333163373366616666356539306238653865616435633734393966333765313134616338
-30383939376335656632626232663061623332636337646234653565363561353334643462376666
+34623337623739333439656638303363323534333165303861363334646137373037653665323961
-30326438303333336265306463313137656334313235643434616238333564373761333235633639
+61623632373330323835653232353961663931326535356162656164616132623437636330653161
-66346161316130633463623435646639366136386335386139613230653064663230366265633036
+65623861396665386331653734373334663532393731656430333933326264323133396463653239
-33613132633035393337653436613031383765616638323663363866393165613030306637393134
+33383662303031356564666531613731663166613061383039393431643530656665306339326436
-38333734373939626364343533306662393463646264666161346434363832623239643864303431
+65303063363163643362643163366365346230643936643231616530373763333536363838656130
-39383931333139633338663761646335613935636239636439383333313531633364653439323036
+39326235373835326635306366653864316534663061323062376666666466363434363661623636
-35373639363164386666366335313934336231333261623763633133393562656237313761356631
+31626332643839346138326336353665363838346535373335656466336665613265633461663134
-39663234653339313466326534333435306662316461333035623339353435383137383735373733
+31323838336465366236353932646330333562363063616437633365353433303962346231663939
-32373535303338646266346539386364356233616631316661633037346665353762353138376538
+31343133343336343431643564393839373139623365386330623665383264646163396438626539
-386535333439313233663464353534376535
+38343464343736363936636139653965303731353330653963383465633037633237383064396162
+33363864336235346663616230636633353361613138333236393866316165666162656565383739
+38653233346135373661613739393735343535623230653739316433376165663932366233643431
+39383261623065353932386632646134383136393664306465326637366639666433386162393237
+63663063656461653233643665306366653965393737376532356132623333383337333266316339
+33323934623734353639643330383066313632623166306337323932323933393536366361616564
+33303830333430663233336662353631663633303136346366376163353235303363326165306131
+62393166633232343065663062646435363563313961396132303737343263363363613137636236
+31316464613164353233366364306136663735343361333335353564666131396332643461303966
+65316339616166343232613632363030386432656339623363356661323163353563326238633863
+6431

set_ceph_secrets.yml

@@ -0,0 +1,17 @@
+---
+
+- hosts: nova-compute
+  become: true
+  tasks:
+    - copy:
+        src: ceph.xml
+        dest: /root/ceph.xml
+        mode: 0644
+    - name: include secrets
+      include_vars:
+        file: secrets.yml
+        name: secrets
+    - command: virsh secret-define --file /root/ceph.xml
+    - command: >
+          virsh secret-set-value --secret d0db6ba7-a0c9-4da6-b0bc-aa7846325333
+          --base64 {{ secrets['ceph_client_volumes_key'] }}

settings.yml

@@ -10,3 +10,5 @@
 - subnet_range: 172.23.128.0/24
 
 - rsa_pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDStPUPXkcu81onUm/le54JCu174yXJJDsthDr96Mv8irBVBWuy5FxnaASuDpmC4QE4s0UAIg1iq/SWrr8qdBQ4OVuYFiW0S7ZJvcoKr/40Wh+T5MeltGQfmkDp6kBsfaMSo6M4tF1c8i+XgOgxb4fxHYb8mFhseztRLx6McxJJJLB0nu+T12WQ01nl0XtwD+3EsZWfxRH0KA59VHZSe3Anc5z+Fm7WU+1Vzy6/pkiIhVReI1L6VVhZsIdSu3fQK6fHQcujtfuw6RKEpisZQqnxMUviWQ98yeQXHk6Nx840WCh3vvKveEAoC4Y/UEZa1TMe6PczfUaLjaidUkpulJsP egon@egon-pc
+
+- use_ceph: True

ubuntucloudrepo.yml

@@ -0,0 +1,18 @@
+---
+- hosts: all
+  name: Dummy to gather facts
+  become: true
+  tasks:
+
+      - name: install openstack repo key host.
+        command: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+        tags: openstackclient
+
+      - name: install openstack repo on host.
+        apt_repository:
+            repo: "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main"
+            filename: ocata
+        tags: openstackclient
+
+      - apt:
+          update_cache: yes