some ceph changes

might dissappear again and moved to ceph-ansible repo.

ansible.cfg

@@ -1,2 +1,5 @@
 [defaults]
 inventory = merlin
+stdout_callback = debug
+vault_password_file = .vault_pass.txt
+forks = 20

groupvars/all.yml

@@ -0,0 +1,9 @@
+---
+use_ceph: true
+ceph_mon_initial_members:
+ceph_mon_host:
+ceph_public_network:
+ceph_osd_pool_default_size:
+ceph_compute_client_keyring:
+ceph_cinder_client_keyring:
+ceph_images_client_keyring:

hosts-openstack03

@@ -0,0 +1,26 @@
+[databases]
+openstack03.gcc.rug.nl
+
+[keystone]
+openstack03.gcc.rug.nl
+
+[glance-controller]
+openstack03.gcc.rug.nl
+
+[horizon]
+openstack03.gcc.rug.nl
+
+[rabbitmq]
+openstack03.gcc.rug.nl
+
+[memcached]
+openstack03.gcc.rug.nl
+
+[neutron-controller]
+openstack03.gcc.rug.nl
+
+[nova-controller]
+openstack03.gcc.rug.nl
+
+[nova-compute]
+openstack03.gcc.rug.nl

merlin

@@ -1,39 +1,54 @@
 [nova-compute]
-merlin-node012 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.12
-merlin-node013 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.13
-merlin-node006 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.6
-merlin-node007 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.7
-merlin-node008 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.8
-merlin-node009 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.9
-merlin-node010 physical_interface_mappings=provider:enp130s0f0 overlay_ip=172.23.43.10
+merlin-managementnode002 physical_interface_mappings=provider:enp5s0f1.1064 overlay_ip=172.23.43.102
+merlin-managementnode003 physical_interface_mappings=provider:enp5s0f1.1064 overlay_ip=172.23.43.103
+merlin-node001 physical_interface_mappings=provider:enp130s0f0.1064 overlay_ip=172.23.43.1
+merlin-node002 physical_interface_mappings=provider:enp130s0f0.1064 overlay_ip=172.23.43.2
+merlin-node003 physical_interface_mappings=provider:enp130s0f0.1064 overlay_ip=172.23.43.3
+merlin-node004 physical_interface_mappings=provider:enp130s0f0.1064 overlay_ip=172.23.43.4
+merlin-node005 physical_interface_mappings=provider:enp130s0f0.1064 overlay_ip=172.23.43.5
 
 [databases]
-merlin-node011
-merlin-node012
-merlin-node013
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
 
 [rabbitmq]
-merlin-node011
-merlin-node012
-merlin-node013
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
 
 [horizon]
-merlin-node011
+merlin-managementnode001
 
 [memcached]
-merlin-node011
+merlin-managementnode001
 
 [nova-controller]
-merlin-node011
+merlin-managementnode001
 
 [keystone]
-merlin-node011
+merlin-managementnode001
 
 [neutron-controller]
-merlin-node011 physical_interface_mappings=provider:enp129s0f1 overlay_ip=172.23.43.11
+merlin-managementnode001 physical_interface_mappings=provider:enp5s0f1.1064 overlay_ip=172.23.43.101
 
 [heat]
-merlin-node011
+merlin-managementnode001
 
 [glance-controller]
-merlin-node011
+merlin-managementnode002
+
+[cinder-controller]
+merlin-managementnode003
+
+[cinder-storage]
+merlin-managementnode002
+merlin-managementnode003
+merlin-node001
+merlin-node002
+merlin-node003
+merlin-node004
+merlin-node005
+
+[stor]
+merlin-stor00[1:8]

merlinsdn

@@ -0,0 +1,37 @@
+[nova-compute]
+merlin-managementnode002 physical_interface_mappings=provider:eno3
+merlin-managementnode003 physical_interface_mappings=provider:eno3
+merlin-node001 physical_interface_mappings=provider:eno3
+merlin-node003 physical_interface_mappings=provider:eno3
+merlin-node004 physical_interface_mappings=provider:eno3
+
+[databases]
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
+
+[rabbitmq]
+merlin-managementnode001
+merlin-managementnode002
+merlin-managementnode003
+
+[horizon]
+merlin-managementnode001
+
+[memcached]
+merlin-managementnode001
+
+[nova-controller]
+merlin-managementnode001
+
+[keystone]
+merlin-managementnode001
+
+[neutron-controller]
+merlin-managementnode001 physical_interface_mappings=provider:eno3
+
+[heat]
+merlin-managementnode001
+
+[glance-controller]
+merlin-managementnode001

nuke.yml

@@ -14,8 +14,3 @@
     - name: remove volumes
       shell: "rm -rf /srv"
 
-- hosts: cinder-storage
-  become: True
-  tasks:
-    - name: wipe cinder storage.
-      shell: "dd if=/dev/zero of={{ storage_volume }} bs=1M count=1 && sync"

roles/cinder-storage/files/ceph.client.volumes.keyring

@@ -0,0 +1,9 @@
+$ANSIBLE_VAULT;1.1;AES256
+39313161646365373665646331613930316437363735326262376531636166346138303139613138
+3361353633616136303365646165643339333130393031380a373934636436626336326436306666
+34316532333165346139633239313930326238333134633365666138326338386632373937343335
+3262383863653136300a393464646365623763663063303936646462313764633736613562633661
+62313961626165363761656363393538396461653936353932303137626435626161316239623338
+65656132353136656430613462663466616432643761303366396461653066616162366666356533
+39386261623861323861633739343237386266306264356436666430313531303238636235393665
+31396533306261393835

roles/cinder-storage/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.volumes]
+keyring = /etc/ceph/ceph.client.volumes.keyring

roles/cinder-storage/files/uuid

@@ -0,0 +1 @@
+d0db6ba7-a0c9-4da6-b0bc-aa7846325333

roles/cinder-storage/tasks/main.yml

@@ -6,8 +6,11 @@
     name: secrets
   tags: vars
 
+#- command: uuidgen
+#  register: uuid
+
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-cinder-storage-merlin:latest
     env_vars: >
         -e "MY_IP={{ listen_ip | default(ansible_default_ipv4.address) }}"
         -e "CINDER_HOST={{ listen_ip | default(hostvars[groups['cinder-storage'][0]]['ansible_default_ipv4']['address']) }}"
@@ -21,6 +24,11 @@
         -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
         -e "RABBIT_USER=openstack"
+        -e "USE_CEPH={{ use_ceph }}"
+        -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}"
+        -e "MON_HOST={{ ceph_mon_host }}"
+        -e "PUBLIC_NETWORK={{ ceph_public_network }}"
+        -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}"
   tags: vars
 
 - name: pull docker image
@@ -34,8 +42,9 @@
     state: directory
     mode: 0777
   with_items:
-      - /srv/cinder-storage
-      - /srv/cinder-storage/root
+    - /srv/cinder-storage
+    - /srv/cinder-storage/root
+    - /srv/cinder-storage/etc/ceph
 
 - name: initial setup
   command: >
@@ -43,10 +52,21 @@
              --privileged
              {{ env_vars }}
              -v /srv/cinder-storage/root:/root \
-             -v "{{ storage_volume }}":/dev/cinder_storage_volume \
              {{ docker_image }} /etc/bootstrap.sh
   tags: bootstrap
 
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/cinder-storage/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.volumes.keyring
+    dest: /srv/cinder-storage/etc/ceph/ceph.client.volumes.keyring
+    mode: 0644
+
 - name: install service file.
   template:
     src: templates/cinder-storage.service
@@ -56,6 +76,12 @@
     group: root
   tags: systemd
 
+- name: set ceph client keyring
+  copy:
+    content: "{{ceph_cinder_client_keyring}}"
+    dest: /srv/cinder-storage/etc/ceph
+  when: use_ceph
+
 - command: systemctl daemon-reload
   tags: systemd
 

roles/cinder-storage/templates/cinder-storage.service

@@ -11,11 +11,8 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
   --privileged \
   {{ env_vars | replace('\n', '') }} \
-  -v "/dev/cinder-volumes/":/dev/cinder-volumes \
   -v /srv/cinder-storage/root:/root \
-  -v "{{ storage_volume }}":/dev/cinder_storage_volume \
-  -v "/dev/lvm":/dev/lvm \
-  -v "/srv/cinder-storage/volumes/:/var/lib/cinder/volumes/" \
+  -v /etc/ceph:/etc/ceph \
   -p 8777:8776 \
   -p 3260:3260 \
   {{ docker_image }}

roles/common/tasks/main.yml

@@ -10,13 +10,13 @@
 - name: common | install packages
   apt: pkg={{ item }} state=latest update_cache=yes
   with_items:
-     - curl
-     - htop
-     - molly-guard
-     - sudo
-     - tree
-     - vim
-     - python-simplejson
+    - curl
+    - htop
+    - molly-guard
+    - sudo
+    - tree
+    - vim
+    - python-simplejson
 
 - name: sshd_config
   file:
@@ -26,4 +26,7 @@
     owner: root
     group: root
 
+- name: disable apparmor
+  apt: pkg=apparmor state=absent
+
 - include: docker.yml

roles/glance-controller/files/ceph.client.images.keyring

@@ -0,0 +1,2 @@
+[client.images]
+	key = AQDCpDNbJ3DqDBAAvUOUcxEoZNvQUfoaU5i8iQ==

roles/glance-controller/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.images]
+keyring = /etc/ceph/ceph.client.images.keyring

roles/glance-controller/tasks/main.yml

@@ -6,7 +6,7 @@
     name: secrets
 
 - set_fact:
-    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance:latest
+    docker_image: registry.webhosting.rug.nl/hpc/openstack-glance-merlin:latest
     env_vars: >
         -e "GLANCE_HOST={{ listen_ip | default(hostvars[groups['glance-controller'][0]]['ansible_default_ipv4']['address']) }}"
         -e "GLANCE_PASSWORD={{ secrets['GLANCE_PASSWORD'] }}"
@@ -19,11 +19,16 @@
         -e "RABBIT_HOST={{ listen_ip | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}"
         -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}"
         -e "RABBIT_USER=openstack"
+        -e "USE_CEPH={{ use_ceph }}"
+        -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}"
+        -e "MON_HOST={{ ceph_mon_host }}"
+        -e "PUBLIC_NETWORK={{ ceph_public_network }}"
+        -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}"
 
-- name: pull docker image
-  docker_image:
-    name: "{{ docker_image }}"
-  tags: pull
+#- name: pull docker image
+#  docker_image:
+#    name: "{{ docker_image }}"
+#  tags: pull
 
 - name: Make build and persistent directories
   file:
@@ -33,6 +38,25 @@
   with_items:
       - /srv/glance
       - /srv/glance/root
+      - /srv/glance/etc/ceph
+
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/glance/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.images.keyring
+    dest: /srv/glance/etc/ceph/ceph.client.images.keyring
+    mode: 0644
+
+- name: set ceph client keyring
+  copy:
+    content: "{{ceph_images_client_keyring}}"
+    dest: /srv/cinder-storage/etc/ceph/ceph.client.images.keyring
+  when: use_ceph
 
 - name: install service file.
   template:

roles/glance-controller/templates/glance.service

@@ -11,7 +11,7 @@ ExecStartPre=-/usr/bin/docker rm %n
 ExecStart=/usr/bin/docker run --name %n \
   {{ env_vars | replace('\n', '') }} \
   -v /srv/glance/root:/root \
-  -v /var/lib/glance/images:/var/lib/glance/images \
+  -v /etc/ceph:/etc/ceph \
   -p 9292:9292 \
   {{ docker_image }}
 

roles/nova-compute/files/ceph.conf

@@ -0,0 +1,14 @@
+[global]
+fsid = ef0b40a2-bc8c-4432-9cde-0ca7c82c8717
+mon_initial_members = merlin-managementnode002
+mon_host = 172.23.59.102
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Your network address
+public network = 172.23.59.0/24
+osd pool default size = 2
+
+[client.compute]
+keyring = /etc/ceph/ceph.client.compute.keyring

roles/nova-compute/files/uuid

@@ -0,0 +1 @@
+b5044271-1918-4070-822c-f19ed14d7494

roles/nova-compute/tasks/main.yml

@@ -16,6 +16,27 @@
     force: True
   tags: pull
 
+- name: Make build and persistent directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0777
+  with_items:
+    - /srv/nova-compute
+    - /srv/nova-compute/etc/ceph
+
+- name: copy ceph-client configurationfile
+  copy:
+    src: files/ceph.conf
+    dest: /srv/nova-compute/etc/ceph/ceph.conf
+    mode: 0644
+
+- name: copy ceph-client-keyring
+  copy:
+    src: files/ceph.client.compute.keyring
+    dest: /srv/nova-compute/etc/ceph/ceph.client.compute.keyring
+    mode: 0644
+
 - name: install service file.
   template:
     src: templates/nova-compute.service
@@ -25,16 +46,22 @@
     group: root
   tags: systemd
 
+- name: set ceph client keyring
+  copy:
+    content: "{{ceph_compute_client_keyring}}"
+    dest: /srv/nova-compute/etc/ceph
+  when: use_ceph
+
 - command: systemctl daemon-reload
   tags: systemd
 
 - apt:
     name: "{{ item }}"
   with_items:
-      - kvm
-      - libvirt0
-      - libvirt-bin
-      - qemu
+    - kvm
+    - libvirt0
+    - libvirt-bin
+    - qemu
 
 - name: make sure service is started
   systemd:

roles/nova-compute/templates/nova-compute.service

@@ -32,6 +32,11 @@ ExecStart=/usr/bin/docker run --name %n \
     -e "RABBIT_HOST={{ hostvars[groups['rabbitmq'][0]]['listen_ip'] | default(hostvars[groups['rabbitmq'][0]]['ansible_default_ipv4']['address']) }}" \
     -e "RABBIT_PASSWORD={{ secrets['RABBIT_PASSWORD'] }}" \
     -e "RABBIT_USER=openstack" \
+    -e "USE_CEPH={{ use_ceph }}" \
+    -e "MON_INITIAL_MEMBERS={{ ceph_mon_initial_members }}" \
+    -e "MON_HOST={{ ceph_mon_host }}" \
+    -e "PUBLIC_NETWORK={{ ceph_public_network }}" \
+    -e "OSD_POOL_DEFAULT_SIZE={{ ceph_osd_pool_default_size }}" \
     --add-host=keystone:{{ hostvars[groups['keystone'][0]]['listen_ip'] | default(hostvars[groups['keystone'][0]]['ansible_default_ipv4']['address']) }} \
     --add-host=mariadb:{{ hostvars[groups['databases'][0]]['listen_ip'] | default(hostvars[groups['databases'][0]]['ansible_default_ipv4']['address']) }} \
     --privileged \
@@ -40,6 +45,7 @@ ExecStart=/usr/bin/docker run --name %n \
     -v /var/lib/nova/instances:/var/lib/nova/instances \
     -v /lib/modules:/lib/modules \
     -v /etc/machine-id:/etc/machine-id \
+    -v /etc/ceph:/etc/ceph \
     --network host \
     {{ docker_image }} /etc/run.sh
 

secrets.yml

@@ -1,32 +1,34 @@
 $ANSIBLE_VAULT;1.1;AES256
-32313865346461323861313234623330633830663561353636396161643566353733393931303232
-3134353364393034626564333866383065633162383561390a656463303464616434303435303037
-63313232373630616136626464373464646535353030396136656361343162666165366566383737
-6133326539393432390a386162393639636137326532393939633665326637373461663766656565
-36636661653939373134313165383261353832373362613533366431626330313831643836363932
-38623937663335646134323130636539333335636265313564323032623065393031343235633832
-33336138653336633932383133366566656464356331306265663662356166613135663234326434
-64343765366439303766613062363830346238376435366138323662366463353634623937376663
-39313263613764623533323562333932656564346462333961663538353366313065343263383431
-39643734353632336134663965343062373933656461393264323938303261316563313962343837
-66623562333031646633643734383439386130653033343962633930613031313433633033383561
-65366230663263346661343339636537336332323266623230646534323563373934356332663136
-30626231623534616562323033333437353239306538653835623931363164383536336562336136
-30666265366536313436646535383632626137613831633132666666653830383566323532306332
-34353534336331653330663137323936303337306134333036633932363664643864333534316438
-31623062303137376637353061643838383831663561663436663130663064323665316261316531
-32616533333165636130623334373130316339326538343330646366383933353137623631323530
-39653437343432383161323661643931346664663265326664336461373033646563666333353661
-61633865373764346131623131346266373331626336663735303439376230333537386562303939
-65363139346564653464663633326639323930333464376136353064393039373535613231623138
-61373434323065646238356436373730333939613965666237616265653033333230353466316266
-30383939376335656632626232663061623332636337646234653565363561353334643462376666
-30326438303333336265306463313137656334313235643434616238333564373761333235633639
-66346161316130633463623435646639366136386335386139613230653064663230366265633036
-33613132633035393337653436613031383765616638323663363866393165613030306637393134
-38333734373939626364343533306662393463646264666161346434363832623239643864303431
-39383931333139633338663761646335613935636239636439383333313531633364653439323036
-35373639363164386666366335313934336231333261623763633133393562656237313761356631
-39663234653339313466326534333435306662316461333035623339353435383137383735373733
-32373535303338646266346539386364356233616631316661633037346665353762353138376538
-386535333439313233663464353534376535
+62326337613938653661303066636362626639393334643233386166383735653137643761653939
+6337383561386130346461373262626235353061393737650a336265653035656135643530393430
+65623533323531393731323765333530386666636531363263393332356137333964653761616265
+3566623031616538630a353265333633643030663735626338646563663938363664313930313037
+62356665316534353433656362303836613365643263343462646366623062656335333366373836
+65363363633636633262663063346231333865316237646164373637396130383563666665333339
+61313632663862626532333364643962383365633361326430353734393134373130666565383361
+31303864343132306264356462613234396166613436646130303761396361613937653334373337
+62393538366665386131646162316134393532396530653264323230633166656632346365323634
+33383337613866333031376461396633336233363565316362386566643262333933643237636234
+35616436383238373661313038343063383332303765316232376635346431336161373065656137
+61313962383634396136616663383566663836613939373932323263616461646531633332386436
+34633761666634363062313563646162346330316363323539343439346434306465626130376632
+34336365656231623836643736653037636635613432333435623461613361653031643433326336
+64646664366437393436623832623138306334353634313038656364353866643039663135363932
+34663361373833626461363165646566666332336636653939616438316265393861326636653134
+65373035396635613039333730363863383039323432623432656639346230366461663565613639
+66653335326337306634626335636439343966376336633834356437323866616132316632333230
+63663532396630666434353932363738666134656262363933633635626430373966646234366435
+39353465333133653932303166346339343465383835353861306363386632353432353262333365
+32316565663161326238613834613761316336326662643532353966616630656263666463386264
+63353037373234663237366433623737356364363932353030646531343033643930343465383465
+64343736386433326230383339666364666664323833626239333865373437393365623331303436
+66313063616336303733306462346133623135386535313665623566326138333132353031333930
+65323436653833393639633462343664346165346538613435323231303263303038326135613433
+33666434303963346436643937653934666336396130356639326439306565613432323734383930
+37616536303038343865343637316430343339616162373232653735373335343765333033663030
+32623434376337623665386639396262326236353130313735336264383631333233383034623133
+66376138336465313431346166346638306631333161373165396437653438323931626230633436
+63313831636132386664353339653331333530633731383738616136316636616136383531303064
+39633232396131336535343263313338376334663337626539316263353038623565383062383639
+64393130353462633865653734323637346436363336393734373263313332633562643732326663
+3437

settings.yml

@@ -10,3 +10,5 @@
 - subnet_range: 172.23.128.0/24
 
 - rsa_pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDStPUPXkcu81onUm/le54JCu174yXJJDsthDr96Mv8irBVBWuy5FxnaASuDpmC4QE4s0UAIg1iq/SWrr8qdBQ4OVuYFiW0S7ZJvcoKr/40Wh+T5MeltGQfmkDp6kBsfaMSo6M4tF1c8i+XgOgxb4fxHYb8mFhseztRLx6McxJJJLB0nu+T12WQ01nl0XtwD+3EsZWfxRH0KA59VHZSe3Anc5z+Fm7WU+1Vzy6/pkiIhVReI1L6VVhZsIdSu3fQK6fHQcujtfuw6RKEpisZQqnxMUviWQ98yeQXHk6Nx840WCh3vvKveEAoC4Y/UEZa1TMe6PczfUaLjaidUkpulJsP egon@egon-pc
+
+- use_ceph: True

ubuntucloudrepo.yml

@@ -0,0 +1,18 @@
+---
+- hosts: all
+  name: Dummy to gather facts
+  become: true
+  tasks:
+
+      - name: install openstack repo key host.
+        command: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5EDB1B62EC4926EA
+        tags: openstackclient
+
+      - name: install openstack repo on host.
+        apt_repository:
+            repo: "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/ocata main"
+            filename: ocata
+        tags: openstackclient
+
+      - apt:
+          update_cache: yes