chore: add forcePathStyle s3 property

We can host s3 compatible storage locally by deploying the stable/minio chart.
Ran into https://github.com/coreos/etcd-operator/issues/1980 and therefore downgrade the backup and restore operator images to 0.8.3.

molgenis-jenkins/resources/README.md

@@ -1,6 +0,0 @@
-To be able to run helm inside a jenkins pod, you'll need to 
-* create a role in the namespace where tiller is installed
-* bind that role to the user that jenkins pods run as
-
-This directory contains yaml for these resources.
-See also https://github.com/helm/helm/blob/master/docs/rbac.md

molgenis-jenkins/resources/jenkins-default-tiller-user-rolebinding.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: RoleBinding
-metadata:
-  name: tiller-jenkins-binding
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: tiller-user
-subjects:
-- kind: ServiceAccount
-  name: default
-  namespace: molgenis-jenkins

molgenis-jenkins/resources/tiller-user-role.yaml

@@ -1,18 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: Role
-metadata:
-  name: tiller-user
-  namespace: kube-system
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods/portforward
-  verbs:
-  - create
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list

molgenis-jenkins/values.yaml

@@ -416,12 +416,6 @@ jenkins:
               key: VAULT_ADDR
               secretName: molgenis-pipeline-vault-secret
               secretKey: addr
-        helm:
-          Image: "lachlanevenson/k8s-helm"
-          ImageTag: "v2.10.0"
-          Command: cat
-          WorkingDir: /home/jenkins
-          TTY: true
       NodeSelector: {}
     node:
       Label: node-carbon
@@ -453,69 +447,6 @@ jenkins:
               secretName: molgenis-pipeline-vault-secret
               secretKey: addr
       NodeSelector: {}
-    molgenis-it:
-      InheritFrom: molgenis
-      Label: molgenis-it
-      NodeUsageMode: EXCLUSIVE
-      Containers:
-        elasticsearch:
-          Image: docker.elastic.co/elasticsearch/elasticsearch
-          ImageTag: 5.5.3
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "1Gi"
-            limits:
-              cpu: "1"
-              memory: "1500Mi"
-          EnvVars:
-          - type: KeyValue
-            key: ES_JAVA_OPTS
-            value: "-Xms512m -Xmx512m"
-          - type: KeyValue
-            key: cluster.name
-            value: molgenis
-          - type: KeyValue
-            key: bootstrap.memory_lock
-            value: "true"
-          - type: KeyValue
-            key: xpack.security.enabled
-            value: "false"
-          - type: KeyValue
-            key: discovery.type
-            value: single-node
-        postgres:
-          Image: postgres
-          ImageTag: 9.6-alpine
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "250Mi"
-            limits:
-              cpu: "1"
-              memory: "250Mi"
-          EnvVars:
-          - type: KeyValue
-            key: POSTGRES_USER
-            value: molgenis
-          - type: KeyValue
-            key: POSTGRES_PASSWORD
-            value: molgenis
-          - type: KeyValue
-            key: POSTGRES_DB
-            value: molgenis
-        opencpu:
-          Image: molgenis/opencpu
-          AlwaysPullImage: true
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "256Mi"
-            limits:
-              cpu: "1"
-              memory: "512Mi"
-      NodeSelector: {}
-
 #secret contains configuration for the kubernetes secrets that jenkins can access
 secret:
   # vault configures the vault secret

molgenis-vault/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v1
 appVersion: "1.0"
 description: MOLGENIS vault
 name: molgenis-vault
-version: 0.1.1
+version: 0.2.1
 icon: https://git.webhosting.rug.nl/molgenis/molgenis-ops-docker-helm/raw/master/molgenis-vault/catalogIcon-molgenis-vault.svg

molgenis-vault/README.md

@@ -13,21 +13,25 @@ See https://github.com/coreos/vault-operator/blob/master/doc/user/vault.md
 ## Parameters
 
 ### Azure cloud credentials
-Define credentials for backup to the Azure Blob Store.
+Define credentials for an S3 compatible backup bucket.
-See [etcd-operator documentation](https://github.com/coreos/etcd-operator/blob/master/doc/user/abs_backup.md).
+See [etcd-operator documentation](https://github.com/coreos/etcd-operator/blob/master/doc/user/walkthrough/backup-operator.md).
+> Default values backup to the minio play server.
+You can host the stable/minio chart to backup to a bucket on the cluster.
 
-| Parameter       | Description                   | Default            |
+| Parameter            | Description                              | Default                                    |
-| --------------- | ----------------------------- | ------------------ |
+| -------------------- | ---------------------------------------- | ------------------------------------------ |
-| `abs.account`   | name of storage account       | `fdlkops`          |
+| `s3.accessKeyId`     | key id storage account                   | `Q3AM3UQ867SPQQA43P2F`                     |
-| `abs.accessKey` | access key of storage account | `xxxx`             |
+| `s3.secretAccessKey` | secret access key of storage account     | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` |
-| `abs.cloud`     | name of cloud environment     | `AzurePublicCloud` |
+| `s3.region`          | region of the storage server             | `us-east-1`                                |
+| `s3.endpoint`        | endpoint for the storage server          | `https://play.minio.io:9000`               |
+| `s3.bucket`          | name of the bucket on the storage server | `vault`                                    |
 
 ### Backup job
 Define the schedule of the backup job
 
 | Parameter            | Description                  | Default       |
 | -------------------- | ---------------------------- | ------------- |
-| `backupJob.enable`   | Enable backup cronjob        | `true`        |
+| `backupJob.suspend`  | Suspend backup cronjob       | `false`       |
 | `backupJob.schedule` | cron schedule for the backup | `0 12 * * 1`  |
 
 ### UI

molgenis-vault/resources/backup.yaml

@@ -0,0 +1,13 @@
+apiVersion: "etcd.database.coreos.com/v1beta2"
+kind: "EtcdBackup"
+metadata:
+  name: vault-backup
+  namespace: "vault-operator"
+spec:
+  etcdEndpoints: ["https://vault-etcd-client:2379"]
+  storageType: S3
+  clientTLSSecret: vault-etcd-client-tls
+  s3:
+    path: vault/backup-manual
+    awsSecret: aws
+    endpoint: http://minio.minio.svc:9000

molgenis-vault/resources/restore.yaml

@@ -9,7 +9,8 @@ spec:
   etcdCluster:
     # The namespace is the same as this EtcdRestore CR
     name: vault-etcd
-  backupStorageType: ABS
+  backupStorageType: S3
-  abs:
+  s3:
-    path: vault/backup-<specify the backup name>
+    path: vault/backup-<name>
-    absSecret: abs
+    awsSecret: aws
+    endpoint: http://minio.minio.svc:9000

molgenis-vault/templates/NOTES.txt

@@ -3,11 +3,15 @@ Vault operator created
 Next steps:
 
 * Manually create a vault using resources/vault.yaml
-* Manually restore a backup using resources/backup.yaml
+* Manually restore a backup using resources/restore.yaml
 * Unseal the vault pods
 
-{{ if .Values.backupJob.enable }}
+{{ if .Values.backupJob.suspend }}
-!! Make sure to check if the backups succeed !!
+!!!!!! BACKUP JOB SUSPENDED !!!!!!
 {{ else }}
-!!!!!! NO BACKUPS CONFIGURED !!!!!!
+{{- if .Values.s3.endpoint -}}
+Backing up to non-standard s3 endpoint {{ .Values.s3.endpoint }} {{ else -}}
+Backing up to S3 on aws {{ end -}}
+in bucket {{ .Values.s3.bucket }}.
+!! Make sure to check if the backups succeed !!
 {{ end }}

molgenis-vault/templates/abs-secret.yaml

@@ -1,10 +0,0 @@
-# Secret to access microsoft azure blob store
-apiVersion: v1
-kind: Secret
-metadata:
-  name: abs
-type: Opaque
-stringData:
-  storage-account: {{ .Values.abs.account }}
-  storage-key: {{ .Values.abs.accessKey }}
-  cloud: {{ .Values.abs.cloud }}

molgenis-vault/templates/aws-secret.yaml

@@ -0,0 +1,10 @@
+# Secret to access s3 compatible store
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws
+type: Opaque
+data:
+  config: {{ printf "[default]\nregion = %s" .Values.s3.region | b64enc | quote }}
+  credentials: {{ printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" .Values.s3.accessKeyId .Values.s3.secretAccessKey | b64enc | quote }}
+

molgenis-vault/templates/backup-configmap.yaml

@@ -11,8 +11,14 @@ data:
       generateName: vault-backup-
     spec:
       etcdEndpoints: ["https://vault-etcd-client:2379"]
-      storageType: ABS
+      storageType: S3
       clientTLSSecret: vault-etcd-client-tls
-      abs:
+      s3:
-        path: vault/backup.<NOW>
+        path: {{ .Values.s3.bucket }}/backup.<NOW>
-        absSecret: abs
+        awsSecret: aws
+{{- if .Values.s3.endpoint }}
+        endpoint: {{ .Values.s3.endpoint }}
+{{- end }}
+{{- if hasKey .Values.s3 "forcePathStyle" }}
+        forcePathStyle: {{ .Values.s3.forcePathStyle }}
+{{- end }}

molgenis-vault/templates/backup-cronjob.yaml

@@ -1,10 +1,10 @@
-{{- if .Values.backupJob.enable }}
 # cronjob that creates etcdbackups using the etcd backup serviceaccount
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
   name: etcd-backup
 spec:
+  suspend: {{ .Values.backupJob.suspend }}
   schedule: {{ .Values.backupJob.schedule | quote }}
   jobTemplate:
     spec:
@@ -27,4 +27,3 @@ spec:
           - name: backup-config
             configMap:
               name: backup-config
-{{- end }}

molgenis-vault/values.yaml

@@ -2,19 +2,26 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-# abs gives details of the credentials to reach the azure backup storage
+# s3 configures s3 backup storage
-abs:
+s3:
-  # account is the name of the Storage account
+  # accessKey for the s3 storage account
-  account: fdlkops
+  accessKeyId: Q3AM3UQ867SPQQA43P2F
-  # access key for the Storage account
+  # secretAccessKey for the s3 storage account
-  accessKey: xxxx
+  secretAccessKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-  # default cloud
+  # region
-  cloud: AzurePublicCloud
+  region: us-east-1
+  # endpoint for the s3 storage
+  endpoint: https://play.minio.io:9000
+  # forcePathStyle if set to true forces requests to use path style
+  # (host/bucket instead of bucket.host)
+  forcePathStyle: true
+  # bucket is the name of the bucket
+  bucket: vault
 
 # backupjob describes the backup cronjob
 backupJob:
-  # enable enables the backup job
+  # suspend suspends the backup job
-  enable: true
+  suspend: false
   # schedule gives the cron schedule for the backup job
   schedule: "0 12 * * 1"
 
@@ -40,10 +47,12 @@ etcd-operator:
       tag: v0.9.2
   backupOperator:
     image:
-      tag: v0.9.2
+      repository: fdlk/etcd-operator
+      tag: latest
   restoreOperator:
     image:
-      tag: v0.9.2
+      repository: fdlk/etcd-operator
+      tag: latest
 
 ui:
   name: "vault-ui"