Merge branch 'master' of https://git.webhosting.rug.nl/molgenis/molgenis-ops-docker-helm

molgenis-jenkins/README.md

@@ -25,47 +25,59 @@ Array values can be added as {value, value, value}.
 jenkins.Master.HostName=jenkins.molgenis.org
 jenkins.Master.AdminPassword=pa$$word
 jenkins.Persistence.Enabled=false
-jenkins.Master.InstallPlugins={kubernetes:1.8.4, workflow-aggregator:2.5, workflow-job:2.21, credentials-binding:1.16, git:3.9.1}
+jenkins.Master.InstallPlugins={kubernetes:1.8.4, workflow-aggregator:2.5, workflow-job:2.21, credentials-binding:1.16, git:3.9.1, blueocean:1.6.2, github-oauth:0.29}
+jenkins.Master.Security.UseGitHub=false
+## if UseGitHub=true
+jenkins.Master.Security.GitHub.ClientID=id
+jenkins.Master.Security.GitHub.ClientSecret=S3cr3t
+## end UseGitHub=true
 PipelineSecrets.Env.PGPPassphrase=literal:S3cr3t
 ```
 
 You can use [all configuration values of the jenkins subchart](https://github.com/kubernetes/charts/tree/master/stable/jenkins).
 > Because we use jenkins as a sub-chart, you should prefix all value keys with `jenkins`!
 
+### GitHub Authentication delegation
+You need to setup a MOLGENIS - Jenkins GitHub OAuth App. You can do this by accessing this url: [add new OAuth app](https://github.com/settings/applications/new).
+
+### Additional configuration
 There is one additional group of configuration items specific for this chart, so not prefixed with `jenkins`:
-## PipelineSecrets
 
-When deployed, the chart creates a couple of kubernetes secrets that get used by jenkins and mounted in the jenkins 
-build pods. The secrets, like the rest of the deployment, is namespaced so multiple instances can run beside
-each other with their own secrets.
+* PipelineSecrets
 
-You can override the values at deploy time but otherwise also configure them 
-[in Rancher](https://rancher.molgenis.org:7443/p/c-mhkqb:project-2pf45/secrets) or through kubectl.
+   When deployed, the chart creates a couple of kubernetes secrets that get used by jenkins and mounted in the jenkins 
+   build pods. The secrets, like the rest of the deployment, is namespaced so multiple instances can run beside
+   each other with their own secrets.
 
-### Env
-Environment variables stored in molgenis-pipeline-env secret, to be added as environment variables
-in the slave pods.
+   You can override the values at deploy time but otherwise also configure them 
+   [in Rancher](https://rancher.molgenis.org:7443/p/c-mhkqb:project-2pf45/secrets) or through kubectl.
 
-| Parameter                          | Description                          | Default         |
-| ---------------------------------- | ------------------------------------ | --------------- |
-| `PipelineSecrets.Env.Replace`      | Replace molgenis-pipeline-env secret | `true`          |
-| `PipelineSecrets.Env.PGPPassphrase`| passphrase for the pgp signing key   | `literal:xxxx`  |
-| `PipelineSecrets.Env.CodecovToken` | token for codecov.io                 | `xxxx`          |
-| `PipelineSecrets.Env.GitHubToken`  | token for GH molgenis-jenkins user   | `xxxx`          |
-| `PipelineSecrets.Env.SonarToken`   | token for sonarcloud.io              | `xxxx`          |                                                            |
+*  Env
+   
+   Environment variables stored in molgenis-pipeline-env secret, to be added as environment variables
+   in the slave pods.
 
-### File
+   | Parameter                              | Description                               | Default         |
+   | -------------------------------------- | ----------------------------------------- | --------------- |
+   | `PipelineSecrets.Env.Replace`          | Replace molgenis-pipeline-env secret      | `true`          |
+   | `PipelineSecrets.Env.PGPPassphrase`    | passphrase for the pgp signing key        | `literal:xxxx`  |
+   | `PipelineSecrets.Env.CodecovToken`     | token for codecov.io                      | `xxxx`          |
+   | `PipelineSecrets.Env.GitHubToken`      | token for GH molgenis-jenkins user        | `xxxx`          |
+   | `PipelineSecrets.Env.NexusPassword`    | token for molgenis-jenkins user in NEXUS  | `xxxx`          |
+   | `PipelineSecrets.Env.DockerHubPassword`| token for molgenis user in hub.docker.com | `xxxx`          |
+   | `PipelineSecrets.Env.SonarToken`       | token for sonarcloud.io                   | `xxxx`          |                                                            |
 
-Environment variables stored in molgenis-pipeline-file secret, to be mounted as files
-in the `/root/.m2` directory of the slave pods.
-> The settings.xml file references the 
+* File
 
-| Parameter                              | Description                           | Default                                                                         |
-| -------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------- |
-| `PipelineSecrets.File.Replace`         | Replace molgenis-pipeline-file secret | `true`                                                                          |
-| `PipelineSecrets.File.PGPPrivateKeyAsc`| pgp signing key in ascii form         | `-----BEGIN PGP PRIVATE KEY BLOCK-----xxxxx-----END PGP PRIVATE KEY BLOCK-----` |
-| `PipelineSecrets.File.MavenSettingsXML`| Maven settings.xml file               | `<settings>[...]</settings>` (see actual [values.yaml](values.yaml))            |
+  Environment variables stored in molgenis-pipeline-file secret, to be mounted as files
+  in the `/root/.m2` directory of the slave pods.
+  > The settings.xml file references the 
 
+  | Parameter                              | Description                           | Default                                                                         |
+  | -------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------- |
+  | `PipelineSecrets.File.Replace`         | Replace molgenis-pipeline-file secret | `true`                                                                          |
+  | `PipelineSecrets.File.PGPPrivateKeyAsc`| pgp signing key in ascii form         | `-----BEGIN PGP PRIVATE KEY BLOCK-----xxxxx-----END PGP PRIVATE KEY BLOCK-----` |
+  | `PipelineSecrets.File.MavenSettingsXML`| Maven settings.xml file               | `<settings>[...]</settings>` (see actual [values.yaml](values.yaml))            |
 
 ## Command line use
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.

molgenis-jenkins/templates/config.tpl

@@ -15,7 +15,17 @@ data:
       <authorizationStrategy class="hudson.security.FullControlOnceLoggedInAuthorizationStrategy">
         <denyAnonymousReadAccess>true</denyAnonymousReadAccess>
       </authorizationStrategy>
+{{- if .Values.jenkins.Master.Security.UseGitHub }}
+      <securityRealm class="org.jenkinsci.plugins.GithubSecurityRealm">
+        <githubWebUri>https://github.com</githubWebUri>
+        <githubApiUri>https://api.github.com</githubApiUri>
+        <clientID>{{ .Values.jenkins.Master.Security.Github.ClientID }}</clientID>
+        <clientSecret>{{ .Values.jenkins.Master.Security.Github.ClientSecret }}</clientSecret>
+        <oauthScopes>read:org,user:email</oauthScopes>
+      </securityRealm>
+{{- else }}
       <securityRealm class="hudson.security.LegacySecurityRealm"/>
+{{- end }}
       <disableRememberMe>false</disableRememberMe>
       <projectNamingStrategy class="jenkins.model.ProjectNamingStrategy$DefaultProjectNamingStrategy"/>
       <workspaceDir>${JENKINS_HOME}/workspace/${ITEM_FULLNAME}</workspaceDir>

molgenis-jenkins/templates/molgenis-jenkins-dockerhub-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+# this is the jenkins id.
+  name: "molgenis-jenkins-dockerhub-secret"
+  labels:
+# so we know what type it is.
+    "jenkins.io/credentials-type": "usernamePassword"
+  annotations: {
+# description - can not be a label as spaces are not allowed
+    "jenkins.io/credentials-description" : "user to authenticate against Docker Hub (hub.docker.com)"
+  }
+type: Opaque
+data:
+  username: {{ "molgenisci" | b64enc | quote }}
+  password: {{ .Values.PipelineSecrets.Env.DockerHubPassword | b64enc | quote }}

molgenis-jenkins/values.yaml

@@ -10,7 +10,13 @@ jenkins:
       - git:3.9.1
       - github-branch-source:2.3.6
       - kubernetes-credentials-provider:0.9
-      - blueocean:1.6.1
+      - blueocean:1.6.2
+      - github-oauth:0.29
+    Security:
+      UseGitHub: false
+      GitHub:
+        ClientID: ""
+        ClienSecret: ""
     Jobs: |-
       molgenis: |-
         <?xml version='1.1' encoding='UTF-8'?>
@@ -269,6 +275,8 @@ PipelineSecrets:
     SonarToken: xxxx
     # Password Local NEXUS
     NexusPassword: xxxx
+    # Password hub.docker.com
+    DockerHubPassword: xxxx
   File:
     # Set to false to keep existing secret
     Replace: true

molgenis/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

molgenis/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: molgenis
+version: 0.1.0

molgenis/templates/NOTES.txt

@@ -0,0 +1,19 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "molgenis.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ template "molgenis.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "molgenis.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "molgenis.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:80
+{{- end }}

molgenis/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "molgenis.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "molgenis.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "molgenis.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

molgenis/templates/deployment.yaml

@@ -0,0 +1,128 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ template "molgenis.fullname" . }}
+  labels:
+    app: {{ template "molgenis.name" . }}
+    chart: {{ template "molgenis.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "molgenis.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "molgenis.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: molgenis
+        {{- with .Values.molgenis }}
+          image: "{{ .image.repository }}:{{ .image.tag }}"
+          imagePullPolicy: {{ .image.pullPolicy }}
+          env:
+            - name: molgenis.home
+              value: /home/molgenis
+            - name: opencpu.uri.host
+              value: opencpu
+            - name: elasticsearch.transport.addresses
+              value: elasticsearch:9300
+            - name: elasticsearch.cluster.name
+              value: {{ $.Values.elasticsearch.clusterName }}
+            - name: db_uri
+              value: "jdbc:postgresql://postgres/{{ $.Values.postgres.db }}"
+            - name: db_user
+              value: {{ $.Values.postgres.user }}
+            - name: db_password
+              value: {{ $.Values.postgres.password }}
+            - name: admin.password
+              value: {{ .adminPassword }}
+            - name: CATALINA_OPTS
+              value: "{{ .javaOpts }}"
+          ports:
+            - containerPort: 8080
+          command: ["sh"]
+          args: ["-c", "cat"]
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /api/v2/version
+              port: http
+          resources:
+{{ toYaml .resources | indent 12 }}
+        {{- end }}
+
+        - name: elasticsearch
+        {{- with .Values.elasticsearch }}
+          image: "{{ .image.repository }}:{{ .image.tag }}"
+          imagePullPolicy: {{ .image.pullPolicy }}
+          env:
+            - name: cluster.name
+              value: {{ .clusterName }}
+            - name: bootstrap.memory_lock
+              value: "true"
+            - name: ES_JAVA_OPTS
+              value: "{{ .javaOpts }}"
+            - name: xpack.security.enabled
+              value: "false"
+            - name: discovery.type
+              value: single-node
+          ports:
+            - containerPort: 9200
+            - containerPort: 9300
+          command: ["sh"]
+          args: ["-c", "cat"]
+          resources:
+{{ toYaml .resources | indent 12 }}
+        {{- end }}
+
+        - name: postgres
+        {{- with .Values.postgres }}
+          image: "{{ .image.repository }}:{{ .image.tag }}"
+          imagePullPolicy: {{ .image.pullPolicy }}
+          env:
+            - name: POSTGRES_USER
+              value: {{ .user }}
+            - name: POSTGRES_PASSWORD
+              value: {{ .password }}
+            - name: POSTGRES_DB
+              value: {{ .db }}
+          ports:
+            - containerPort: 5432
+          command: ["sh"]
+          args: ["-c", "cat"]
+          resources:
+{{ toYaml .resources | indent 12 }}
+        {{- end }}
+
+        - name: opencpu
+        {{- with .Values.opencpu }}
+          image: "{{ .image.repository }}:{{ .image.tag }}"
+          imagePullPolicy: {{ .image.pullPolicy }}
+          ports:
+            - containerPort: 8004
+          command: ["sh"]
+          args: ["-c", "cat"]
+          resources:
+{{ toYaml .resources | indent 12 }}
+        {{- end }}
+
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}

molgenis/templates/ingress.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "molgenis.fullname" . -}}
+{{- $ingressPath := .Values.ingress.path -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    app: {{ template "molgenis.name" . }}
+    chart: {{ template "molgenis.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- with .Values.ingress.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+{{- if .Values.ingress.tls }}
+  tls:
+  {{- range .Values.ingress.tls }}
+    - hosts:
+      {{- range .hosts }}
+        - {{ . }}
+      {{- end }}
+      secretName: {{ .secretName }}
+  {{- end }}
+{{- end }}
+  rules:
+  {{- range .Values.ingress.hosts }}
+    - host: {{ . }}
+      http:
+        paths:
+          - path: {{ $ingressPath }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: http
+  {{- end }}
+{{- end }}

molgenis/templates/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "molgenis.fullname" . }}
+  labels:
+    app: {{ template "molgenis.name" . }}
+    chart: {{ template "molgenis.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ template "molgenis.name" . }}
+    release: {{ .Release.Name }}

molgenis/test.yaml

@@ -0,0 +1,120 @@
+# Source: molgenis/templates/deployment.yaml
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: lanky-ragdoll-molgenis
+  labels:
+    app: molgenis
+    chart: molgenis-0.1.0
+    release: lanky-ragdoll
+    heritage: Tiller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: molgenis
+      release: lanky-ragdoll
+  template:
+    metadata:
+      labels:
+        app: molgenis
+        release: lanky-ragdoll
+    spec:
+      containers:
+        - name: molgenis
+          image: "registry.molgenis.org/molgenis/molgenis-app:latest"
+          imagePullPolicy: Always
+          env:
+            - name: molgenis.home
+              value: /home/molgenis
+            - name: opencpu.uri.host
+              value: opencpu
+            - name: elasticsearch.transport.addresses
+              value: elasticsearch:9300
+            - name: elasticsearch.cluster.name
+              value: molgenis
+            - name: db_uri
+              value: "jdbc:postgresql://postgres/molgenis"
+            - name: db_user
+              value: molgenis
+            - name: db_password
+              value: molgenis
+            - name: admin.password
+              value: admin
+            - name: CATALINA_OPTS
+              value: "-Xmx1g -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled"
+          ports:
+            - containerPort: 8080
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /api/v2/version
+              port: http
+          resources:
+            limits:
+              cpu: 1
+              memory: 1250Mi
+            requests:
+              cpu: 200m
+              memory: 1Gi
+
+
+        - name: elasticsearch
+          image: "docker.elastic.co/elasticsearch/elasticsearch:5.5.3"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: cluster.name
+              value: molgenis
+            - name: bootstrap.memory_lock
+              value: true
+            - name: ES_JAVA_OPTS
+              value: "-Xms512m -Xmx512m"
+            - name: xpack.security.enabled
+              value: false
+            - name: discovery.type
+              value: single-node
+          ports:
+            - containerPort: 9200
+            - containerPort: 9300
+            limits:
+              cpu: 1
+              memory: 1500Mi
+            requests:
+              cpu: 100m
+              memory: 1Gi
+
+
+        - name: postgres
+          image: "postgres:9.6-alpine"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: POSTGRES_USER
+              value: molgenis
+            - name: POSTGRES_PASSWORD
+              value: molgenis
+            - name: POSTGRES_DB
+              value: molgenis
+          ports:
+            - containerPort: 5432
+            limits:
+              cpu: 1
+              memory: 250Mi
+            requests:
+              cpu: 100m
+              memory: 250Mi
+
+
+        - name: opencpu
+          image: "molgenis/opencpu:latest"
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8004
+            limits:
+              cpu: 1
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 256Mi

molgenis/values.yaml

@@ -0,0 +1,82 @@
+# Default values for molgenis.
+
+replicaCount: 1
+
+service:
+  type: ClusterIP
+  port: 8080
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  path: /
+  hosts:
+    - molgenis.local
+  tls: []
+
+molgenis:
+  image:
+    repository: registry.molgenis.org/molgenis/molgenis-app
+    tag: 7.0.0-SNAPSHOT
+    pullPolicy: Always
+  adminPassword: admin
+  javaOpts: "-Xmx1g -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled"
+  resources:
+   limits:
+    cpu: 1
+    memory: 1250Mi
+   requests:
+    cpu: 200m
+    memory: 1Gi
+
+postgres:
+  image:
+    repository: postgres
+    tag: 9.6-alpine
+    pullPolicy: IfNotPresent
+  user: molgenis
+  password: molgenis
+  db: molgenis
+  resources:
+   limits:
+    cpu: 1
+    memory: 250Mi
+   requests:
+    cpu: 100m
+    memory: 250Mi
+
+elasticsearch:
+  image:
+    repository: docker.elastic.co/elasticsearch/elasticsearch
+    tag: 5.5.3
+    pullPolicy: IfNotPresent
+  javaOpts: "-Xms512m -Xmx512m"
+  clusterName: molgenis
+  resources:
+   limits:
+    cpu: 1
+    memory: 1500Mi
+   requests:
+    cpu: 100m
+    memory: 1Gi
+
+opencpu:
+  image:
+    repository: molgenis/opencpu
+    tag: latest
+    pullPolicy: Always
+  resources:
+    limits:
+      cpu: 1
+      memory: 512Mi
+    requests:
+      cpu: 100m
+      memory: 256Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}