add slack integration

alertmanager/config.yml

@@ -1,33 +1,10 @@
-global:
-  # The smarthost and SMTP sender used for mail notifications.
-  smtp_smarthost: 'localhost:25'
-  smtp_from: 'alertmanager@example.org'
-  smtp_auth_username: 'alertmanager'
-  smtp_auth_password: 'password'
-  # The auth token for Hipchat.
-  hipchat_auth_token: '1234556789'
-  # Alternative host for Hipchat.
-  hipchat_url: 'https://hipchat.foobar.org/'
-
-# The directory from which notification templates are read.
-templates: 
-- '/etc/alertmanager/template/*.tmpl'
-
-# The root route on which each incoming alert enters.
 route:
-  group_by: [cluster]
+    receiver: 'slack'
-  # If an alert isn't caught by a route, send it slack.
-  receiver: slack_general
 
-  # The child route trees.
-  routes:
-  # Send severity=slack alerts to slack.
-    - match:
-        severity: slack
-        receiver: slack_general
 receivers:
-- name: slack_general
+    - name: 'slack'
       slack_configs:
-  - api_url: 'https://hooks.slack.com/services/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'
+          - send_resolved: true
+            username: 'llitfkitfk'
             channel: '#prometheus'
-    send_resolved: true
+            api_url: 'https://hooks.slack.com/services/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'

docker-compose.yml

@@ -23,6 +23,8 @@ services:
       - '-alertmanager.url=http://alertmanager:9093'
     expose:
       - 9090
+    ports:
+      - 9090:9090
     links:
       - cadvisor:cadvisor
       - alertmanager:alertmanager

prometheus/alert.rules

@@ -1,7 +1,9 @@
-ALERT instance_down
+ALERT service_down
-IF up == 0
+  IF up == 0
-FOR 5s
+  
-LABELS {severity="page"}
+ALERT high_load
-ANNOTATIONS {
+  IF node_load1 > 0.5
-   DESCRIPTION="{{$labels.instance}} of job {{$labels.job}} has been down for more than 5 seconds.",
+  ANNOTATIONS {
-   SUMMARY="Instance {{$labels.instance}} down"}
+      summary = "Instance {{ $labels.instance }} under high load",
+      description = "{{ $labels.instance }} of job {{ $labels.job }} is under high load.",
+  }