created docker-prometheus compose file and edited the Prometheus yml file
This commit is contained in:
Brian Christner
276
prom/rules/alerting.go
Normal file
276
prom/rules/alerting.go
Normal file
@ -0,0 +1,276 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
clientmodel "github.com/prometheus/client_golang/model"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
|
||||
const (
|
||||
// AlertMetricName is the metric name for synthetic alert timeseries.
|
||||
alertMetricName clientmodel.LabelValue = "ALERTS"
|
||||
|
||||
// AlertNameLabel is the label name indicating the name of an alert.
|
||||
alertNameLabel clientmodel.LabelName = "alertname"
|
||||
// AlertStateLabel is the label name indicating the state of an alert.
|
||||
alertStateLabel clientmodel.LabelName = "alertstate"
|
||||
)
|
||||
|
||||
// AlertState denotes the state of an active alert.
|
||||
type AlertState int
|
||||
|
||||
func (s AlertState) String() string {
|
||||
switch s {
|
||||
case StateInactive:
|
||||
return "inactive"
|
||||
case StatePending:
|
||||
return "pending"
|
||||
case StateFiring:
|
||||
return "firing"
|
||||
default:
|
||||
panic("undefined")
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
// Inactive alerts are neither firing nor pending.
|
||||
StateInactive AlertState = iota
|
||||
// Pending alerts have been active for less than the configured
|
||||
// threshold duration.
|
||||
StatePending
|
||||
// Firing alerts have been active for longer than the configured
|
||||
// threshold duration.
|
||||
StateFiring
|
||||
)
|
||||
|
||||
// Alert is used to track active (pending/firing) alerts over time.
|
||||
type Alert struct {
|
||||
// The name of the alert.
|
||||
Name string
|
||||
// The vector element labelset triggering this alert.
|
||||
Labels clientmodel.LabelSet
|
||||
// The state of the alert (Pending or Firing).
|
||||
State AlertState
|
||||
// The time when the alert first transitioned into Pending state.
|
||||
ActiveSince clientmodel.Timestamp
|
||||
// The value of the alert expression for this vector element.
|
||||
Value clientmodel.SampleValue
|
||||
}
|
||||
|
||||
// sample returns a Sample suitable for recording the alert.
|
||||
func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleValue) *promql.Sample {
|
||||
recordedMetric := clientmodel.Metric{}
|
||||
for label, value := range a.Labels {
|
||||
recordedMetric[label] = value
|
||||
}
|
||||
|
||||
recordedMetric[clientmodel.MetricNameLabel] = alertMetricName
|
||||
recordedMetric[alertNameLabel] = clientmodel.LabelValue(a.Name)
|
||||
recordedMetric[alertStateLabel] = clientmodel.LabelValue(a.State.String())
|
||||
|
||||
return &promql.Sample{
|
||||
Metric: clientmodel.COWMetric{
|
||||
Metric: recordedMetric,
|
||||
Copied: true,
|
||||
},
|
||||
Value: value,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
}
|
||||
|
||||
// An AlertingRule generates alerts from its vector expression.
|
||||
type AlertingRule struct {
|
||||
// The name of the alert.
|
||||
name string
|
||||
// The vector expression from which to generate alerts.
|
||||
vector promql.Expr
|
||||
// The duration for which a labelset needs to persist in the expression
|
||||
// output vector before an alert transitions from Pending to Firing state.
|
||||
holdDuration time.Duration
|
||||
// Extra labels to attach to the resulting alert sample vectors.
|
||||
labels clientmodel.LabelSet
|
||||
// Short alert summary, suitable for email subjects.
|
||||
summary string
|
||||
// More detailed alert description.
|
||||
description string
|
||||
// A reference to a runbook for the alert.
|
||||
runbook string
|
||||
|
||||
// Protects the below.
|
||||
mutex sync.Mutex
|
||||
// A map of alerts which are currently active (Pending or Firing), keyed by
|
||||
// the fingerprint of the labelset they correspond to.
|
||||
activeAlerts map[clientmodel.Fingerprint]*Alert
|
||||
}
|
||||
|
||||
// NewAlertingRule constructs a new AlertingRule.
|
||||
func NewAlertingRule(
|
||||
name string,
|
||||
vector promql.Expr,
|
||||
holdDuration time.Duration,
|
||||
labels clientmodel.LabelSet,
|
||||
summary string,
|
||||
description string,
|
||||
runbook string,
|
||||
) *AlertingRule {
|
||||
return &AlertingRule{
|
||||
name: name,
|
||||
vector: vector,
|
||||
holdDuration: holdDuration,
|
||||
labels: labels,
|
||||
summary: summary,
|
||||
description: description,
|
||||
runbook: runbook,
|
||||
|
||||
activeAlerts: map[clientmodel.Fingerprint]*Alert{},
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the alert.
|
||||
func (rule *AlertingRule) Name() string {
|
||||
return rule.name
|
||||
}
|
||||
|
||||
// eval evaluates the rule expression and then creates pending alerts and fires
|
||||
// or removes previously pending alerts accordingly.
|
||||
func (rule *AlertingRule) eval(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
|
||||
query, err := engine.NewInstantQuery(rule.vector.String(), timestamp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exprResult, err := query.Exec().Vector()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rule.mutex.Lock()
|
||||
defer rule.mutex.Unlock()
|
||||
|
||||
// Create pending alerts for any new vector elements in the alert expression
|
||||
// or update the expression value for existing elements.
|
||||
resultFPs := map[clientmodel.Fingerprint]struct{}{}
|
||||
for _, sample := range exprResult {
|
||||
fp := sample.Metric.Metric.Fingerprint()
|
||||
resultFPs[fp] = struct{}{}
|
||||
|
||||
if alert, ok := rule.activeAlerts[fp]; !ok {
|
||||
labels := clientmodel.LabelSet{}
|
||||
labels.MergeFromMetric(sample.Metric.Metric)
|
||||
labels = labels.Merge(rule.labels)
|
||||
if _, ok := labels[clientmodel.MetricNameLabel]; ok {
|
||||
delete(labels, clientmodel.MetricNameLabel)
|
||||
}
|
||||
rule.activeAlerts[fp] = &Alert{
|
||||
Name: rule.name,
|
||||
Labels: labels,
|
||||
State: StatePending,
|
||||
ActiveSince: timestamp,
|
||||
Value: sample.Value,
|
||||
}
|
||||
} else {
|
||||
alert.Value = sample.Value
|
||||
}
|
||||
}
|
||||
|
||||
vector := promql.Vector{}
|
||||
|
||||
// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
|
||||
for fp, activeAlert := range rule.activeAlerts {
|
||||
if _, ok := resultFPs[fp]; !ok {
|
||||
vector = append(vector, activeAlert.sample(timestamp, 0))
|
||||
delete(rule.activeAlerts, fp)
|
||||
continue
|
||||
}
|
||||
|
||||
if activeAlert.State == StatePending && timestamp.Sub(activeAlert.ActiveSince) >= rule.holdDuration {
|
||||
vector = append(vector, activeAlert.sample(timestamp, 0))
|
||||
activeAlert.State = StateFiring
|
||||
}
|
||||
|
||||
vector = append(vector, activeAlert.sample(timestamp, 1))
|
||||
}
|
||||
|
||||
return vector, nil
|
||||
}
|
||||
|
||||
func (rule *AlertingRule) String() string {
|
||||
s := fmt.Sprintf("ALERT %s", rule.name)
|
||||
s += fmt.Sprintf("\n\tIF %s", rule.vector)
|
||||
if rule.holdDuration > 0 {
|
||||
s += fmt.Sprintf("\n\tFOR %s", strutil.DurationToString(rule.holdDuration))
|
||||
}
|
||||
if len(rule.labels) > 0 {
|
||||
s += fmt.Sprintf("\n\tWITH %s", rule.labels)
|
||||
}
|
||||
s += fmt.Sprintf("\n\tSUMMARY %q", rule.summary)
|
||||
s += fmt.Sprintf("\n\tDESCRIPTION %q", rule.description)
|
||||
s += fmt.Sprintf("\n\tRUNBOOK %q", rule.runbook)
|
||||
return s
|
||||
}
|
||||
|
||||
// HTMLSnippet returns an HTML snippet representing this alerting rule. The
|
||||
// resulting snippet is expected to be presented in a <pre> element, so that
|
||||
// line breaks and other returned whitespace is respected.
|
||||
func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
|
||||
alertMetric := clientmodel.Metric{
|
||||
clientmodel.MetricNameLabel: alertMetricName,
|
||||
alertNameLabel: clientmodel.LabelValue(rule.name),
|
||||
}
|
||||
s := fmt.Sprintf("ALERT <a href=%q>%s</a>", pathPrefix+strutil.GraphLinkForExpression(alertMetric.String()), rule.name)
|
||||
s += fmt.Sprintf("\n IF <a href=%q>%s</a>", pathPrefix+strutil.GraphLinkForExpression(rule.vector.String()), rule.vector)
|
||||
if rule.holdDuration > 0 {
|
||||
s += fmt.Sprintf("\n FOR %s", strutil.DurationToString(rule.holdDuration))
|
||||
}
|
||||
if len(rule.labels) > 0 {
|
||||
s += fmt.Sprintf("\n WITH %s", rule.labels)
|
||||
}
|
||||
s += fmt.Sprintf("\n SUMMARY %q", rule.summary)
|
||||
s += fmt.Sprintf("\n DESCRIPTION %q", rule.description)
|
||||
s += fmt.Sprintf("\n RUNBOOK %q", rule.runbook)
|
||||
return template.HTML(s)
|
||||
}
|
||||
|
||||
// State returns the "maximum" state: firing > pending > inactive.
|
||||
func (rule *AlertingRule) State() AlertState {
|
||||
rule.mutex.Lock()
|
||||
defer rule.mutex.Unlock()
|
||||
|
||||
maxState := StateInactive
|
||||
for _, activeAlert := range rule.activeAlerts {
|
||||
if activeAlert.State > maxState {
|
||||
maxState = activeAlert.State
|
||||
}
|
||||
}
|
||||
return maxState
|
||||
}
|
||||
|
||||
// ActiveAlerts returns a slice of active alerts.
|
||||
func (rule *AlertingRule) ActiveAlerts() []Alert {
|
||||
rule.mutex.Lock()
|
||||
defer rule.mutex.Unlock()
|
||||
|
||||
alerts := make([]Alert, 0, len(rule.activeAlerts))
|
||||
for _, alert := range rule.activeAlerts {
|
||||
alerts = append(alerts, *alert)
|
||||
}
|
||||
return alerts
|
||||
}
|
||||
398
prom/rules/manager.go
Normal file
398
prom/rules/manager.go
Normal file
@ -0,0 +1,398 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
html_template "html/template"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/log"
|
||||
|
||||
clientmodel "github.com/prometheus/client_golang/model"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/notification"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/template"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
|
||||
// Constants for instrumentation.
|
||||
const (
|
||||
namespace = "prometheus"
|
||||
|
||||
ruleTypeLabel = "rule_type"
|
||||
ruleTypeAlerting = "alerting"
|
||||
ruleTypeRecording = "recording"
|
||||
)
|
||||
|
||||
var (
|
||||
evalDuration = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Name: "rule_evaluation_duration_milliseconds",
|
||||
Help: "The duration for a rule to execute.",
|
||||
},
|
||||
[]string{ruleTypeLabel},
|
||||
)
|
||||
evalFailures = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "rule_evaluation_failures_total",
|
||||
Help: "The total number of rule evaluation failures.",
|
||||
},
|
||||
)
|
||||
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Name: "evaluator_duration_milliseconds",
|
||||
Help: "The duration for all evaluations to execute.",
|
||||
Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001},
|
||||
})
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(iterationDuration)
|
||||
prometheus.MustRegister(evalFailures)
|
||||
prometheus.MustRegister(evalDuration)
|
||||
}
|
||||
|
||||
// A Rule encapsulates a vector expression which is evaluated at a specified
|
||||
// interval and acted upon (currently either recorded or used for alerting).
|
||||
type Rule interface {
|
||||
// Name returns the name of the rule.
|
||||
Name() string
|
||||
// Eval evaluates the rule, including any associated recording or alerting actions.
|
||||
eval(clientmodel.Timestamp, *promql.Engine) (promql.Vector, error)
|
||||
// String returns a human-readable string representation of the rule.
|
||||
String() string
|
||||
// HTMLSnippet returns a human-readable string representation of the rule,
|
||||
// decorated with HTML elements for use the web frontend.
|
||||
HTMLSnippet(pathPrefix string) html_template.HTML
|
||||
}
|
||||
|
||||
// The Manager manages recording and alerting rules.
|
||||
type Manager struct {
|
||||
// Protects the rules list.
|
||||
sync.Mutex
|
||||
rules []Rule
|
||||
|
||||
done chan bool
|
||||
|
||||
interval time.Duration
|
||||
queryEngine *promql.Engine
|
||||
|
||||
sampleAppender storage.SampleAppender
|
||||
notificationHandler *notification.NotificationHandler
|
||||
|
||||
externalURL *url.URL
|
||||
}
|
||||
|
||||
// ManagerOptions bundles options for the Manager.
|
||||
type ManagerOptions struct {
|
||||
EvaluationInterval time.Duration
|
||||
QueryEngine *promql.Engine
|
||||
|
||||
NotificationHandler *notification.NotificationHandler
|
||||
SampleAppender storage.SampleAppender
|
||||
|
||||
ExternalURL *url.URL
|
||||
}
|
||||
|
||||
// NewManager returns an implementation of Manager, ready to be started
|
||||
// by calling the Run method.
|
||||
func NewManager(o *ManagerOptions) *Manager {
|
||||
manager := &Manager{
|
||||
rules: []Rule{},
|
||||
done: make(chan bool),
|
||||
|
||||
interval: o.EvaluationInterval,
|
||||
sampleAppender: o.SampleAppender,
|
||||
queryEngine: o.QueryEngine,
|
||||
notificationHandler: o.NotificationHandler,
|
||||
externalURL: o.ExternalURL,
|
||||
}
|
||||
return manager
|
||||
}
|
||||
|
||||
// Run the rule manager's periodic rule evaluation.
|
||||
func (m *Manager) Run() {
|
||||
defer log.Info("Rule manager stopped.")
|
||||
|
||||
m.Lock()
|
||||
lastInterval := m.interval
|
||||
m.Unlock()
|
||||
|
||||
ticker := time.NewTicker(lastInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
// The outer select clause makes sure that m.done is looked at
|
||||
// first. Otherwise, if m.runIteration takes longer than
|
||||
// m.interval, there is only a 50% chance that m.done will be
|
||||
// looked at before the next m.runIteration call happens.
|
||||
select {
|
||||
case <-m.done:
|
||||
return
|
||||
default:
|
||||
select {
|
||||
case <-ticker.C:
|
||||
start := time.Now()
|
||||
m.runIteration()
|
||||
iterationDuration.Observe(float64(time.Since(start) / time.Millisecond))
|
||||
|
||||
m.Lock()
|
||||
if lastInterval != m.interval {
|
||||
ticker.Stop()
|
||||
ticker = time.NewTicker(m.interval)
|
||||
lastInterval = m.interval
|
||||
}
|
||||
m.Unlock()
|
||||
case <-m.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the rule manager's rule evaluation cycles.
|
||||
func (m *Manager) Stop() {
|
||||
log.Info("Stopping rule manager...")
|
||||
m.done <- true
|
||||
}
|
||||
|
||||
func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmodel.Timestamp) {
|
||||
activeAlerts := rule.ActiveAlerts()
|
||||
if len(activeAlerts) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
notifications := make(notification.NotificationReqs, 0, len(activeAlerts))
|
||||
for _, aa := range activeAlerts {
|
||||
if aa.State != StateFiring {
|
||||
// BUG: In the future, make AlertManager support pending alerts?
|
||||
continue
|
||||
}
|
||||
|
||||
// Provide the alert information to the template.
|
||||
l := map[string]string{}
|
||||
for k, v := range aa.Labels {
|
||||
l[string(k)] = string(v)
|
||||
}
|
||||
tmplData := struct {
|
||||
Labels map[string]string
|
||||
Value clientmodel.SampleValue
|
||||
}{
|
||||
Labels: l,
|
||||
Value: aa.Value,
|
||||
}
|
||||
// Inject some convenience variables that are easier to remember for users
|
||||
// who are not used to Go's templating system.
|
||||
defs := "{{$labels := .Labels}}{{$value := .Value}}"
|
||||
|
||||
expand := func(text string) string {
|
||||
tmpl := template.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
|
||||
result, err := tmpl.Expand()
|
||||
if err != nil {
|
||||
result = err.Error()
|
||||
log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
notifications = append(notifications, ¬ification.NotificationReq{
|
||||
Summary: expand(rule.summary),
|
||||
Description: expand(rule.description),
|
||||
Runbook: rule.runbook,
|
||||
Labels: aa.Labels.Merge(clientmodel.LabelSet{
|
||||
alertNameLabel: clientmodel.LabelValue(rule.Name()),
|
||||
}),
|
||||
Value: aa.Value,
|
||||
ActiveSince: aa.ActiveSince.Time(),
|
||||
RuleString: rule.String(),
|
||||
GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
|
||||
})
|
||||
}
|
||||
m.notificationHandler.SubmitReqs(notifications)
|
||||
}
|
||||
|
||||
func (m *Manager) runIteration() {
|
||||
now := clientmodel.Now()
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
m.Lock()
|
||||
rulesSnapshot := make([]Rule, len(m.rules))
|
||||
copy(rulesSnapshot, m.rules)
|
||||
m.Unlock()
|
||||
|
||||
for _, rule := range rulesSnapshot {
|
||||
wg.Add(1)
|
||||
// BUG(julius): Look at fixing thundering herd.
|
||||
go func(rule Rule) {
|
||||
defer wg.Done()
|
||||
|
||||
start := time.Now()
|
||||
vector, err := rule.eval(now, m.queryEngine)
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
evalFailures.Inc()
|
||||
log.Warnf("Error while evaluating rule %q: %s", rule, err)
|
||||
return
|
||||
}
|
||||
|
||||
switch r := rule.(type) {
|
||||
case *AlertingRule:
|
||||
m.queueAlertNotifications(r, now)
|
||||
evalDuration.WithLabelValues(ruleTypeAlerting).Observe(
|
||||
float64(duration / time.Millisecond),
|
||||
)
|
||||
case *RecordingRule:
|
||||
evalDuration.WithLabelValues(ruleTypeRecording).Observe(
|
||||
float64(duration / time.Millisecond),
|
||||
)
|
||||
default:
|
||||
panic(fmt.Errorf("Unknown rule type: %T", rule))
|
||||
}
|
||||
|
||||
for _, s := range vector {
|
||||
m.sampleAppender.Append(&clientmodel.Sample{
|
||||
Metric: s.Metric.Metric,
|
||||
Value: s.Value,
|
||||
Timestamp: s.Timestamp,
|
||||
})
|
||||
}
|
||||
}(rule)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// transferAlertState makes a copy of the state of alerting rules and returns a function
|
||||
// that restores them in the current state.
|
||||
func (m *Manager) transferAlertState() func() {
|
||||
|
||||
alertingRules := map[string]*AlertingRule{}
|
||||
for _, r := range m.rules {
|
||||
if ar, ok := r.(*AlertingRule); ok {
|
||||
alertingRules[ar.name] = ar
|
||||
}
|
||||
}
|
||||
|
||||
return func() {
|
||||
// Restore alerting rule state.
|
||||
for _, r := range m.rules {
|
||||
ar, ok := r.(*AlertingRule)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if old, ok := alertingRules[ar.name]; ok {
|
||||
ar.activeAlerts = old.activeAlerts
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyConfig updates the rule manager's state as the config requires. If
|
||||
// loading the new rules failed the old rule set is restored. Returns true on success.
|
||||
func (m *Manager) ApplyConfig(conf *config.Config) bool {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
defer m.transferAlertState()()
|
||||
|
||||
success := true
|
||||
m.interval = time.Duration(conf.GlobalConfig.EvaluationInterval)
|
||||
|
||||
rulesSnapshot := make([]Rule, len(m.rules))
|
||||
copy(rulesSnapshot, m.rules)
|
||||
m.rules = m.rules[:0]
|
||||
|
||||
var files []string
|
||||
for _, pat := range conf.RuleFiles {
|
||||
fs, err := filepath.Glob(pat)
|
||||
if err != nil {
|
||||
// The only error can be a bad pattern.
|
||||
log.Errorf("Error retrieving rule files for %s: %s", pat, err)
|
||||
success = false
|
||||
}
|
||||
files = append(files, fs...)
|
||||
}
|
||||
if err := m.loadRuleFiles(files...); err != nil {
|
||||
// If loading the new rules failed, restore the old rule set.
|
||||
m.rules = rulesSnapshot
|
||||
log.Errorf("Error loading rules, previous rule set restored: %s", err)
|
||||
success = false
|
||||
}
|
||||
|
||||
return success
|
||||
}
|
||||
|
||||
// loadRuleFiles loads alerting and recording rules from the given files.
|
||||
func (m *Manager) loadRuleFiles(filenames ...string) error {
|
||||
for _, fn := range filenames {
|
||||
content, err := ioutil.ReadFile(fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stmts, err := promql.ParseStmts(string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error parsing %s: %s", fn, err)
|
||||
}
|
||||
|
||||
for _, stmt := range stmts {
|
||||
switch r := stmt.(type) {
|
||||
case *promql.AlertStmt:
|
||||
rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description, r.Runbook)
|
||||
m.rules = append(m.rules, rule)
|
||||
case *promql.RecordStmt:
|
||||
rule := NewRecordingRule(r.Name, r.Expr, r.Labels)
|
||||
m.rules = append(m.rules, rule)
|
||||
default:
|
||||
panic("retrieval.Manager.LoadRuleFiles: unknown statement type")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rules returns the list of the manager's rules.
|
||||
func (m *Manager) Rules() []Rule {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
rules := make([]Rule, len(m.rules))
|
||||
copy(rules, m.rules)
|
||||
return rules
|
||||
}
|
||||
|
||||
// AlertingRules returns the list of the manager's alerting rules.
|
||||
func (m *Manager) AlertingRules() []*AlertingRule {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
alerts := []*AlertingRule{}
|
||||
for _, rule := range m.rules {
|
||||
if alertingRule, ok := rule.(*AlertingRule); ok {
|
||||
alerts = append(alerts, alertingRule)
|
||||
}
|
||||
}
|
||||
return alerts
|
||||
}
|
||||
183
prom/rules/manager_test.go
Normal file
183
prom/rules/manager_test.go
Normal file
@ -0,0 +1,183 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
clientmodel "github.com/prometheus/client_golang/model"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
)
|
||||
|
||||
func TestAlertingRule(t *testing.T) {
|
||||
suite, err := promql.NewTest(t, `
|
||||
load 5m
|
||||
http_requests{job="api-server", instance="0", group="production"} 0+10x10
|
||||
http_requests{job="api-server", instance="1", group="production"} 0+20x10
|
||||
http_requests{job="api-server", instance="0", group="canary"} 0+30x10
|
||||
http_requests{job="api-server", instance="1", group="canary"} 0+40x10
|
||||
http_requests{job="app-server", instance="0", group="production"} 0+50x10
|
||||
http_requests{job="app-server", instance="1", group="production"} 0+60x10
|
||||
http_requests{job="app-server", instance="0", group="canary"} 0+70x10
|
||||
http_requests{job="app-server", instance="1", group="canary"} 0+80x10
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer suite.Close()
|
||||
|
||||
if err := suite.Run(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expr, err := promql.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to parse alert expression: %s", err)
|
||||
}
|
||||
|
||||
rule := NewAlertingRule(
|
||||
"HTTPRequestRateLow",
|
||||
expr,
|
||||
time.Minute,
|
||||
clientmodel.LabelSet{"severity": "critical"},
|
||||
"summary", "description", "runbook",
|
||||
)
|
||||
|
||||
var tests = []struct {
|
||||
time time.Duration
|
||||
result []string
|
||||
}{
|
||||
{
|
||||
time: 0,
|
||||
result: []string{
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
|
||||
},
|
||||
}, {
|
||||
time: 5 * time.Minute,
|
||||
result: []string{
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
|
||||
},
|
||||
}, {
|
||||
time: 10 * time.Minute,
|
||||
result: []string{
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
|
||||
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
|
||||
},
|
||||
},
|
||||
{
|
||||
time: 15 * time.Minute,
|
||||
result: nil,
|
||||
},
|
||||
{
|
||||
time: 20 * time.Minute,
|
||||
result: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
evalTime := clientmodel.Timestamp(0).Add(test.time)
|
||||
|
||||
res, err := rule.eval(evalTime, suite.QueryEngine())
|
||||
if err != nil {
|
||||
t.Fatalf("Error during alerting rule evaluation: %s", err)
|
||||
}
|
||||
|
||||
actual := strings.Split(res.String(), "\n")
|
||||
expected := annotateWithTime(test.result, evalTime)
|
||||
if actual[0] == "" {
|
||||
actual = []string{}
|
||||
}
|
||||
|
||||
if len(actual) != len(expected) {
|
||||
t.Errorf("%d. Number of samples in expected and actual output don't match (%d vs. %d)", i, len(expected), len(actual))
|
||||
}
|
||||
|
||||
for j, expectedSample := range expected {
|
||||
found := false
|
||||
for _, actualSample := range actual {
|
||||
if actualSample == expectedSample {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%d.%d. Couldn't find expected sample in output: '%v'", i, j, expectedSample)
|
||||
}
|
||||
}
|
||||
|
||||
if t.Failed() {
|
||||
t.Errorf("%d. Expected and actual outputs don't match:", i)
|
||||
t.Fatalf("Expected:\n%v\n----\nActual:\n%v", strings.Join(expected, "\n"), strings.Join(actual, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func annotateWithTime(lines []string, timestamp clientmodel.Timestamp) []string {
|
||||
annotatedLines := []string{}
|
||||
for _, line := range lines {
|
||||
annotatedLines = append(annotatedLines, fmt.Sprintf(line, timestamp))
|
||||
}
|
||||
return annotatedLines
|
||||
}
|
||||
|
||||
func TestTransferAlertState(t *testing.T) {
|
||||
m := NewManager(&ManagerOptions{})
|
||||
|
||||
alert := &Alert{
|
||||
Name: "testalert",
|
||||
State: StateFiring,
|
||||
}
|
||||
|
||||
arule := AlertingRule{
|
||||
name: "test",
|
||||
activeAlerts: map[clientmodel.Fingerprint]*Alert{},
|
||||
}
|
||||
aruleCopy := arule
|
||||
|
||||
m.rules = append(m.rules, &arule)
|
||||
|
||||
// Set an alert.
|
||||
arule.activeAlerts[0] = alert
|
||||
|
||||
// Save state and get the restore function.
|
||||
restore := m.transferAlertState()
|
||||
|
||||
// Remove arule from the rule list and add an unrelated rule and the
|
||||
// stateless copy of arule.
|
||||
m.rules = []Rule{
|
||||
&AlertingRule{
|
||||
name: "test_other",
|
||||
activeAlerts: map[clientmodel.Fingerprint]*Alert{},
|
||||
},
|
||||
&aruleCopy,
|
||||
}
|
||||
|
||||
// Apply the restore function.
|
||||
restore()
|
||||
|
||||
if ar := m.rules[0].(*AlertingRule); len(ar.activeAlerts) != 0 {
|
||||
t.Fatalf("unexpected alert for unrelated alerting rule")
|
||||
}
|
||||
if ar := m.rules[1].(*AlertingRule); !reflect.DeepEqual(ar.activeAlerts[0], alert) {
|
||||
t.Fatalf("alert state was not restored")
|
||||
}
|
||||
}
|
||||
85
prom/rules/recording.go
Normal file
85
prom/rules/recording.go
Normal file
@ -0,0 +1,85 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
|
||||
clientmodel "github.com/prometheus/client_golang/model"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
|
||||
// A RecordingRule records its vector expression into new timeseries.
|
||||
type RecordingRule struct {
|
||||
name string
|
||||
vector promql.Expr
|
||||
labels clientmodel.LabelSet
|
||||
}
|
||||
|
||||
// NewRecordingRule returns a new recording rule.
|
||||
func NewRecordingRule(name string, vector promql.Expr, labels clientmodel.LabelSet) *RecordingRule {
|
||||
return &RecordingRule{
|
||||
name: name,
|
||||
vector: vector,
|
||||
labels: labels,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the rule name.
|
||||
func (rule RecordingRule) Name() string { return rule.name }
|
||||
|
||||
// eval evaluates the rule and then overrides the metric names and labels accordingly.
|
||||
func (rule RecordingRule) eval(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
|
||||
query, err := engine.NewInstantQuery(rule.vector.String(), timestamp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vector, err := query.Exec().Vector()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Override the metric name and labels.
|
||||
for _, sample := range vector {
|
||||
sample.Metric.Set(clientmodel.MetricNameLabel, clientmodel.LabelValue(rule.name))
|
||||
for label, value := range rule.labels {
|
||||
if value == "" {
|
||||
sample.Metric.Delete(label)
|
||||
} else {
|
||||
sample.Metric.Set(label, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return vector, nil
|
||||
}
|
||||
|
||||
func (rule RecordingRule) String() string {
|
||||
return fmt.Sprintf("%s%s = %s\n", rule.name, rule.labels, rule.vector)
|
||||
}
|
||||
|
||||
// HTMLSnippet returns an HTML snippet representing this rule.
|
||||
func (rule RecordingRule) HTMLSnippet(pathPrefix string) template.HTML {
|
||||
ruleExpr := rule.vector.String()
|
||||
return template.HTML(fmt.Sprintf(
|
||||
`<a href="%s">%s</a>%s = <a href="%s">%s</a>`,
|
||||
pathPrefix+strutil.GraphLinkForExpression(rule.name),
|
||||
rule.name,
|
||||
rule.labels,
|
||||
pathPrefix+strutil.GraphLinkForExpression(ruleExpr),
|
||||
ruleExpr))
|
||||
}
|
||||
Reference in New Issue
Block a user