This commit is contained in:
Alexander Zhebrak 2017-08-04 13:16:44 +03:00
commit c92b168ae0
3 changed files with 125 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
nvidia_smi_exporter
sample.csv

57
README.md Normal file
View File

@ -0,0 +1,57 @@
# nvidia_smi_exporter
nvidia-smi metrics exporter for Prometheus
## Build
```
> go build -v nvidia_smi_exporter
```
## Run
```
> ./nvidia_smi_exporter [<port>]
```
Default port is 9101
### localhost:9101/metrics
```
temperature_gpu{gpu="TITAN X (Pascal)[0]"} 41
utilization_gpu{gpu="TITAN X (Pascal)[0]"} 0
utilization_memory{gpu="TITAN X (Pascal)[0]"} 0
memory_total{gpu="TITAN X (Pascal)[0]"} 12189
memory_free{gpu="TITAN X (Pascal)[0]"} 12189
memory_used{gpu="TITAN X (Pascal)[0]"} 0
temperature_gpu{gpu="TITAN X (Pascal)[1]"} 78
utilization_gpu{gpu="TITAN X (Pascal)[1]"} 95
utilization_memory{gpu="TITAN X (Pascal)[1]"} 59
memory_total{gpu="TITAN X (Pascal)[1]"} 12189
memory_free{gpu="TITAN X (Pascal)[1]"} 1738
memory_used{gpu="TITAN X (Pascal)[1]"} 10451
temperature_gpu{gpu="TITAN X (Pascal)[2]"} 83
utilization_gpu{gpu="TITAN X (Pascal)[2]"} 99
utilization_memory{gpu="TITAN X (Pascal)[2]"} 82
memory_total{gpu="TITAN X (Pascal)[2]"} 12189
memory_free{gpu="TITAN X (Pascal)[2]"} 190
memory_used{gpu="TITAN X (Pascal)[2]"} 11999
temperature_gpu{gpu="TITAN X (Pascal)[3]"} 84
utilization_gpu{gpu="TITAN X (Pascal)[3]"} 97
utilization_memory{gpu="TITAN X (Pascal)[3]"} 76
memory_total{gpu="TITAN X (Pascal)[3]"} 12189
memory_free{gpu="TITAN X (Pascal)[3]"} 536
memory_used{gpu="TITAN X (Pascal)[3]"} 11653
```
### Exact command
```
nvidia-smi --query-gpu=name,index,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv,noheader,nounits
```
### Prometheus example config
```
- job_name: "gpu_exporter"
static_configs:
- targets: ['localhost:9101']
```

66
nvidia_smi_exporter.go Normal file
View File

@ -0,0 +1,66 @@
package main
import (
"bytes"
"encoding/csv"
"fmt"
"net/http"
"log"
"os"
"os/exec"
"strings"
)
// name, index, temperature.gpu, utilization.gpu,
// utilization.memory, memory.total, memory.free, memory.used
func metrics(response http.ResponseWriter, request *http.Request) {
out, err := exec.Command(
"nvidia-smi",
"--query-gpu=name,index,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used",
"--format=csv,noheader,nounits").Output()
if err != nil {
fmt.Printf("%s\n", err)
return
}
csvReader := csv.NewReader(bytes.NewReader(out))
csvReader.TrimLeadingSpace = true
records, err := csvReader.ReadAll()
if err != nil {
fmt.Printf("%s\n", err)
return
}
metricList := []string {
"temperature.gpu", "utilization.gpu",
"utilization.memory", "memory.total", "memory.free", "memory.used"}
result := ""
for _, row := range records {
name := fmt.Sprintf("%s[%s]", row[0], row[1])
for idx, value := range row[2:] {
result = fmt.Sprintf(
"%s%s{gpu=\"%s\"} %s\n", result,
metricList[idx], name, value)
}
}
fmt.Fprintf(response, strings.Replace(result, ".", "_", -1))
}
func main() {
port := ":9101"
if len(os.Args) > 1 {
port = ":" + os.Args[1]
}
http.HandleFunc("/metrics/", metrics)
err := http.ListenAndServe(port, nil)
if err != nil {
log.Fatal("ListenAndServe: ", err)
}
}