mirror of
https://github.com/grafana/grafana.git
synced 2026-01-14 13:21:26 +00:00
Compare commits
246 Commits
gabor/no-p
...
v11.4.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
da7be26d34 | ||
|
|
350d9c71ce | ||
|
|
a7ea2a3b73 | ||
|
|
690a223f6b | ||
|
|
8dd4637706 | ||
|
|
0fc69d16e9 | ||
|
|
f433bc770f | ||
|
|
d4331c9d0b | ||
|
|
074a1cf348 | ||
|
|
9aa8526e7d | ||
|
|
122642a6d7 | ||
|
|
805ab7bd8a | ||
|
|
86bb6dd4b1 | ||
|
|
7b5235f06d | ||
|
|
f83e0a8a39 | ||
|
|
0c7f575149 | ||
|
|
696d22a3fe | ||
|
|
fc1d9c1719 | ||
|
|
de310cb88c | ||
|
|
227c8392e0 | ||
|
|
dc2c257c1a | ||
|
|
97517f41cf | ||
|
|
a13443ad11 | ||
|
|
2d06a936c3 | ||
|
|
860a25f7fd | ||
|
|
ede278c767 | ||
|
|
dab5a49df0 | ||
|
|
49ea20d027 | ||
|
|
ed927a143c | ||
|
|
4e072ffe50 | ||
|
|
07841886b6 | ||
|
|
1547a09645 | ||
|
|
ea930b2a51 | ||
|
|
2c408c8139 | ||
|
|
4a6f5ebe23 | ||
|
|
86c21cee85 | ||
|
|
0573e4d9a9 | ||
|
|
6a08b1cdc2 | ||
|
|
b12cab40d1 | ||
|
|
9b1414e2ef | ||
|
|
96060e359f | ||
|
|
cd060b0791 | ||
|
|
f2c2883a0f | ||
|
|
b7fff35a8a | ||
|
|
122c417f8c | ||
|
|
fb86150321 | ||
|
|
e0dd77c05a | ||
|
|
a69e5fb0a2 | ||
|
|
e1ba2dd355 | ||
|
|
4c288f39ee | ||
|
|
d9045acddb | ||
|
|
c705a3eb36 | ||
|
|
03e84aca2b | ||
|
|
20f9416142 | ||
|
|
7e1f67f9d7 | ||
|
|
66a7e6a60a | ||
|
|
620534891d | ||
|
|
de338282c1 | ||
|
|
72acd7f000 | ||
|
|
0e6b320f4d | ||
|
|
bed84aefdf | ||
|
|
cb0f9b4093 | ||
|
|
9bfc35ec25 | ||
|
|
e076682a17 | ||
|
|
01a0baf5c1 | ||
|
|
8a0b736610 | ||
|
|
006d2b2b72 | ||
|
|
df43a3fff4 | ||
|
|
93c507cc8f | ||
|
|
626c15c1a5 | ||
|
|
ac481509c7 | ||
|
|
e4a7eb6b69 | ||
|
|
c9859478ee | ||
|
|
0b732db623 | ||
|
|
88aa744512 | ||
|
|
443dcc026d | ||
|
|
b58701869e | ||
|
|
b12308499b | ||
|
|
80691e4297 | ||
|
|
8bc94106e2 | ||
|
|
54ff80afb1 | ||
|
|
780e191177 | ||
|
|
e80e6e532c | ||
|
|
17d7ca7232 | ||
|
|
bc3fd9d808 | ||
|
|
1de9f9477f | ||
|
|
4428bf7025 | ||
|
|
b35522672b | ||
|
|
c8031ed338 | ||
|
|
9c606f2302 | ||
|
|
8395bda60f | ||
|
|
f8dfaebf4c | ||
|
|
1c201629d5 | ||
|
|
37858e7e24 | ||
|
|
60370b4ea6 | ||
|
|
70c37c9b03 | ||
|
|
55cb58b8be | ||
|
|
1695d2716f | ||
|
|
9ab6d23f77 | ||
|
|
13e8921d27 | ||
|
|
d0e5c87992 | ||
|
|
4cb555fa8b | ||
|
|
98be4ac047 | ||
|
|
9c7c333856 | ||
|
|
d15321179e | ||
|
|
f0e56bdf64 | ||
|
|
bc89fe04b9 | ||
|
|
e3367ccf9a | ||
|
|
dfb1f9a3c1 | ||
|
|
f5cf121eb5 | ||
|
|
64b556c137 | ||
|
|
8d6bbb7a4b | ||
|
|
08e10ff665 | ||
|
|
13d60c9bd4 | ||
|
|
96d1d5a07b | ||
|
|
c97b64e9ff | ||
|
|
54eca5c73f | ||
|
|
4b634e887d | ||
|
|
9225f4a1cb | ||
|
|
a09dcda92d | ||
|
|
a6bc76df32 | ||
|
|
d5cbf07090 | ||
|
|
d32c02ae14 | ||
|
|
d815e1107d | ||
|
|
700f59a0ea | ||
|
|
a6cdb48249 | ||
|
|
ecf127c0fa | ||
|
|
9241bc5f5c | ||
|
|
a2b1cda210 | ||
|
|
50762226a0 | ||
|
|
58333e0507 | ||
|
|
5cc78ea219 | ||
|
|
a8cb62f05a | ||
|
|
89ea2a15ce | ||
|
|
5e1479d887 | ||
|
|
f1c4e83865 | ||
|
|
cfa3ef73a1 | ||
|
|
c99c8c56e2 | ||
|
|
5b1a386dac | ||
|
|
8bc0014845 | ||
|
|
8f10dbe12d | ||
|
|
546e0e214b | ||
|
|
3704073e6f | ||
|
|
c9c1bbfa6e | ||
|
|
402b023b45 | ||
|
|
be895603c8 | ||
|
|
c2d85230b1 | ||
|
|
3824dc412d | ||
|
|
0e0a2c0371 | ||
|
|
8273628550 | ||
|
|
d489ecf794 | ||
|
|
16411b1b4a | ||
|
|
8567089ed2 | ||
|
|
1100a0409f | ||
|
|
a7c1f37435 | ||
|
|
cbf9e323b6 | ||
|
|
0826aa0078 | ||
|
|
9bb7d77cdb | ||
|
|
e31c4fc59b | ||
|
|
28f9eb3221 | ||
|
|
693ed6dddd | ||
|
|
8f030aa210 | ||
|
|
ef12cfe67d | ||
|
|
4f80c4c55c | ||
|
|
66518db34c | ||
|
|
855dcd7b7b | ||
|
|
ab85eacea6 | ||
|
|
11aedea347 | ||
|
|
fdc1859d0d | ||
|
|
1540c767cc | ||
|
|
7f36355451 | ||
|
|
397c96c1cc | ||
|
|
13336dd198 | ||
|
|
cb5df488a8 | ||
|
|
5af40ed44f | ||
|
|
c081bb53d0 | ||
|
|
8ea5af7335 | ||
|
|
d763aae5f2 | ||
|
|
4a30c85b77 | ||
|
|
a78607432e | ||
|
|
12a54c2414 | ||
|
|
bb4e253c76 | ||
|
|
90a562257c | ||
|
|
5889d680e9 | ||
|
|
86deb2b4c4 | ||
|
|
c5d7a954ef | ||
|
|
54e5e6c1b5 | ||
|
|
c99956285b | ||
|
|
9d867a4441 | ||
|
|
a9088a7bbb | ||
|
|
7cf08ebaf9 | ||
|
|
c76d97bb13 | ||
|
|
f8cb0ccb71 | ||
|
|
f702577e83 | ||
|
|
b8c43bcf3d | ||
|
|
d865fa64ce | ||
|
|
a292f838fe | ||
|
|
50c5d806c1 | ||
|
|
6d5fa7b7a0 | ||
|
|
035f6dde73 | ||
|
|
494ddad79d | ||
|
|
2e81653b68 | ||
|
|
6924b9641a | ||
|
|
f68938c842 | ||
|
|
c426f12df7 | ||
|
|
bbc76071b5 | ||
|
|
ca7156d411 | ||
|
|
8e7ba6f123 | ||
|
|
6e0e8d0370 | ||
|
|
073436f2f8 | ||
|
|
367c82bd51 | ||
|
|
a2db7105a9 | ||
|
|
bdf41feda9 | ||
|
|
aa8c58abd0 | ||
|
|
48d083d495 | ||
|
|
e0da40ea8c | ||
|
|
aba5388a42 | ||
|
|
48ce1636bd | ||
|
|
71e980ee9a | ||
|
|
060846d056 | ||
|
|
df57ba1b70 | ||
|
|
4a800ace30 | ||
|
|
a577031ba9 | ||
|
|
efeb05386b | ||
|
|
1ac972f086 | ||
|
|
4590698e10 | ||
|
|
ab368bfb06 | ||
|
|
489c9db173 | ||
|
|
8bbf0a1869 | ||
|
|
8b8a67fad6 | ||
|
|
2e956696a4 | ||
|
|
d58468b60b | ||
|
|
d784d370d3 | ||
|
|
d4beddfb0e | ||
|
|
ae3419cd79 | ||
|
|
65c759c76a | ||
|
|
e6475b80c2 | ||
|
|
749714fe50 | ||
|
|
e1a4fcda48 | ||
|
|
9af2af8b9f | ||
|
|
50de92f873 | ||
|
|
2a7d8279a2 | ||
|
|
f5d29152c9 | ||
|
|
c938b751cf | ||
|
|
f24f59225d | ||
|
|
5f7e46f886 |
@@ -1697,9 +1697,6 @@ exports[`better eslint`] = {
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "10"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "11"]
|
||||
],
|
||||
"public/app/features/alerting/unified/components/expressions/ExpressionStatusIndicator.tsx:5381": [
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"]
|
||||
],
|
||||
"public/app/features/alerting/unified/components/extensions/AlertInstanceExtensionPointMenu.tsx:5381": [
|
||||
[0, 0, 0, "Do not re-export imported variable (\`app/features/explore/extensions/ToolbarExtensionPointMenu\`)", "0"]
|
||||
],
|
||||
@@ -1935,8 +1932,7 @@ exports[`better eslint`] = {
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "10"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "11"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "12"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "13"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "14"]
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "13"]
|
||||
],
|
||||
"public/app/features/alerting/unified/components/rule-editor/NeedHelpInfo.tsx:5381": [
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"]
|
||||
@@ -2035,16 +2031,13 @@ exports[`better eslint`] = {
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "1"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "2"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "3"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "4"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "5"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "6"]
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "4"]
|
||||
],
|
||||
"public/app/features/alerting/unified/components/rule-editor/labels/LabelsFieldInForm.tsx:5381": [
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "1"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "2"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "3"],
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "4"]
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "3"]
|
||||
],
|
||||
"public/app/features/alerting/unified/components/rule-editor/notificaton-preview/NotificationPolicyMatchers.tsx:5381": [
|
||||
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"],
|
||||
@@ -5686,9 +5679,6 @@ exports[`better eslint`] = {
|
||||
"public/app/plugins/datasource/cloudwatch/components/ConfigEditor/XrayLinkConfig.tsx:5381": [
|
||||
[0, 0, 0, "Styles should be written using objects.", "0"]
|
||||
],
|
||||
"public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx:5381": [
|
||||
[0, 0, 0, "Styles should be written using objects.", "0"]
|
||||
],
|
||||
"public/app/plugins/datasource/cloudwatch/components/QueryEditor/MetricsQueryEditor/DynamicLabelsField.tsx:5381": [
|
||||
[0, 0, 0, "Styles should be written using objects.", "0"]
|
||||
],
|
||||
@@ -7399,14 +7389,6 @@ exports[`no gf-form usage`] = {
|
||||
"public/app/plugins/datasource/cloudwatch/components/ConfigEditor/XrayLinkConfig.tsx:5381": [
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"]
|
||||
],
|
||||
"public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx:5381": [
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"]
|
||||
],
|
||||
"public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryField.tsx:5381": [
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"],
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"],
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"]
|
||||
],
|
||||
"public/app/plugins/datasource/cloudwatch/components/shared/LogGroups/LegacyLogGroupNamesSelection.tsx:5381": [
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"],
|
||||
[0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"]
|
||||
|
||||
13
.drone.star
13
.drone.star
@@ -18,18 +18,10 @@ load(
|
||||
"publish_packages_pipeline",
|
||||
)
|
||||
load("scripts/drone/events/rrc-patch.star", "rrc_patch_pipelines")
|
||||
load(
|
||||
"scripts/drone/pipelines/ci_images.star",
|
||||
"publish_ci_windows_test_image_pipeline",
|
||||
)
|
||||
load(
|
||||
"scripts/drone/pipelines/publish_images.star",
|
||||
"publish_image_pipelines_public",
|
||||
)
|
||||
load(
|
||||
"scripts/drone/pipelines/windows.star",
|
||||
"windows_test_backend",
|
||||
)
|
||||
load(
|
||||
"scripts/drone/rgm.star",
|
||||
"rgm",
|
||||
@@ -46,12 +38,7 @@ def main(_ctx):
|
||||
publish_npm_pipelines() +
|
||||
publish_packages_pipeline() +
|
||||
rgm() +
|
||||
[windows_test_backend({
|
||||
"event": ["promote"],
|
||||
"target": ["test-windows"],
|
||||
}, "oss", "testing")] +
|
||||
integration_test_pipelines() +
|
||||
publish_ci_windows_test_image_pipeline() +
|
||||
cronjobs() +
|
||||
secrets()
|
||||
)
|
||||
|
||||
1520
.drone.yml
1520
.drone.yml
File diff suppressed because it is too large
Load Diff
8
.github/CODEOWNERS
vendored
8
.github/CODEOWNERS
vendored
@@ -38,17 +38,11 @@
|
||||
/docs/.codespellignore @grafana/docs-tooling
|
||||
/docs/sources/ @irenerl24
|
||||
|
||||
/docs/sources/administration/ @jdbaldry
|
||||
/docs/sources/alerting/ @brendamuir
|
||||
/docs/sources/dashboards/ @imatwawana
|
||||
/docs/sources/datasources/ @jdbaldry
|
||||
/docs/sources/explore/ @grafana/explore-squad @lwandz13
|
||||
/docs/sources/fundamentals @irenerl24
|
||||
/docs/sources/getting-started/ @irenerl24
|
||||
/docs/sources/introduction/ @irenerl24
|
||||
/docs/sources/panels-visualizations/ @imatwawana
|
||||
/docs/sources/release-notes/ @Eve832 @GrafanaWriter
|
||||
/docs/sources/setup-grafana/ @irenerl24
|
||||
/docs/sources/release-notes/ @irenerl24 @GrafanaWriter
|
||||
/docs/sources/upgrade-guide/ @imatwawana
|
||||
/docs/sources/whatsnew/ @imatwawana
|
||||
|
||||
|
||||
21
.github/workflows/doc-validator.yml
vendored
21
.github/workflows/doc-validator.yml
vendored
@@ -1,13 +1,18 @@
|
||||
name: "doc-validator"
|
||||
on:
|
||||
pull_request:
|
||||
paths: ["docs/sources/**"]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
include:
|
||||
description: |
|
||||
Regular expression that matches paths to include in linting.
|
||||
|
||||
For example: docs/sources/(?:alerting|fundamentals)/.+\.md
|
||||
required: true
|
||||
jobs:
|
||||
doc-validator:
|
||||
runs-on: "ubuntu-latest"
|
||||
container:
|
||||
image: "grafana/doc-validator:v5.0.0"
|
||||
image: "grafana/doc-validator:v5.2.0"
|
||||
steps:
|
||||
- name: "Checkout code"
|
||||
uses: "actions/checkout@v4"
|
||||
@@ -15,15 +20,7 @@ jobs:
|
||||
# Only run doc-validator on specific directories.
|
||||
run: >
|
||||
doc-validator
|
||||
'--include=^docs/sources/(?:alerting|fundamentals|getting-started|introduction|setup-grafana|upgrade-guide|whatsnew/whats-new-in-v(?:9|10))/.+\.md$'
|
||||
'--include=${{ inputs.include }}'
|
||||
'--skip-checks=^(?:image.+|canonical-does-not-match-pretty-URL)$'
|
||||
./docs/sources
|
||||
/docs/grafana/latest
|
||||
| reviewdog
|
||||
-f=rdjsonl
|
||||
--fail-on-error
|
||||
--filter-mode=nofilter
|
||||
--name=doc-validator
|
||||
--reporter=github-pr-review
|
||||
env:
|
||||
REVIEWDOG_GITHUB_API_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
3
.github/workflows/github-release.yml
vendored
3
.github/workflows/github-release.yml
vendored
@@ -8,7 +8,7 @@ on:
|
||||
type: string
|
||||
latest:
|
||||
required: false
|
||||
default: false
|
||||
default: "0"
|
||||
description: Mark this release as latest (`1`) or not (`0`, default)
|
||||
type: string
|
||||
dry_run:
|
||||
@@ -23,6 +23,7 @@ on:
|
||||
type: string
|
||||
latest:
|
||||
required: false
|
||||
default: "0"
|
||||
description: Mark this release as latest (`1`) or not (`0`, default)
|
||||
type: string
|
||||
dry_run:
|
||||
|
||||
@@ -20,10 +20,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: grafana/writers-toolkit/publish-technical-documentation-release@publish-technical-documentation-release/v1
|
||||
- uses: grafana/writers-toolkit/publish-technical-documentation-release@publish-technical-documentation-release/v2
|
||||
with:
|
||||
release_tag_regexp: "^v(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$"
|
||||
release_branch_regexp: "^v(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.x$"
|
||||
release_branch_with_patch_regexp: "^v(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$"
|
||||
release_tag_regexp: "^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
|
||||
release_branch_regexp: "^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.x$"
|
||||
release_branch_with_patch_regexp: "^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
|
||||
website_directory: content/docs/grafana
|
||||
version_suffix: ""
|
||||
|
||||
8
.github/workflows/release-comms.yml
vendored
8
.github/workflows/release-comms.yml
vendored
@@ -8,10 +8,11 @@ on:
|
||||
dry_run:
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
version:
|
||||
required: true
|
||||
latest:
|
||||
type: bool
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types:
|
||||
@@ -30,17 +31,18 @@ jobs:
|
||||
latest: ${{ steps.output.outputs.latest }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# The github-release action expects a `LATEST` value of a string of either '1' or '0'
|
||||
- if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
run: |
|
||||
echo setting up GITHUB_ENV for ${{ github.event_name }}
|
||||
echo "VERSION=${{ inputs.version }}" >> $GITHUB_ENV
|
||||
echo "DRY_RUN=${{ inputs.dry_run }}" >> $GITHUB_ENV
|
||||
echo "LATEST=${{ inputs.latest }}" >> $GITHUB_ENV
|
||||
echo "LATEST=${{ inputs.latest && '1' || '0' }}" >> $GITHUB_ENV
|
||||
- if: ${{ github.event.pull_request.merged == true && startsWith(github.head_ref, 'release/') }}
|
||||
run: |
|
||||
echo "VERSION=$(echo ${{ github.head_ref }} | sed -e 's/release\/.*\///g')" >> $GITHUB_ENV
|
||||
echo "DRY_RUN=${{ contains(github.event.pull_request.labels.*.name, 'release/dry-run') }}" >> $GITHUB_ENV
|
||||
echo "LATEST=${{ contains(github.event.pull_request.labels.*.name, 'release/latest') }}" >> $GITHUB_ENV
|
||||
echo "LATEST=${{ contains(github.event.pull_request.labels.*.name, 'release/latest') && '1' || '0' }}" >> $GITHUB_ENV
|
||||
- id: output
|
||||
run: |
|
||||
echo "dry_run: $DRY_RUN"
|
||||
|
||||
93
CHANGELOG.md
93
CHANGELOG.md
@@ -1,3 +1,96 @@
|
||||
<!-- 11.4.0 START -->
|
||||
|
||||
# 11.4.0 (2024-12-05)
|
||||
|
||||
### Features and enhancements
|
||||
|
||||
- **Cloudwatch:** OpenSearch PPL and SQL support in Logs Insights
|
||||
|
||||
<!-- 11.4.0 END -->
|
||||
<!-- 11.3.1 START -->
|
||||
|
||||
# 11.3.1 (2024-11-19)
|
||||
|
||||
### Features and enhancements
|
||||
|
||||
- **Alerting:** Make context deadline on AlertNG service startup configurable [#96135](https://github.com/grafana/grafana/pull/96135), [@fayzal-g](https://github.com/fayzal-g)
|
||||
- **MigrationAssistant:** Restrict dashboards, folders and datasources by the org id of the signed in user [#96345](https://github.com/grafana/grafana/pull/96345), [@leandro-deveikis](https://github.com/leandro-deveikis)
|
||||
- **User:** Check SignedInUser OrgID in RevokeInvite [#95490](https://github.com/grafana/grafana/pull/95490), [@mgyongyosi](https://github.com/mgyongyosi)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- **Alerting:** Fix escaping of silence matchers in utf8 mode [#95347](https://github.com/grafana/grafana/pull/95347), [@tomratcliffe](https://github.com/tomratcliffe)
|
||||
- **Alerting:** Fix overflow for long receiver names [#95133](https://github.com/grafana/grafana/pull/95133), [@gillesdemey](https://github.com/gillesdemey)
|
||||
- **Alerting:** Fix saving advanced mode toggle state in the alert rule editor [#95981](https://github.com/grafana/grafana/pull/95981), [@alexander-akhmetov](https://github.com/alexander-akhmetov)
|
||||
- **Alerting:** Fix setting datasource uid, when datasource is string in old version [#96273](https://github.com/grafana/grafana/pull/96273), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Alerting:** Force refetch prom rules when refreshing panel [#96125](https://github.com/grafana/grafana/pull/96125), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Anonymous User:** Adds validator service for anonymous users [#94994](https://github.com/grafana/grafana/pull/94994), [@leandro-deveikis](https://github.com/leandro-deveikis)
|
||||
- **Anonymous User:** Adds validator service for anonymous users (Enterprise)
|
||||
- **Azure Monitor:** Support metric namespaces fallback [#95155](https://github.com/grafana/grafana/pull/95155), [@aangelisc](https://github.com/aangelisc)
|
||||
- **Azure:** Fix duplicated traces in multi-resource trace query [#95247](https://github.com/grafana/grafana/pull/95247), [@aangelisc](https://github.com/aangelisc)
|
||||
- **Azure:** Handle namespace request rejection [#95909](https://github.com/grafana/grafana/pull/95909), [@aangelisc](https://github.com/aangelisc)
|
||||
- **CloudWatch:** Interpolate region in log context query [#94990](https://github.com/grafana/grafana/pull/94990), [@iwysiu](https://github.com/iwysiu)
|
||||
- **Dashboard datasource:** Return annotations as series when query topic is "annotations" [#95971](https://github.com/grafana/grafana/pull/95971), [@kaydelaney](https://github.com/kaydelaney)
|
||||
- **Dashboard:** Append orgId to URL [#95963](https://github.com/grafana/grafana/pull/95963), [@bfmatei](https://github.com/bfmatei)
|
||||
- **Dashboards:** Fixes performance issue expanding a row [#95321](https://github.com/grafana/grafana/pull/95321), [@torkelo](https://github.com/torkelo)
|
||||
- **Flame Graph:** Fix crash when it receives empty data [#96211](https://github.com/grafana/grafana/pull/96211), [@yincongcyincong](https://github.com/yincongcyincong)
|
||||
- **Folders:** Add admin permissions upon creation of a folder w. SA [#95365](https://github.com/grafana/grafana/pull/95365), [@eleijonmarck](https://github.com/eleijonmarck)
|
||||
- **Folders:** Don't show error pop-up if the user can't fetch the root folder [#95600](https://github.com/grafana/grafana/pull/95600), [@IevaVasiljeva](https://github.com/IevaVasiljeva)
|
||||
- **Migration:** Remove table aliasing in delete statement to make it work for mariadb [#95232](https://github.com/grafana/grafana/pull/95232), [@kalleep](https://github.com/kalleep)
|
||||
- **ServerLock:** Fix pg concurrency/locking issue [#95935](https://github.com/grafana/grafana/pull/95935), [@mgyongyosi](https://github.com/mgyongyosi)
|
||||
- **Service Accounts:** Run service account creation in transaction [#94803](https://github.com/grafana/grafana/pull/94803), [@IevaVasiljeva](https://github.com/IevaVasiljeva)
|
||||
- **Table:** Fix text wrapping applying to wrong field [#95425](https://github.com/grafana/grafana/pull/95425), [@codeincarnate](https://github.com/codeincarnate)
|
||||
- **Unified Storage:** Use ssl_mode instead of sslmode [#95662](https://github.com/grafana/grafana/pull/95662), [@chaudyg](https://github.com/chaudyg)
|
||||
|
||||
<!-- 11.3.1 END -->
|
||||
<!-- 11.3.0+security-01 START -->
|
||||
|
||||
# 11.3.0+security-01 (2024-11-12)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- **MigrationAssistant:** Fix Migration Assistant issue [CVE-2024-9476]
|
||||
|
||||
<!-- 11.3.0+security-01 END -->
|
||||
<!-- 11.3.0 START -->
|
||||
|
||||
# 11.3.0 (2024-10-22)
|
||||
|
||||
### Features and enhancements
|
||||
|
||||
- **Alerting:** Add manage permissions UI logic for Contact Points [#92885](https://github.com/grafana/grafana/pull/92885), [@tomratcliffe](https://github.com/tomratcliffe)
|
||||
- **Alerting:** Allow linking to silence form with `__alert_rule_uid__` value preset [#93526](https://github.com/grafana/grafana/pull/93526), [@tomratcliffe](https://github.com/tomratcliffe)
|
||||
- **Alerting:** Hide query name when using simplified mode in the alert rule [#93779](https://github.com/grafana/grafana/pull/93779), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Alerting:** Limit and clean up old alert rules versions [#89754](https://github.com/grafana/grafana/pull/89754), [@igloo12](https://github.com/igloo12)
|
||||
- **Alerting:** Style nits for the simple query mode [#93930](https://github.com/grafana/grafana/pull/93930), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Alerting:** Update texts in annotations step [#93977](https://github.com/grafana/grafana/pull/93977), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Alerting:** Use useProduceNewAlertmanagerConfiguration for contact points [#88456](https://github.com/grafana/grafana/pull/88456), [@gillesdemey](https://github.com/gillesdemey)
|
||||
- **Auth:** Attach external session info to Grafana session [#93849](https://github.com/grafana/grafana/pull/93849), [@mgyongyosi](https://github.com/mgyongyosi)
|
||||
- **Auth:** Replace jmespath/go-jmespath with jmespath-community/go-jmespath [#94203](https://github.com/grafana/grafana/pull/94203), [@mgyongyosi](https://github.com/mgyongyosi)
|
||||
- **CloudMigrations:** Add support for migration of Library Elements (Panels) resources [#93898](https://github.com/grafana/grafana/pull/93898), [@macabu](https://github.com/macabu)
|
||||
- **Cloudwatch:** Update grafana-aws-sdk [#94155](https://github.com/grafana/grafana/pull/94155), [@iwysiu](https://github.com/iwysiu)
|
||||
- **Explore Logs:** Preinstall for onprem Grafana instances [#94221](https://github.com/grafana/grafana/pull/94221), [@svennergr](https://github.com/svennergr)
|
||||
- **ExploreMetrics:** Ensure compatibility with Incremental Querying [#94355](https://github.com/grafana/grafana/pull/94355), [@NWRichmond](https://github.com/NWRichmond)
|
||||
- **FieldConfig:** Add support for Actions [#92874](https://github.com/grafana/grafana/pull/92874), [@adela-almasan](https://github.com/adela-almasan)
|
||||
- **Plugin Extensions:** Require meta-data to be defined in `plugin.json` during development mode [#93429](https://github.com/grafana/grafana/pull/93429), [@leventebalogh](https://github.com/leventebalogh)
|
||||
- **Reporting:** Display template variables in the PDF (Enterprise)
|
||||
- **Tempo:** Add deprecation notice for Aggregate By [#94050](https://github.com/grafana/grafana/pull/94050), [@joey-grafana](https://github.com/joey-grafana)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- **Alerting/Chore:** Fix TimeRangeInput not working across multiple months [#93622](https://github.com/grafana/grafana/pull/93622), [@tomratcliffe](https://github.com/tomratcliffe)
|
||||
- **Alerting:** Fix default value for input in simple condition [#94248](https://github.com/grafana/grafana/pull/94248), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron)
|
||||
- **Alerting:** Fix eval interval not being saved when creating a new group [#93821](https://github.com/grafana/grafana/pull/93821), [@tomratcliffe](https://github.com/tomratcliffe)
|
||||
- **Alerting:** Fix incorrect permission on POST external rule groups endpoint [CVE-2024-8118] [#93940](https://github.com/grafana/grafana/pull/93940), [@alexweav](https://github.com/alexweav)
|
||||
- **Alerting:** Fix panics when attempting to create an Alertmanager after failing [#94023](https://github.com/grafana/grafana/pull/94023), [@santihernandezc](https://github.com/santihernandezc)
|
||||
- **DashboardScene:** Fixes url issue with subpath when exiting edit mode [#93962](https://github.com/grafana/grafana/pull/93962), [@torkelo](https://github.com/torkelo)
|
||||
- **Dashboards:** Enable scenes by default [#93818](https://github.com/grafana/grafana/pull/93818), [@ivanortegaalba](https://github.com/ivanortegaalba)
|
||||
- **Dashboards:** Fixes view & edit keyboard shortcuts when grafana is behind a subpath [#93955](https://github.com/grafana/grafana/pull/93955), [@torkelo](https://github.com/torkelo)
|
||||
- **ElasticSearch:** Fix errorsource in newInstanceSettings [#93859](https://github.com/grafana/grafana/pull/93859), [@iwysiu](https://github.com/iwysiu)
|
||||
- **Reporting:** Fix reports on multi-org instance (Enterprise)
|
||||
- **SubMenu:** Fix expanding sub menu items on touch devices [#93208](https://github.com/grafana/grafana/pull/93208), [@yincongcyincong](https://github.com/yincongcyincong)
|
||||
|
||||
<!-- 11.3.0 END -->
|
||||
<!-- 11.2.2 START -->
|
||||
|
||||
# 11.2.2 (2024-10-01)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
ARG BASE_IMAGE=alpine:3.19.1
|
||||
ARG JS_IMAGE=node:20-alpine
|
||||
ARG JS_PLATFORM=linux/amd64
|
||||
ARG GO_IMAGE=golang:1.23.1-alpine
|
||||
ARG GO_IMAGE=golang:1.23.5-alpine
|
||||
|
||||
ARG GO_SRC=go-builder
|
||||
ARG JS_SRC=js-builder
|
||||
|
||||
2
Makefile
2
Makefile
@@ -8,7 +8,7 @@ WIRE_TAGS = "oss"
|
||||
include .bingo/Variables.mk
|
||||
|
||||
GO = go
|
||||
GO_VERSION = 1.23.1
|
||||
GO_VERSION = 1.23.5
|
||||
GO_LINT_FILES ?= $(shell ./scripts/go-workspace/golangci-lint-includes.sh)
|
||||
GO_TEST_FILES ?= $(shell ./scripts/go-workspace/test-includes.sh)
|
||||
SH_FILES ?= $(shell find ./scripts -name *.sh)
|
||||
|
||||
@@ -26,7 +26,7 @@ require (
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
golang.org/x/net v0.29.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -79,8 +79,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
|
||||
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
|
||||
@@ -1209,6 +1209,9 @@ enabled =
|
||||
# Comma-separated list of organization IDs for which to disable unified alerting. Only supported if unified alerting is enabled.
|
||||
disabled_orgs =
|
||||
|
||||
# Specify how long to wait for the alerting service to initialize
|
||||
initialization_timeout = 30s
|
||||
|
||||
# Specify the frequency of polling for admin config changes.
|
||||
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
|
||||
admin_config_poll_interval = 60s
|
||||
|
||||
@@ -1197,6 +1197,9 @@
|
||||
# Comma-separated list of organization IDs for which to disable unified alerting. Only supported if unified alerting is enabled.
|
||||
;disabled_orgs =
|
||||
|
||||
# Specify how long to wait for the alerting service to initialize
|
||||
;initialization_timeout = 30s
|
||||
|
||||
# Specify the frequency of polling for admin config changes.
|
||||
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
|
||||
;admin_config_poll_interval = 60s
|
||||
|
||||
@@ -15,6 +15,6 @@ require (
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
)
|
||||
|
||||
@@ -20,7 +20,7 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA=
|
||||
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
aks
|
||||
eror
|
||||
iam
|
||||
wan
|
||||
[Operato Windrose](https://grafana.com/grafana/plugins/operato-windrose-panel/)
|
||||
sergent
|
||||
@@ -6,6 +6,20 @@
|
||||
# [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes.
|
||||
# Changes are relevant to this script and the support docs.mk GNU Make interface.
|
||||
#
|
||||
# ## 8.3.0 (2024-12-27)
|
||||
#
|
||||
# ### Added
|
||||
#
|
||||
# - Debug output of the final command when DEBUG=true.
|
||||
#
|
||||
# Useful to inspect if the script is correctly constructing the final command.
|
||||
#
|
||||
# ## 8.2.0 (2024-12-22)
|
||||
#
|
||||
# ### Removed
|
||||
#
|
||||
# - Special cases for Oracle and Datadog plugins now that they exist in the plugins monorepo.
|
||||
#
|
||||
# ## 8.1.0 (2024-08-22)
|
||||
#
|
||||
# ### Added
|
||||
@@ -13,7 +27,7 @@
|
||||
# - Additional website mounts for projects that use the website repository.
|
||||
#
|
||||
# Mounts are required for `make docs` to work in the website repository or with the website project.
|
||||
# The Makefile is also mounted for convenient development of the procedure that repository.
|
||||
# The Makefile is also mounted for convenient development of the procedure in that repository.
|
||||
#
|
||||
# ## 8.0.1 (2024-07-01)
|
||||
#
|
||||
@@ -355,8 +369,6 @@ SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk'
|
||||
SOURCES_helm_charts_mimir_distributed='mimir'
|
||||
SOURCES_helm_charts_tempo_distributed='tempo'
|
||||
SOURCES_opentelemetry='opentelemetry-docs'
|
||||
SOURCES_plugins_grafana_datadog_datasource='datadog-datasource'
|
||||
SOURCES_plugins_grafana_oracle_datasource='oracle-datasource'
|
||||
SOURCES_resources='website'
|
||||
|
||||
VERSIONS_as_code='UNVERSIONED'
|
||||
@@ -367,8 +379,6 @@ VERSIONS_grafana_cloud_k6='UNVERSIONED'
|
||||
VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED'
|
||||
VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED'
|
||||
VERSIONS_opentelemetry='UNVERSIONED'
|
||||
VERSIONS_plugins_grafana_datadog_datasource='latest'
|
||||
VERSIONS_plugins_grafana_oracle_datasource='latest'
|
||||
VERSIONS_resources='UNVERSIONED'
|
||||
VERSIONS_technical_documentation='UNVERSIONED'
|
||||
VERSIONS_website='UNVERSIONED'
|
||||
@@ -378,8 +388,6 @@ PATHS_grafana_cloud='content/docs/grafana-cloud'
|
||||
PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed'
|
||||
PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed'
|
||||
PATHS_mimir='docs/sources/mimir'
|
||||
PATHS_plugins_grafana_datadog_datasource='docs/sources'
|
||||
PATHS_plugins_grafana_oracle_datasource='docs/sources'
|
||||
PATHS_resources='content'
|
||||
PATHS_tempo='docs/sources/tempo'
|
||||
PATHS_website='content'
|
||||
@@ -631,7 +639,7 @@ POSIX_HERESTRING
|
||||
|
||||
case "${_project}" in
|
||||
# Workaround for arbitrary mounts where the version field is expected to be the local directory
|
||||
# and the repo field is expected to be the container directory.
|
||||
# and the repo field is expected to be the container directory.
|
||||
arbitrary)
|
||||
echo "${_project}^${_version}^${_repo}^" # TODO
|
||||
;;
|
||||
@@ -801,6 +809,10 @@ case "${image}" in
|
||||
| sed "s#$(proj_dst "${proj}")#sources#"
|
||||
EOF
|
||||
|
||||
if [ -n "${DEBUG}" ]; then
|
||||
debg "${cmd}"
|
||||
fi
|
||||
|
||||
case "${OUTPUT_FORMAT}" in
|
||||
human)
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
@@ -837,6 +849,10 @@ EOF
|
||||
/hugo/content/docs
|
||||
EOF
|
||||
|
||||
if [ -n "${DEBUG}" ]; then
|
||||
debg "${cmd}"
|
||||
fi
|
||||
|
||||
case "${OUTPUT_FORMAT}" in
|
||||
human)
|
||||
${cmd} --output=line \
|
||||
|
||||
@@ -15,6 +15,8 @@ weight: 250
|
||||
|
||||
LBAC for data sources is available in private preview on Grafana Cloud for Loki created with basic authentication. Loki data sources for LBAC for data sources can only be created, provisioning is currently not available.
|
||||
|
||||
You cannot configure LBAC rules for Grafana-provisioned data sources from the UI. Alternatively, you can replicate the setting of the provisioned data source in a new data source as described in [LBAC Configuration for New Loki Data Source](https://grafana.com/docs/grafana/latest/administration/data-source-management/teamlbac/configure-teamlbac-for-loki/#task-1-lbac-configuration-for-new-loki-data-source) and then add the LBAC configuration to the new data source.
|
||||
|
||||
## Before you begin
|
||||
|
||||
To be able to use LBAC for data sources rules, you need to enable the feature toggle `teamHttpHeaders` on your Grafana instance. Contact support to enable the feature toggle for you.
|
||||
|
||||
@@ -256,14 +256,24 @@ The system creates a session when a user signs in to Grafana from a new device,
|
||||
|
||||
When a user reaches the session limit, the fourth connection succeeds and the longest inactive session is signed out.
|
||||
|
||||
### Request usage billing
|
||||
|
||||
You can request Grafana Labs to activate usage billing which allows an unlimited number of active users. When usage billing is enabled, Grafana does not enforce active user limits or display warning banners. Instead, you are charged for active users that exceed the limit, according to your customer contract.
|
||||
|
||||
Usage billing involves a contractual agreement between you and Grafana Labs, and it is only available if Grafana Enterprise is configured to [automatically refresh its license token]({{< relref "../../setup-grafana/configure-grafana/enterprise-configuration/#auto_refresh_license" >}}).
|
||||
|
||||
### Request a change to your license
|
||||
|
||||
To increase the number of licensed users within Grafana, extend a license, or change your licensed URL, contact [Grafana support](/profile/org#support) or your Grafana Labs account team. They will update your license, which you can activate from within Grafana.
|
||||
|
||||
For instructions about how to activate your license after it is updated, refer to [Activate an Enterprise license]({{< relref "#activate-an-enterprise-license" >}}).
|
||||
|
||||
## Usage billing
|
||||
|
||||
Standard Grafana Enterprise licenses include a certain number of seats that can be used, and prevent more users logging into Grafana than have been licensed. This makes sense if you prefer a predictable bill. It can however be a problem if you anticipate uneven usage patterns over time or when it's critical that no user ever be prevented from logging into Grafana due to capacity constraints.
|
||||
|
||||
For those use-cases we support usage-based billing, where your license includes a certain number of included users and you are billed on a monthly basis for any excess active users during the month.
|
||||
|
||||
Usage billing involves a contractual agreement between you and Grafana Labs and an update to your license, and it is only available if Grafana Enterprise version 10.0.0 or higher is configured to [automatically refresh its license token]({{< relref "../../setup-grafana/configure-grafana/enterprise-configuration/#auto_refresh_license" >}}).
|
||||
|
||||
### User deduplication
|
||||
|
||||
If your organization has multiple Grafana Enterprise instances with usage billing enabled, then each active user counts only once toward your license, regardless of how many instances that user signs into. Each Grafana Enterprise instance submits a hashed list of users to Grafana Labs via API every day. Each user email address or anonymous device ID is hashed using a one-way sha256 algorithm, and submitted to Grafana where the hashed users are deduplicated across instances.
|
||||
|
||||
### Request usage billing
|
||||
|
||||
To request usage billing, contact your Grafana Labs account team or [submit a support ticket](https://grafana.com/profile/org#support).
|
||||
|
||||
@@ -35,9 +35,9 @@ To activate your license, complete the following tasks.
|
||||
|
||||
For more information about deploying an application on Amazon ECS, refer to [Creating an Amazon ECS service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-service.html).
|
||||
|
||||
1. As you create the Amazon ECS service, use the Grafana Enterprise version 8.3.0 or later container image.
|
||||
1. As you create the Amazon ECS service, use the intended Grafana Enterprise container image version.
|
||||
|
||||
For example, enter `grafana/grafana-enterprise:8.3.3`.
|
||||
For example, enter `grafana/grafana-enterprise:11.3.0`.
|
||||
|
||||
> Only Grafana Enterprise versions 8.3.0 and later support licenses granted through AWS Marketplace.
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ weight: 200
|
||||
|
||||
# Activate a Grafana Enterprise license from AWS Marketplace on EKS
|
||||
|
||||
If you have purchased a Grafana Enterprise subscription through AWS Marketplace, you must activate it in order to use Grafana Enterprise data source plugins and features in Grafana.
|
||||
If you have purchased a Grafana Enterprise subscription through AWS Marketplace, you must activate it to use Grafana Enterprise data source plugins and features in Grafana.
|
||||
|
||||
## Before you begin:
|
||||
|
||||
@@ -37,9 +37,9 @@ To activate your license, complete the following tasks:
|
||||
|
||||
For more information about installing Grafana on Kubernetes using the Helm Chart, refer to the [Grafana Helm Chart](https://github.com/grafana/helm-charts/tree/main/charts/grafana#readme).
|
||||
|
||||
1. Use `kubectl set image deployment/my-release grafana=grafana/grafana-enterprise:<version>` to update the container image to Grafana Enterprise version 8.3.0 or later.
|
||||
1. Use `kubectl set image deployment/my-release grafana=grafana/grafana-enterprise:<version>` to update the container image.
|
||||
|
||||
For example, enter `grafana/grafana-enterprise:8.3.3`.
|
||||
For example, enter `grafana/grafana-enterprise:11.3.0`.
|
||||
|
||||
> Only Grafana Enterprise versions 8.3.0 and later support licenses granted through AWS Marketplace.
|
||||
|
||||
|
||||
@@ -103,53 +103,45 @@ To browse for available plugins:
|
||||
|
||||
### Install a plugin
|
||||
|
||||
To install a plugin:
|
||||
The most common way to install a plugin is through the Grafana UI, but alternative methods are also available.
|
||||
|
||||
1. In Grafana, click **Administration > Plugins and data > Plugins** in the side navigation menu to view all plugins.
|
||||
1. Browse and find a plugin.
|
||||
1. Click the plugin's logo.
|
||||
1. Click **Install**.
|
||||
|
||||
When the update is complete, you'll see a confirmation message that the installation was successful.
|
||||
There are also additional ways to install plugins depending on your setup.
|
||||
|
||||
### Update a plugin
|
||||
#### Install a plugin using Grafana CLI
|
||||
|
||||
To update a plugin:
|
||||
Grafana CLI allows you to install, upgrade, and manage your Grafana plugins using a command line. For more information about Grafana CLI plugin commands, refer to [Plugin commands]({{< relref "../../cli/#plugins-commands" >}}).
|
||||
|
||||
1. In Grafana, click **Administration > Plugins and data > Plugins** in the side navigation menu to view all plugins.
|
||||
1. Click the **Installed** filter to show only installed plugins.
|
||||
1. Click the plugin's logo.
|
||||
1. Click **Update**.
|
||||
#### Install a plugin from a ZIP file
|
||||
|
||||
When the update is complete, you'll see a confirmation message that the update was successful.
|
||||
This method is typically used for plugins not available in the Plugin Catalog or in environments without internet access.
|
||||
|
||||
### Uninstall a plugin
|
||||
Download the archive containing the plugin assets, and install it by extracting the archive into the plugin directory. For example:
|
||||
|
||||
To uninstall a plugin:
|
||||
```bash
|
||||
unzip my-plugin-0.2.0.zip -d YOUR_PLUGIN_DIR/my-plugin
|
||||
```
|
||||
|
||||
1. In Grafana, click **Administration > Plugins and data > Plugins** in the side navigation menu to view all plugins.
|
||||
1. Click the plugin's logo.
|
||||
1. Click the **Installed** filter to show only installed plugins.
|
||||
1. Click **Uninstall**.
|
||||
The path to the plugin directory is defined in the configuration file. For more information, refer to [Configuration]({{< relref "../../setup-grafana/configure-grafana/#plugins" >}}).
|
||||
|
||||
When the update is complete, you'll see a confirmation message that the uninstall was successful.
|
||||
#### Install a plugin in air-gapped environment
|
||||
|
||||
## Install Grafana plugins
|
||||
Plugin installation usually requires an internet connection. You can check which endpoints are used during the installation on your instance and add them to your instance’s allowlist.
|
||||
|
||||
Grafana supports data source, panel, and app plugins.
|
||||
If this is not possible you can go via installing a plugin using [Grafana CLI](#install-a-plugin-using-grafana-cli) or as a [ZIP file](#install-a-plugin-from-a-zip-file).
|
||||
|
||||
1. In a web browser, navigate to the [Grafana plugin catalog](https://grafana.com/plugins) and find a plugin that you want to install.
|
||||
1. Click the plugin, and then click the **Installation** tab.
|
||||
You can fetch any plugin from Grafana.com API following the download link referenced in the API.
|
||||
Here is an example based on `grafana-lokiexplore-app` plugins.
|
||||
|
||||
### Install plugin on Grafana Cloud
|
||||
1. Open `https://grafana.com/api/plugins/grafana-lokiexplore-app` and look for `links` section
|
||||
1. Find a `download` url which looks something like `https://grafana.com/api/plugins/grafana-lokiexplore-app/versions/1.0.2/download`
|
||||
1. Use this URL to download the plugin ZIP file, which you can then install as described above.
|
||||
|
||||
On the **Installation tab**, in the **For** field, click the name of the Grafana instance on which you want to install the plugin.
|
||||
|
||||
Grafana Cloud handles the plugin installation automatically.
|
||||
|
||||
If you're logged in to Grafana Cloud when you add a plugin, log out and then log back in again to use the new plugin.
|
||||
|
||||
### Install plugins using the Grafana Helm chart
|
||||
#### Install plugins using the Grafana Helm chart
|
||||
|
||||
With the Grafana Helm chart, add the plugins you want to install as a list using the `plugins` field in the your values file. For more information about the configuration, refer to [the Helm chart configuration reference](https://github.com/grafana/helm-charts/tree/main/charts/grafana#configuration).
|
||||
|
||||
@@ -162,21 +154,29 @@ plugins:
|
||||
- redis-datasource
|
||||
```
|
||||
|
||||
### Install plugin on local Grafana
|
||||
When the update is complete, a confirmation message will indicate the installation was successful.
|
||||
|
||||
Follow the instructions on the **Install** tab. You can either install the plugin with a Grafana CLI command or by downloading and uncompressing a zip file into the Grafana plugins directory. We recommend using Grafana CLI in most instances. The zip option is available if your Grafana server doesn't have access to the internet.
|
||||
### Update a plugin
|
||||
|
||||
For more information about Grafana CLI plugin commands, refer to [Plugin commands]({{< relref "../../cli/#plugins-commands" >}}).
|
||||
To update a plugin:
|
||||
|
||||
#### Install a packaged plugin
|
||||
1. In Grafana, click **Administration > Plugins and data > Plugins** in the side navigation menu to view all plugins.
|
||||
1. Click the **Installed** filter to show only installed plugins.
|
||||
1. Click the plugin's logo.
|
||||
1. Click **Update**.
|
||||
|
||||
After the user has downloaded the archive containing the plugin assets, they can install it by extracting the archive into their plugin directory. For example:
|
||||
When the update is complete, a confirmation message will indicate the installation was successful.
|
||||
|
||||
```bash
|
||||
unzip my-plugin-0.2.0.zip -d YOUR_PLUGIN_DIR/my-plugin
|
||||
```
|
||||
### Uninstall a plugin
|
||||
|
||||
The path to the plugin directory is defined in the configuration file. For more information, refer to [Configuration]({{< relref "../../setup-grafana/configure-grafana/#plugins" >}}).
|
||||
To uninstall a plugin:
|
||||
|
||||
1. In Grafana, click **Administration > Plugins and data > Plugins** in the side navigation menu to view all plugins.
|
||||
1. Click the plugin's logo.
|
||||
1. Click the **Installed** filter to show only installed plugins.
|
||||
1. Click **Uninstall**.
|
||||
|
||||
When the update is complete, a confirmation message will indicate the installation was successful.
|
||||
|
||||
## Plugin signatures
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ Therefore, we heavily rely on the expertise of the community.
|
||||
|
||||
## Data sources
|
||||
|
||||
You can manage data sources in Grafana by adding YAML configuration files in the [`provisioning/data sources`]({{< relref "../../setup-grafana/configure-grafana#provisioning" >}}) directory.
|
||||
You can manage data sources in Grafana by adding YAML configuration files in the [`provisioning/datasources`]({{< relref "../../setup-grafana/configure-grafana#provisioning" >}}) directory.
|
||||
Each configuration file can contain a list of `datasources` to add or update during startup.
|
||||
If the data source already exists, Grafana reconfigures it to match the provisioned configuration file.
|
||||
|
||||
@@ -257,17 +257,17 @@ All of these settings are optional.
|
||||
The _HTTP\*_ tag denotes data sources that communicate using the HTTP protocol, including all core data source plugins except MySQL, PostgreSQL, and MS SQL.
|
||||
{{< /admonition >}}
|
||||
|
||||
| Name | Type | Data source | Description |
|
||||
| ----------------- | ------ | ---------------------------------- | -------------------------------------------------------- |
|
||||
| tlsCACert | string | _HTTP\*_, MySQL, PostgreSQL | CA cert for out going requests |
|
||||
| tlsClientCert | string | _HTTP\*_, MySQL, PostgreSQL | TLS Client cert for outgoing requests |
|
||||
| tlsClientKey | string | _HTTP\*_, MySQL, PostgreSQL | TLS Client key for outgoing requests |
|
||||
| password | string | _HTTP\*_, MySQL, PostgreSQL, MSSQL | password |
|
||||
| basicAuthPassword | string | _HTTP\*_ | password for basic authentication |
|
||||
| accessKey | string | Cloudwatch | Access key for connecting to Cloudwatch |
|
||||
| secretKey | string | Cloudwatch | Secret key for connecting to Cloudwatch |
|
||||
| sigV4AccessKey | string | Elasticsearch and Prometheus | SigV4 access key. Required when using keys auth provider |
|
||||
| sigV4SecretKey | string | Elasticsearch and Prometheus | SigV4 secret key. Required when using keys auth provider |
|
||||
| Name | Type | Data source | Description |
|
||||
| ----------------- | ------ | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| tlsCACert | string | _HTTP\*_, MySQL, PostgreSQL | CA cert for out going requests. You can point directly to your stored cert by using an environment variable following the `$__file{path/to/ca}` format. |
|
||||
| tlsClientCert | string | _HTTP\*_, MySQL, PostgreSQL | TLS Client cert for outgoing requests. You can point directly to your stored cert by using an environment variable following the `$__file{path/to/cert}` format. |
|
||||
| tlsClientKey | string | _HTTP\*_, MySQL, PostgreSQL | TLS Client key for outgoing requests. You can point directly to your stored key by using an environment variable following the `$__file{path/to/key}` format. |
|
||||
| password | string | _HTTP\*_, MySQL, PostgreSQL, MSSQL | password |
|
||||
| basicAuthPassword | string | _HTTP\*_ | password for basic authentication |
|
||||
| accessKey | string | Cloudwatch | Access key for connecting to Cloudwatch |
|
||||
| secretKey | string | Cloudwatch | Secret key for connecting to Cloudwatch |
|
||||
| sigV4AccessKey | string | Elasticsearch and Prometheus | SigV4 access key. Required when using keys auth provider |
|
||||
| sigV4SecretKey | string | Elasticsearch and Prometheus | SigV4 secret key. Required when using keys auth provider |
|
||||
|
||||
#### Custom HTTP headers for data sources
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ The following list contains role-based access control actions.
|
||||
| `dashboards.permissions:write` | <ul><li>`dashboards:*`</li><li>`dashboards:uid:*`</li><li>`folders:*`</li><li>`folders:uid:*`</li></ul> | Update permissions for one or more dashboards. |
|
||||
| `dashboards:read` | <ul><li>`dashboards:*`</li><li>`dashboards:uid:*`</li><li>`folders:*`</li><li>`folders:uid:*`</li></ul> | Read one or more dashboards. |
|
||||
| `dashboards:write` | <ul><li>`dashboards:*`</li><li>`dashboards:uid:*`</li><li>`folders:*`</li><li>`folders:uid:*`</li></ul> | Update one or more dashboards. |
|
||||
| `dashboards.public:write` | <ul><li>`dashboards:*`</li><li>`dashboards:uid:*`</li></ul> | Write shared dashboard configuration. |
|
||||
| `dashboards.public:write` | <ul><li>`dashboards:*`</li><li>`dashboards:uid:*`</li></ul> | Write public dashboard configuration. |
|
||||
| `datasources.caching:read` | <ul><li>`datasources:*`</li><li>`datasources:uid:*`</li></ul> | Read data source query caching settings. |
|
||||
| `datasources.caching:write` | <ul><li>`datasources:*`</li><li>`datasources:uid:*`</li></ul> | Update data source query caching settings. |
|
||||
| `datasources:create` | None | Create data sources. |
|
||||
@@ -199,6 +199,26 @@ The following list contains role-based access control actions used by Grafana Ad
|
||||
| `grafana‑adaptive‑metrics‑app.exemptions:read` | None | Read recommendation exemptions. |
|
||||
| `grafana‑adaptive‑metrics‑app.exemptions:write` | None | Create, update, and delete recommendation exemptions. |
|
||||
|
||||
### Grafana Alerting Notification action definitions
|
||||
|
||||
To enable these permissions, enable the `alertingApiServer` feature toggle.
|
||||
|
||||
| Action | Applicable scopes | Description |
|
||||
| -------------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| `alert.notifications.receivers:read` | `receivers:*`<br>`receivers:uid:*` | Read contact points. None |
|
||||
| `alert.notifications.receivers.secrets:read` | `receivers:*`<br>`receivers:uid:*` | Export contact points with decrypted secrets.None |
|
||||
| `alert.notifications.receivers:create` | None | Create a new contact points. The creator is automatically granted full access to the created contact point.None |
|
||||
| `alert.notifications.receivers:write` | `receivers:*`<br>`receivers:uid:*` | Update existing contact points.None |
|
||||
| `alert.notifications.receivers:delete` | `receivers:*`<br>`receivers:uid:*` | Update and delete existing contact points.None |
|
||||
| `receivers.permissions:read` | `receivers:*`<br>`receivers:uid:*` | Read permissions for contact points.None |
|
||||
| `receivers.permissions:write` | `receivers:*`<br>`receivers:uid:*` | Manage permissions for contact points.None |
|
||||
| `alert.notifications.time-intervals:read` | None | Read mute time intervals.None |
|
||||
| `alert.notifications.time-intervals:write` | None | Create new or update existing mute time intervals.None |
|
||||
| `alert.notifications.time-intervals:delete` | None | Delete existing time intervals.None |
|
||||
| `alert.notifications.templates:read` | None | Read templates. |
|
||||
| `alert.notifications.templates:write` | None | Create new or update existing templates.None |
|
||||
| `alert.notifications.templates:delete` | None | Delete existing templates.None |
|
||||
|
||||
## Scope definitions
|
||||
|
||||
The following list contains role-based access control scopes.
|
||||
|
||||
@@ -98,7 +98,7 @@ To learn how to use the roles API to determine the role UUIDs, refer to [Manage
|
||||
| `fixed:dashboards.insights:reader` | `fixed_JlBJ2_gizP8zhgaeGE2rjyZe2Rs` | `dashboards.insights:read` | Read dashboard insights data and see presence indicators. |
|
||||
| `fixed:dashboards.permissions:reader` | `fixed_f17oxuXW_58LL8mYJsm4T_mCeIw` | `dashboards.permissions:read` | Read all dashboard permissions. |
|
||||
| `fixed:dashboards.permissions:writer` | `fixed_CcznxhWX_Yqn8uWMXMQ-b5iFW9k` | All permissions from `fixed:dashboards.permissions:reader` and <br>`dashboards.permissions:write` | Read and update all dashboard permissions. |
|
||||
| `fixed:dashboards.public:writer` | `fixed_f_GHHRBciaqESXfGz2oCcooqHxs` | `dashboards.public:write` | Create, update, delete or pause a shared dashboard. |
|
||||
| `fixed:dashboards.public:writer` | `fixed_f_GHHRBciaqESXfGz2oCcooqHxs` | `dashboards.public:write` | Create, update, delete or pause a public dashboard. |
|
||||
| `fixed:datasources:creator` | `fixed_XX8jHREgUt-wo1A-rPXIiFlX6Zw` | `datasources:create` | Create data sources. |
|
||||
| `fixed:datasources:explorer` | `fixed_qDzW9mzx9yM91T5Bi8dHUM2muTw` | `datasources:explore` | Enable the Explore feature. Data source permissions still apply, you can only query data sources for which you have query permissions. |
|
||||
| `fixed:datasources:reader` | `fixed_C2x8IxkiBc1KZVjyYH775T9jNMQ` | `datasources:read`<br>`datasources:query` | Read and query data sources. |
|
||||
|
||||
@@ -27,10 +27,6 @@ cards:
|
||||
href: ./fundamentals/
|
||||
description: Learn more about the fundamentals and available features that help you create, manage, and respond to alerts; and improve your team’s ability to resolve issues quickly.
|
||||
height: 24
|
||||
- title: Set up
|
||||
href: ./set-up/
|
||||
description: Set up your implementation of Grafana Alerting.
|
||||
height: 24
|
||||
- title: Configure alert rules
|
||||
href: ./alerting-rules/
|
||||
description: Create, manage, view, and adjust alert rules to alert on your metrics data or log entries from multiple data sources — no matter where your data is stored.
|
||||
@@ -39,13 +35,13 @@ cards:
|
||||
href: ./configure-notifications/
|
||||
description: Choose how, when, and where to send your alert notifications.
|
||||
height: 24
|
||||
- title: Detect and respond
|
||||
href: ./manage-notifications/
|
||||
- title: Monitor status
|
||||
href: ./monitor-status/
|
||||
description: Monitor, respond to, and triage issues within your services.
|
||||
height: 24
|
||||
- title: Monitor
|
||||
href: ./monitor/
|
||||
description: Monitor your alerting metrics to ensure you identify potential issues before they become critical.
|
||||
- title: Additional configuration
|
||||
href: ./set-up/
|
||||
description: Use advanced configuration options to further tailor your alerting setup. These options can enhance security, scalability, and automation in complex environments.
|
||||
height: 24
|
||||
---
|
||||
|
||||
|
||||
@@ -12,37 +12,59 @@ labels:
|
||||
- oss
|
||||
title: Configure alert rules
|
||||
weight: 120
|
||||
refs:
|
||||
alert-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/
|
||||
configure-grafana-alerts:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-grafana-managed-rule/
|
||||
configure-ds-alerts:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-data-source-managed-rule/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-data-source-managed-rule/
|
||||
recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/
|
||||
alert-types-comparison-table:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/#comparison-between-alert-rule-types
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/#comparison-between-alert-rule-types
|
||||
templating-labels-annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/templates/
|
||||
---
|
||||
|
||||
# Configure alert rules
|
||||
|
||||
An alert rule consists of one or more queries and expressions that select the data you want to measure. It also contains a condition, which is the threshold that an alert rule must meet or exceed in order to fire.
|
||||
[Alert rules](ref:alert-rules) are the central component of your alerting system.
|
||||
|
||||
Create, manage, view, and adjust alert rules to alert on your metrics data or log entries from multiple data sources — no matter where your data is stored.
|
||||
An alert rule consists of one or more queries and expressions that select the data you want to measure. It contains a condition to trigger the alert, an evaluation period that determines how often the rule is evaluated, and additional options to manage alert events and their notifications.
|
||||
|
||||
The main parts of alert rule creation are:
|
||||
Grafana supports two types of alert rules:
|
||||
|
||||
1. Select your data source
|
||||
1. Query your data
|
||||
1. Normalize your data
|
||||
1. Set your threshold
|
||||
1. Grafana-managed alert rules: These can query multiple data sources.
|
||||
|
||||
**Query, expressions, and alert condition**
|
||||
1. Data source-managed alert rules: These can only query Prometheus-based data sources and support horizontal scaling.
|
||||
|
||||
What are you monitoring? How are you measuring it?
|
||||
We recommend using Grafana-managed alert rules whenever possible, and opting for data source-managed alert rules when horizontal scaling is required. Refer to the [comparison table of alert rule types](ref:alert-types-comparison-table) for a more detailed overview.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Expressions can only be used for Grafana-managed alert rules.
|
||||
{{< /admonition >}}
|
||||
Both types of alert rules can be configured in Grafana using the **+ New alert rule** flow. For step-by-step instructions, refer to:
|
||||
|
||||
**Evaluation**
|
||||
- [Configure Grafana-managed alert rules](ref:configure-grafana-alerts)
|
||||
- [Configure data source-managed alert rules](ref:configure-ds-alerts)
|
||||
- [Create and link alert rules to panels](ref:templating-labels-annotations)
|
||||
|
||||
How do you want your alert to be evaluated?
|
||||
Alert rules can also query metrics generated by recording rules. To learn more, refer to:
|
||||
|
||||
**Labels and notifications**
|
||||
|
||||
How do you want to route your alert? What kind of additional labels could you add to annotate your alert rules and ease searching?
|
||||
|
||||
**Annotations**
|
||||
|
||||
Do you want to add more context on the alert in your notification messages, for example, what caused the alert to fire? Which server did it happen on?
|
||||
- [Create recording rules](ref:recording-rules)
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-alerts-panels/
|
||||
description: Create alert rules from panels. Reuse the queries in the panel and create alert rules based on them.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- panels
|
||||
- create
|
||||
- grafana-managed
|
||||
- data source-managed
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create alert rules from panels
|
||||
weight: 400
|
||||
---
|
||||
|
||||
## Create alert rules from panels
|
||||
|
||||
Create alert rules from time series panels. By doing so, you can reuse the queries in the panel and create alert rules based on them.
|
||||
|
||||
1. Navigate to a dashboard in the **Dashboards** section.
|
||||
2. Hover over the top-right corner of a time series panel and click the panel menu icon.
|
||||
3. From the dropdown menu, select **More...** > **New alert rule**.
|
||||
|
||||
The New alert rule form opens where you can configure and create your alert rule based on the query used in the panel.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Changes to the panel aren't reflected on the linked alert rules. If you change a query, you have to update it in both the panel and the alert rule.
|
||||
|
||||
Alert rules are only supported in [time series](ref:time-series) visualizations.
|
||||
{{% /admonition %}}
|
||||
|
||||
{{< docs/play title="visualizations with linked alerts in Grafana" url="https://play.grafana.org/d/000000074/" >}}
|
||||
|
||||
## View alert rules from panels
|
||||
|
||||
To view alert rules associated with a time series panel, complete the following steps.
|
||||
|
||||
1. Open the panel editor by hovering over the top-right corner of any panel
|
||||
1. Click the panel menu icon that appears.
|
||||
1. Click **Edit**.
|
||||
1. Click the **Alert** tab to view existing alert rules or create a new one.
|
||||
@@ -0,0 +1,157 @@
|
||||
---
|
||||
aliases:
|
||||
- ../unified-alerting/alerting-rules/create-mimir-loki-managed-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-mimir-loki-managed-rule/
|
||||
- ../unified-alerting/alerting-rules/edit-cortex-loki-namespace-group/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/edit-cortex-loki-namespace-group/
|
||||
- ../unified-alerting/alerting-rules/edit-mimir-loki-namespace-group/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/edit-mimir-loki-namespace-group/
|
||||
- ../alerting-rules/create-mimir-loki-managed-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-mimir-loki-managed-rule/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-data-source-managed-rule/
|
||||
description: Configure data source-managed alert rules alert for an external Grafana Mimir or Loki instance
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- create
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure data source-managed alert rules
|
||||
weight: 200
|
||||
refs:
|
||||
shared-configure-prometheus-data-source-alerting:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/datasources/prometheus/configure-prometheus-data-source/#alerting
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/prometheus/configure-prometheus-data-source/#alerting
|
||||
configure-grafana-managed-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-grafana-managed-rule/notification-policies/
|
||||
notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
pending-period:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/#pending-period
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/#pending-period
|
||||
alert-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/
|
||||
alert-rule-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
alert-rule-evaluation:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/
|
||||
shared-provision-alerting-resources:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/provision-alerting-resources/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/provision-alerting-resources/
|
||||
shared-alert-rule-template:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
shared-annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
shared-link-alert-rules-to-panels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
---
|
||||
|
||||
# Configure data source-managed alert rules
|
||||
|
||||
Data source-managed alert rules can only be created using Grafana Mimir or Grafana Loki data sources.
|
||||
|
||||
The rules are stored within the data source. In a distributed architecture, they can scale horizontally to provide high-availability. For more details, refer to [alert rule types](ref:alert-rules).
|
||||
|
||||
We recommend using [Grafana-managed alert rules](ref:configure-grafana-managed-rules) whenever possible and opting for data source-managed alert rules when scaling your alerting setup is necessary.
|
||||
|
||||
{{< docs/shared lookup="alerts/note-prometheus-ds-rules.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
To create or edit data source-managed alert rules, follow these instructions.
|
||||
|
||||
## Before you begin
|
||||
|
||||
Verify that you have write permission to the Mimir or Loki data source. Otherwise, you cannot create or update data source-managed alert rules.
|
||||
|
||||
### Enable the Ruler API
|
||||
|
||||
For more information, refer to the [Mimir Ruler API](/docs/mimir/latest/references/http-api/#ruler) or [Loki Ruler API](/docs/loki/latest/api/#ruler).
|
||||
|
||||
- **Mimir** - use the `/prometheus` prefix. The Prometheus data source supports both Grafana Mimir and Prometheus, and Grafana expects that both the [Query API](/docs/mimir/latest/operators-guide/reference-http-api/#querier--query-frontend) and [Ruler API](/docs/mimir/latest/operators-guide/reference-http-api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API.
|
||||
|
||||
- **Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types.
|
||||
|
||||
### Permissions
|
||||
|
||||
Alert rules for Mimir or Loki instances can be edited or deleted by users with **Editor** or **Admin** roles.
|
||||
|
||||
If you do not want to manage alert rules for a particular data source, go to its settings and clear the **Manage alerts via Alerting UI** checkbox.
|
||||
|
||||
{{< docs/shared lookup="alerts/configure-provisioning-before-begin.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
{{< docs/shared lookup="alerts/configure-alert-rule-name.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
## Define query and condition
|
||||
|
||||
Define a query to get the data you want to measure and a condition that needs to be met before an alert rule fires.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
By default, new alert rules are Grafana-managed. To switch to **Data source-managed**, follow these instructions.
|
||||
{{% /admonition %}}
|
||||
|
||||
1. Select a Prometheus-based data source from the drop-down list.
|
||||
|
||||
You can also click **Open advanced data source picker** to find more options.
|
||||
|
||||
1. Enter a PromQL or LogQL query, including the alert condition.
|
||||
1. In the **Rule type** option, select **Data source-managed**.
|
||||
1. Click **Preview alerts**.
|
||||
|
||||
## Set alert evaluation behavior
|
||||
|
||||
Use [alert rule evaluation](ref:alert-rule-evaluation) to determine how frequently an alert rule should be evaluated and how quickly it should change its state.
|
||||
|
||||
1. Select a namespace or click **+ New namespace**.
|
||||
1. Select an evaluation group or click **+ New evaluation group**.
|
||||
|
||||
If you are creating a new evaluation group, specify the interval for the group.
|
||||
|
||||
All rules within the same group are evaluated sequentially over the same time interval. You can reorder them from the **Alert rules** page.
|
||||
|
||||
1. Enter a pending period.
|
||||
|
||||
The [pending period](ref:pending-period) is the period in which an alert rule can be in breach of the condition until it fires.
|
||||
|
||||
Once a condition is met, the alert goes into the **Pending** state. If the condition remains active for the duration specified, the alert transitions to the **Firing** state, else it reverts to the **Normal** state.
|
||||
|
||||
## Configure labels and notifications
|
||||
|
||||
Add [labels](ref:alert-rule-labels) to your alert rules to set which [notification policy](ref:notification-policies) should handle your firing alert instances.
|
||||
|
||||
All alert rules and instances, irrespective of their labels, match the default notification policy. If there are no nested policies, or no nested policies match the labels in the alert rule or alert instance, then the default notification policy is the matching policy.
|
||||
|
||||
1. Add labels if you want to change the way your notifications are routed.
|
||||
|
||||
Add custom labels by selecting existing key-value pairs from the drop down, or add new labels by entering the new key or value.
|
||||
|
||||
{{< docs/shared lookup="alerts/configure-notification-message.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
@@ -24,108 +24,144 @@ refs:
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/dashboards/use-dashboards/#time-units-and-relative-ranges
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/dashboards/use-dashboards/#time-units-and-relative-ranges
|
||||
fundamentals:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/
|
||||
alert-instance-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
keep-last-state:
|
||||
modify-the-no-data-or-error-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#keep-last-state
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#modify-the-no-data-or-error-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#keep-last-state
|
||||
add-a-query:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/query-transform-data/#add-a-query
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/query-transform-data/#add-a-query
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#modify-the-no-data-or-error-state
|
||||
pending-period:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/#pending-period
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/#pending-period
|
||||
alerting-on-numeric-data:
|
||||
alert-rule-evaluation:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#alert-on-numeric-data
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#alert-on-numeric-data
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/
|
||||
mute-timings:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/mute-timings/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/mute-timings/
|
||||
alert-rule-query:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/
|
||||
alert-rule-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
expression-queries:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/query-transform-data/expression-queries/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/query-transform-data/expression-queries/
|
||||
annotation-label:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
alert-condition:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
alert-list:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
|
||||
contact-points:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/alert-list/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/contact-points/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/alert-list/
|
||||
time-series:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/contact-points/
|
||||
notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/time-series/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
data-sources:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/datasources/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
- destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/
|
||||
alert-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/
|
||||
compatible-data-sources:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
shared-provision-alerting-resources:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/provision-alerting-resources/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/provision-alerting-resources/
|
||||
shared-alert-rule-template:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
shared-annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
shared-link-alert-rules-to-panels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
---
|
||||
|
||||
# Configure Grafana-managed alert rules
|
||||
|
||||
Grafana-managed rules are the most flexible alert rule type. They allow you to create alerts that can act on data from any of our supported data sources. In addition to supporting multiple data sources, you can also add expressions to transform your data and set alert conditions. Using images in alert notifications is also supported. This is the only type of rule that allows alerting from multiple data sources in a single rule definition.
|
||||
|
||||
Multiple alert instances can be created as a result of one alert rule (also known as a multi-dimensional alerting).
|
||||
Grafana-managed rules can query data from multiple data sources in a single alert rule. They are the most flexible [alert rule type](ref:alert-rules). You can also add expressions to transform your data, set alert conditions, and images in alert notifications.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
For Grafana Cloud Free Forever, you can create up to 100 free Grafana-managed alert rules with each alert rule having a maximum of 1000 alert instances.
|
||||
In Grafana Cloud, the number of Grafana-managed alert rules you can create depends on your Grafana Cloud plan.
|
||||
|
||||
- Free Forever plan: You can create up to 100 free alert rules, with each alert rule having a maximum of 1000 alert instances.
|
||||
- All paid plans (Pro and Advanced): They have a soft limit of 2000 alert rules and support unlimited alert instances. To increase the limit, open a support ticket from the [Cloud portal](/docs/grafana-cloud/account-management/support/).
|
||||
|
||||
{{% /admonition %}}
|
||||
|
||||
Grafana-managed alert rules can only be edited or deleted by users with Edit permissions for the folder storing the rules.
|
||||
|
||||
If you delete an alert resource created in the UI, you can no longer retrieve it.
|
||||
To make a backup of your configuration and to be able to restore deleted alerting resources, create your alerting resources using file provisioning, Terraform, or the Alerting API.
|
||||
To create or edit Grafana-managed alert rules, follow the instructions below. For a practical example, check out our [tutorial on getting started with Grafana alerting](http://grafana.com/tutorials/alerting-get-started/).
|
||||
|
||||
## Before you begin
|
||||
|
||||
Verify that the data sources you plan to query in the alert rule are [compatible with Grafana-managed alert rules](ref:compatible-data-sources) and are properly configured.
|
||||
|
||||
### Permissions
|
||||
|
||||
Only users with **Edit** permissions for the folder storing the rules can edit or delete Grafana-managed alert rules.
|
||||
|
||||
{{< docs/shared lookup="alerts/configure-provisioning-before-begin.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
### Default vs Advanced options
|
||||
|
||||
You can use default or advanced options for Grafana-managed alert rule creation. The default options streamline rule creation with a cleaner header and a single query and condition. For more complex rules, use advanced options to add multiple queries and expressions.
|
||||
|
||||
Default and advanced options are enabled by default for Grafana Cloud users and this feature is being rolled out progressively.
|
||||
You can toggle between the two options. Once you have created an alert rule, the system defaults to your previous choice for the next alert rule.
|
||||
|
||||
For OSS users,enable the `alertingQueryAndExpressionsStepMode` feature toggle.
|
||||
Switching from advanced to default may result in queries and expressions that cannot be converted. In this case, a warning message asks if you want to continue to reset to default settings.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Once you have created an alert rule using one of the options, the system defaults to this option for the next alert rule you create.
|
||||
Default and advanced options are enabled by default for Grafana Cloud users and this feature is being rolled out progressively. OSS users can enable them via the [`alertingQueryAndExpressionsStepMode` feature toggle](/setup-grafana/configure-grafana/feature-toggles/).
|
||||
|
||||
You can toggle between the two options. However, if you want to switch from advanced options to the default, it may be that your query and expressions cannot be converted. In this case, a warning message checks whether you want to continue to reset to default settings.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Steps
|
||||
|
||||
To create a Grafana-managed alert rule, use the in-product alert creation flow and follow these steps.
|
||||
|
||||
To get started quickly, refer to our [tutorial on getting started with Grafana alerting](http://grafana.com/tutorials/alerting-get-started/).
|
||||
|
||||
## Set alert rule name
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alert rules** -> **+ New alert rule**.
|
||||
1. Enter a name to identify your alert rule.
|
||||
|
||||
This name is displayed in the alert rule list. It is also the `alertname` label for every alert instance that is created from this rule.
|
||||
{{< docs/shared lookup="alerts/configure-alert-rule-name.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
## Define query and condition
|
||||
|
||||
Define a query to get the data you want to measure and a condition that needs to be met before an alert rule fires.
|
||||
|
||||
You can toggle between **Default** and **Advanced** options. If the [Default vs. Advanced feature](#default-vs-advanced-options) is not enabled in your Grafana instance, follow the **Advanced options** instructions.
|
||||
|
||||
{{< collapse title="Default options" >}}
|
||||
|
||||
1. Add a query.
|
||||
1. Add an alert condition.
|
||||
1. Add a [query](ref:alert-rule-query).
|
||||
1. Add an [alert condition](ref:alert-condition).
|
||||
|
||||
The **When** input includes the reducer function and the last input is the threshold.
|
||||
|
||||
@@ -137,15 +173,11 @@ Define a query to get the data you want to measure and a condition that needs to
|
||||
1. Select a data source.
|
||||
1. From the **Options** dropdown, specify a [time range](ref:time-units-and-relative-ranges).
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Grafana Alerting only supports fixed relative time ranges, for example, `now-24hr: now`.
|
||||
|
||||
It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:59:592` or semi-relative time ranges: `now/d to: now`.
|
||||
{{% /admonition %}}
|
||||
Note that Grafana Alerting only supports fixed relative time ranges, for example, `now-24hr: now`. It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:59:592` or semi-relative time ranges: `now/d to: now`.
|
||||
|
||||
1. Add a query.
|
||||
|
||||
To add multiple [queries](ref:add-a-query), click **Add query**.
|
||||
To add multiple [queries](ref:alert-rule-query), click **Add query**.
|
||||
|
||||
All alert rules are managed by Grafana by default. If you want to switch to a data source-managed alert rule, click **Switch to data source-managed alert rule**.
|
||||
|
||||
@@ -153,9 +185,7 @@ It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:
|
||||
|
||||
a. For each expression, select either **Classic condition** to create a single alert rule, or choose from the **Math**, **Reduce**, and **Resample** options to generate separate alert for each series.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
When using Prometheus, you can use an instant vector and built-in functions, so you don't need to add additional expressions.
|
||||
{{% /admonition %}}
|
||||
|
||||
b. Click **Preview** to verify that the expression is successful.
|
||||
|
||||
@@ -163,12 +193,24 @@ It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:
|
||||
|
||||
You can only add one recovery threshold in a query and it must be the alert condition.
|
||||
|
||||
1. Click **Set as alert condition** on the query or expression you want to set as your alert condition.
|
||||
1. Click **Set as alert condition** on the query or expression you want to set as your [alert condition](ref:alert-condition).
|
||||
{{< /collapse >}}
|
||||
|
||||
## Set alert evaluation behavior
|
||||
## Set folder and labels
|
||||
|
||||
Use alert rule evaluation to determine how frequently an alert rule should be evaluated and how quickly it should change its state.
|
||||
Organize your alert rule with a folder and set of labels.
|
||||
|
||||
In the **Labels** section, you can optionally choose whether to add labels to organize your alert rules and their notifications. For more details, refer to [alert rule labels](ref:alert-rule-labels).
|
||||
|
||||
1. Select a folder or click **+ New folder**.
|
||||
|
||||
1. Add labels, if required.
|
||||
|
||||
Add custom labels by selecting existing key-value pairs from the drop down, or add new labels by entering the new key or value.
|
||||
|
||||
## Configure alert evaluation behavior
|
||||
|
||||
Use [alert rule evaluation](ref:alert-rule-evaluation) to determine how frequently an alert rule should be evaluated and how quickly it should change its state.
|
||||
|
||||
To do this, you need to make sure that your alert rule is in the right evaluation group and set a pending period time that works best for your use case.
|
||||
|
||||
@@ -179,7 +221,7 @@ To do this, you need to make sure that your alert rule is in the right evaluatio
|
||||
|
||||
All rules within the same group are evaluated concurrently over the same time interval.
|
||||
|
||||
1. Enter a pending period.
|
||||
1. Enter a [pending period](ref:pending-period).
|
||||
|
||||
The pending period is the period in which an alert rule can be in breach of the condition until it fires.
|
||||
|
||||
@@ -187,15 +229,20 @@ To do this, you need to make sure that your alert rule is in the right evaluatio
|
||||
|
||||
1. Turn on pause alert notifications, if required.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
You can pause alert rule evaluation to prevent noisy alerting while tuning your alerts.
|
||||
Pausing stops alert rule evaluation and doesn't create any alert instances.
|
||||
This is different to mute timings, which stop notifications from being delivered, but still allows for alert rule evaluation and the creation of alert instances.
|
||||
{{< /admonition >}}
|
||||
This is different to [mute timings](ref:mute-timings), which stop notifications from being delivered, but still allows for alert rule evaluation and the creation of alert instances.
|
||||
|
||||
1. In **Configure no data and error handling**, configure alerting behavior in the absence of data.
|
||||
1. In **Configure no data and error handling**, you can define the alerting behavior and alerting state for two scenarios:
|
||||
|
||||
Use the guidelines in [No data and error handling](#configure-no-data-and-error-handling).
|
||||
- When the evaluation returns **No data** or all values are null.
|
||||
- When the evaluation returns **Error** or timeout.
|
||||
|
||||
### Configure no data and error handling
|
||||
|
||||
{{< docs/shared lookup="alerts/table-configure-no-data-and-error.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
For more details, refer to [alert instance states](ref:alert-instance-state) and [modify the no data or error state](ref:modify-the-no-data-or-error-state).
|
||||
|
||||
## Configure labels and notifications
|
||||
|
||||
@@ -213,7 +260,7 @@ Complete the following steps to set up labels and notifications.
|
||||
|
||||
**Select contact point**
|
||||
|
||||
1. Choose this option to select an existing contact point.
|
||||
1. Choose this option to select an existing [contact point](ref:contact-points).
|
||||
|
||||
All notifications for this alert rule are sent to this contact point automatically and notification policies are not used.
|
||||
|
||||
@@ -224,71 +271,18 @@ Complete the following steps to set up labels and notifications.
|
||||
|
||||
**Use notification policy**
|
||||
|
||||
3. Choose this option to use the notification policy tree to direct your notifications.
|
||||
1. Choose this option to use the [notification policy tree](ref:notification-policies) to direct your notifications.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
All alert rules and instances, irrespective of their labels, match the default notification policy. If there are no nested policies, or no nested policies match the labels in the alert rule or alert instance, then the default notification policy is the matching policy.
|
||||
{{< /admonition >}}
|
||||
|
||||
4. Preview your alert instance routing set up.
|
||||
2. Preview your alert instance routing set up.
|
||||
|
||||
Based on the labels added, alert instances are routed to the following notification policies displayed.
|
||||
|
||||
5. Expand each notification policy below to view more details.
|
||||
3. Expand each notification policy below to view more details.
|
||||
|
||||
6. Click **See details** to view alert routing details and an email preview.
|
||||
4. Click **See details** to view alert routing details and an email preview.
|
||||
|
||||
## Add annotations
|
||||
|
||||
Add [annotations](ref:annotation-label). to provide more context on the alert in your alert notification message.
|
||||
|
||||
Annotations add metadata to provide more information on the alert in your alert notification message. For example, add a **Summary** annotation to tell you which value caused the alert to fire or which server it happened on.
|
||||
|
||||
1. Optional: Add a summary.
|
||||
|
||||
Short summary of what happened and why.
|
||||
|
||||
1. Optional: Add a description.
|
||||
|
||||
Description of what the alert rule does.
|
||||
|
||||
1. Optional: Add a Runbook URL.
|
||||
|
||||
Webpage where you keep your runbook for the alert
|
||||
|
||||
1. Optional: Add a custom annotation
|
||||
1. Optional: Add a **dashboard and panel link**.
|
||||
|
||||
Links alert rules to panels in a dashboard.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
At the moment, alert rules are only supported in [time series](ref:time-series) and [alert list](ref:alert-list) visualizations.
|
||||
{{% /admonition %}}
|
||||
|
||||
1. Click **Save rule**.
|
||||
|
||||
## Configure no data and error handling
|
||||
|
||||
In **Configure no data and error handling**, you can define the alerting behavior when the evaluation returns no data or an error.
|
||||
|
||||
For details about alert states, refer to [lifecycle of alert instances](ref:alert-instance-state).
|
||||
|
||||
You can configure the alert instance state when its evaluation returns no data:
|
||||
|
||||
| No Data configuration | Description |
|
||||
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| No Data | The default option. Sets alert instance state to `No data`. <br/> The alert rule also creates a new alert instance `DatasourceNoData` with the name and UID of the alert rule, and UID of the datasource that returned no data as labels. |
|
||||
| Alerting | Sets the alert instance state to `Pending` and then transitions to `Alerting` once the [pending period](ref:pending-period) ends. If you sent the pending period to 0, the alert instance state is immediately set to `Alerting`. |
|
||||
| Normal | Sets alert instance state to `Normal`. |
|
||||
| Keep Last State | Maintains the alert instance in its last state. Useful for mitigating temporary issues, refer to [Keep last state](ref:keep-last-state). |
|
||||
|
||||
You can also configure the alert instance state when its evaluation returns an error or timeout.
|
||||
|
||||
| Error configuration | Description |
|
||||
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Error | The default option. Sets alert instance state to `Error`. <br/> The alert rule also creates a new alert instance `DatasourceError` with the name and UID of the alert rule, and UID of the datasource that returned no data as labels. |
|
||||
| Alerting | Sets alert instance state to `Alerting`. It transitions from `Pending` to `Alerting` after the [pending period](ref:pending-period) has finished. |
|
||||
| Normal | Sets alert instance state to `Normal`. |
|
||||
| Keep Last State | Maintains the alert instance in its last state. Useful for mitigating temporary issues, refer to [Keep last state](ref:keep-last-state). |
|
||||
|
||||
When you configure the No data or Error behavior to `Alerting` or `Normal`, Grafana will attempt to keep a stable set of fields under notification `Values`. If your query returns no data or an error, Grafana re-uses the latest known set of fields in `Values`, but will use `-1` in place of the measured value.
|
||||
{{< docs/shared lookup="alerts/configure-notification-message.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/
|
||||
- ../unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
description: Create recording rules for an external Grafana Mimir or Loki instance
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- recording rules
|
||||
- configure
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure recording rules
|
||||
weight: 300
|
||||
refs:
|
||||
configure-grafana:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/
|
||||
annotation-label:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
---
|
||||
|
||||
# Configure recording rules
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
In Grafana Cloud, you can only create data source-managed recording rules.
|
||||
|
||||
In Grafana OSS and Enterprise, you can create both Grafana-managed and data source-managed recording rules if you enable the `grafanaManagedRecordingRules` feature flag.
|
||||
|
||||
For more information on enabling feature toggles, refer to [Configure feature toggles](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/feature-toggles).
|
||||
{{< /admonition >}}
|
||||
|
||||
Recording rules calculate frequently needed expressions or computationally expensive expressions in advance and save the result as a new set of time series. Querying this new time series is faster, especially for dashboards since they query the same expression every time the dashboards refresh.
|
||||
|
||||
For more information on recording rules in Prometheus, refer to [Defining recording rules in Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
|
||||
|
||||
Recording rules are run as instant rules, which means that they run every 10s. To overwrite this configuration, update the min_interval in your custom configuration file.
|
||||
|
||||
[min_interval](ref:configure-grafana) sets the minimum interval to enforce between rule evaluations. The default value is 10s which equals the scheduler interval. Rules will be adjusted if they are less than this value or if they are not multiple of the scheduler interval (10s). Higher values can help with resource management as fewer evaluations are scheduled over time.
|
||||
|
||||
This setting has precedence over each individual rule frequency. If a rule frequency is lower than this value, then this value is enforced.
|
||||
|
||||
## Configure data source-managed recording rules
|
||||
|
||||
Configure data source-managed recording rules.
|
||||
|
||||
### Before you begin
|
||||
|
||||
- Verify that you have write permission to the Prometheus or Loki data source. Otherwise, you will not be able to create or update Grafana Mimir managed alerting rules.
|
||||
|
||||
- For Grafana Mimir and Loki data sources, enable the ruler API by configuring their respective services.
|
||||
|
||||
- **Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types.
|
||||
|
||||
- **Grafana Mimir** - use the `/prometheus` prefix. The Prometheus data source supports both Grafana Mimir and Prometheus, and Grafana expects that both the [Query API](/docs/mimir/latest/operators-guide/reference-http-api/#querier--query-frontend) and [Ruler API](/docs/mimir/latest/operators-guide/reference-http-api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API.
|
||||
|
||||
To configure data-source managed recording rules, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting** ->
|
||||
**Alert rules**.
|
||||
1. Scroll to the **Data source-managed section** and click **+New recording rule**.
|
||||
|
||||
#### Enter recording rule name
|
||||
|
||||
The recording rule name must be a Prometheus metric name and contain no whitespace.
|
||||
|
||||
#### Define recording rule
|
||||
|
||||
Select your data source and enter a query.
|
||||
|
||||
#### Add namespace and group
|
||||
|
||||
1. From the **Namespace** dropdown, select an existing rule namespace or add a new one.
|
||||
|
||||
Namespaces can contain one or more rule groups and only have an organizational purpose.
|
||||
|
||||
1. From the **Group** dropdown, select an existing group within the selected namespace or add a new one.
|
||||
|
||||
Newly created rules are appended to the end of the group. Rules within a group are run sequentially at a regular interval, with the same evaluation time.
|
||||
|
||||
#### Add labels
|
||||
|
||||
1. Add custom labels selecting existing key-value pairs from the drop down, or add new labels by entering the new key or value.
|
||||
|
||||
1. Click **Save rule** to save the rule or **Save rule and exit** to save the rule and go back to the Alerting page.
|
||||
|
||||
## Configure Grafana-managed recording rules
|
||||
|
||||
Configure Grafana-managed recording rules.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
This feature is only available for Grafana OSS and Enterprise users. It is not available in Grafana Cloud.
|
||||
{{< /admonition >}}
|
||||
|
||||
### Before you begin
|
||||
|
||||
- Enable the `grafanaManagedRecordingRules` [feature flag](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/feature-toggles/).
|
||||
|
||||
To configure Grafana-managed recording rules, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting** ->
|
||||
**Alert rules**.
|
||||
1. Scroll to the **Grafana-managed section** and click **+New recording rule**.
|
||||
|
||||
#### Enter a recording rule and metric name
|
||||
|
||||
Enter a names to identify your recording rule and metric. The metric name must be a Prometheus metric name and contain no whitespace.
|
||||
|
||||
For more information, refer to [Metrics and labels](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels).
|
||||
|
||||
#### Define recording rule
|
||||
|
||||
Define a query to get the data you want to measure and a condition that needs to be met before an alert rule fires.
|
||||
|
||||
1. Select a data source.
|
||||
1. From the **Options** dropdown, specify a time range.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Grafana Alerting only supports fixed relative time ranges, for example, `now-24hr: now`.
|
||||
|
||||
It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:59:592` or semi-relative time ranges: `now/d to: now`.
|
||||
{{< /admonition >}}
|
||||
|
||||
1. Add a query.
|
||||
|
||||
To add multiple queries, click **Add query**.
|
||||
|
||||
All alert rules are managed by Grafana by default. If you want to switch to a data source-managed alert rule, click **Switch to data source-managed alert rule**.
|
||||
|
||||
2. Add one or more [expressions].
|
||||
|
||||
a. For each expression, select either **Classic condition** to create a single alert rule, or choose from the **Math**, **Reduce**, and **Resample** options to generate separate alert for each series.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
When using Prometheus, you can use an instant vector and built-in functions, so you don't need to add additional expressions.
|
||||
{{% /admonition %}}
|
||||
|
||||
b. Click **Preview** to verify that the expression is successful.
|
||||
|
||||
3. To add a recovery threshold, turn the **Custom recovery threshold** toggle on and fill in a value for when your alert rule should stop firing.
|
||||
|
||||
You can only add one recovery threshold in a query and it must be the alert condition.
|
||||
|
||||
4. Click **Set as alert condition** on the query or expression you want to set as your alert condition.
|
||||
|
||||
#### Set evaluation behavior
|
||||
|
||||
Use alert rule evaluation to determine how frequently an alert rule should be evaluated and how quickly it should change its state.
|
||||
|
||||
To do this, you need to make sure that your alert rule is in the right evaluation group and set a pending period time that works best for your use case.
|
||||
|
||||
1. Select a folder or click **+ New folder**.
|
||||
1. Select an evaluation group or click **+ New evaluation group**.
|
||||
|
||||
If you are creating a new evaluation group, specify the interval for the group.
|
||||
|
||||
All rules within the same group are evaluated concurrently over the same time interval.
|
||||
|
||||
1. Enter a pending period.
|
||||
|
||||
The pending period is the period in which an alert rule can be in breach of the condition until it fires.
|
||||
|
||||
Once a condition is met, the alert goes into the **Pending** state. If the condition remains active for the duration specified, the alert transitions to the **Firing** state, else it reverts to the **Normal** state.
|
||||
|
||||
1. Turn on pause alert notifications, if required.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
You can pause alert rule evaluation to prevent noisy alerting while tuning your alerts.
|
||||
Pausing stops alert rule evaluation and doesn't create any alert instances.
|
||||
This is different to mute timings, which stop notifications from being delivered, but still allows for alert rule evaluation and the creation of alert instances.
|
||||
{{< /admonition >}}
|
||||
|
||||
#### Add labels
|
||||
|
||||
Add labels to your rule for searching, silencing, or routing to a notification policy.
|
||||
@@ -1,153 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/
|
||||
- ../unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
- ../unified-alerting/alerting-rules/create-mimir-loki-managed-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-mimir-loki-managed-rule/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-mimir-loki-managed-rule/
|
||||
description: Configure data source-managed alert rules alert for an external Grafana Mimir or Loki instance
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- create
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure data source-managed alert rules
|
||||
weight: 200
|
||||
refs:
|
||||
alerting:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/
|
||||
annotation-label:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
alert-list:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/alert-list/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/alert-list/
|
||||
time-series:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/time-series/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series/
|
||||
---
|
||||
|
||||
# Configure data source-managed alert rules
|
||||
|
||||
Create data source-managed alert rules for Grafana Mimir or Grafana Loki data sources, which have been configured to support rule creation.
|
||||
|
||||
To configure your Grafana Mimir or Loki data source for alert rule creation, enable either the Loki Ruler API or the Mimir Ruler API.
|
||||
|
||||
For more information, refer to [Loki Ruler API](/docs/loki/<GRAFANA_VERSION>/api/#ruler) or [Mimir Ruler API](/docs/mimir/<GRAFANA_VERSION>/references/http-api/#ruler).
|
||||
|
||||
**Note**:
|
||||
|
||||
Alert rules for a Grafana Mimir or Loki instance can be edited or deleted by users with Editor or Admin roles.
|
||||
|
||||
If you delete an alerting resource created in the UI, you can no longer retrieve it.
|
||||
To make a backup of your configuration and to be able to restore deleted alerting resources, create your alerting resources using file provisioning, Terraform, or the Alerting API.
|
||||
|
||||
## Before you begin
|
||||
|
||||
- Verify that you have write permission to the Mimir or Loki data source. Otherwise, you cannot create or update Grafana Mimir or Loki-managed alert rules.
|
||||
|
||||
- Enable the Mimir or Loki Ruler API.
|
||||
|
||||
- **Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types.
|
||||
|
||||
- **Grafana Mimir** - use the `/prometheus` prefix. The Prometheus data source supports both Grafana Mimir and Prometheus, and Grafana expects that both the [Query API](/docs/mimir/latest/operators-guide/reference-http-api/#querier--query-frontend) and [Ruler API](/docs/mimir/latest/operators-guide/reference-http-api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API.
|
||||
|
||||
Watch this video to learn more about how to create a Mimir-managed alert rule: {{< vimeo 720001865 >}}
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
If you do not want to manage alert rules for a particular Loki or Mimir data source, go to its settings and clear the **Manage alerts via Alerting UI** checkbox.
|
||||
{{% /admonition %}}
|
||||
|
||||
To create a data source-managed alert rule, use the in-product alert creation flow and follow these steps to help you.
|
||||
|
||||
## Set alert rule name
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alert rules** -> **+ New alert rule**.
|
||||
1. Enter a name to identify your alert rule.
|
||||
|
||||
This name is displayed in the alert rule list. It is also the `alertname` label for every alert instance that is created from this rule.
|
||||
|
||||
## Define query and condition
|
||||
|
||||
Define a query to get the data you want to measure and a condition that needs to be met before an alert rule fires.
|
||||
|
||||
**Note**:
|
||||
|
||||
All alert rules are managed by Grafana by default. To switch to a data source-managed alert rule, click **Switch to data source-managed alert rule**.
|
||||
|
||||
1. Select a data source from the drop-down list.
|
||||
|
||||
You can also click **Open advanced data source picker** to see more options, including adding a data source (Admins only).
|
||||
|
||||
1. Enter a PromQL or LogQL query.
|
||||
1. Click **Preview alerts**.
|
||||
|
||||
## Set alert evaluation behavior
|
||||
|
||||
Use alert rule evaluation to determine how frequently an alert rule should be evaluated and how quickly it should change its state.
|
||||
|
||||
1. Select a namespace or click **+ New namespace**.
|
||||
1. Select an evaluation group or click **+ New evaluation group**.
|
||||
|
||||
If you are creating a new evaluation group, specify the interval for the group.
|
||||
|
||||
All rules within the same group are evaluated sequentially over the same time interval.
|
||||
|
||||
1. Enter a pending period.
|
||||
|
||||
The pending period is the period in which an alert rule can be in breach of the condition until it fires.
|
||||
|
||||
Once a condition is met, the alert goes into the **Pending** state. If the condition remains active for the duration specified, the alert transitions to the **Firing** state, else it reverts to the **Normal** state.
|
||||
|
||||
## Configure notifications
|
||||
|
||||
Add labels to your alert rules to set which notification policy should handle your firing alert instances.
|
||||
|
||||
All alert rules and instances, irrespective of their labels, match the default notification policy. If there are no nested policies, or no nested policies match the labels in the alert rule or alert instance, then the default notification policy is the matching policy.
|
||||
|
||||
1. Add labels if you want to change the way your notifications are routed.
|
||||
|
||||
Add custom labels by selecting existing key-value pairs from the drop down, or add new labels by entering the new key or value.
|
||||
|
||||
## Add annotations
|
||||
|
||||
Add [annotations](ref:annotation-label). to provide more context on the alert in your alert notifications.
|
||||
|
||||
Annotations add metadata to provide more information on the alert in your alert notifications. For example, add a **Summary** annotation to tell you which value caused the alert to fire or which server it happened on.
|
||||
|
||||
1. Optional: Add a summary.
|
||||
|
||||
Short summary of what happened and why.
|
||||
|
||||
1. Optional: Add a description.
|
||||
|
||||
Description of what the alert rule does.
|
||||
|
||||
1. Optional: Add a Runbook URL.
|
||||
|
||||
Webpage where you keep your runbook for the alert
|
||||
|
||||
1. Optional: Add a custom annotation
|
||||
1. Optional: Add a **dashboard and panel link**.
|
||||
|
||||
Links alerts to panels in a dashboard.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
At the moment, alert rules are only supported in [time series](ref:time-series) and [alert list](ref:alert-list) visualizations.
|
||||
{{% /admonition %}}
|
||||
|
||||
1. Click **Save rule**.
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
aliases:
|
||||
- ../fundamentals/alert-rules/recording-rules/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/recording-rules/
|
||||
- ../unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-recording-rule/
|
||||
- ../unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
- ../alerting-rules/create-mimir-loki-managed-recording-rule/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-recording-rules/
|
||||
description: Recording rules allow you to pre-compute frequently needed or computationally expensive expressions and save the results as a new set of time series. Querying precomputed results is faster and can reduce system load.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- recording rules
|
||||
- configure
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create recording rules
|
||||
weight: 400
|
||||
refs:
|
||||
grafana-managed-recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/create-grafana-managed-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/create-grafana-managed-recording-rules/
|
||||
data-source-managed-recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/create-data-source-managed-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/create-data-source-managed-recording-rules/
|
||||
---
|
||||
|
||||
# Configure recording rules
|
||||
|
||||
Recording rules allows you to periodically pre-compute frequently used or computationally expensive queries, saving the results as a new time series metric.
|
||||
|
||||
For instance, you can create a recording rule generating a new metric, `error_9001_count`, which counts occurrences of a specific log error within one minute. Then, query the `error_9001_count` metric in dashboards and alert rules.
|
||||
|
||||
Recording rules can be helpful in various scenarios, such as:
|
||||
|
||||
- **Faster queries** are needed: Performing heavy aggregations or querying large data sets is quicker with precomputed results than real-time queries.
|
||||
- **Reducing system load:** Precomputing specific queries in advance can reduce system overload caused by multiple simultaneous queries.
|
||||
- **Simplifying complex aggregations:** Create a new metric from complex aggregations to facilitate alert and dashboard setup.
|
||||
- **Reusing queries across alerts:** Improve efficiency by reusing the same query across similar alert rules and dashboards.
|
||||
|
||||
The evaluation group of the recording rule determines how often the metric is pre-computed.
|
||||
|
||||
Similar to alert rules, Grafana supports two types of recording rules:
|
||||
|
||||
1. [Grafana-managed recording rules](ref:grafana-managed-recording-rules), which can query any Grafana data source supported by alerting.
|
||||
2. [Data source-managed recording rules](ref:data-source-managed-recording-rules), which can query Prometheus-based data sources like Mimir or Loki.
|
||||
@@ -0,0 +1,81 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-recording-rules/create-data-source-managed-recording-rules/
|
||||
description: Recording rules allow you to pre-compute expensive queries in advance and save the results as a new set of time series. Data source-managed recording rules can create a recording rule for Prometheus-based data sources like Mimir or Loki.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- recording rules
|
||||
- configure
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create data source-managed recording rules
|
||||
weight: 402
|
||||
refs:
|
||||
create-recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/
|
||||
---
|
||||
|
||||
# Create data source-managed recording rules
|
||||
|
||||
[Recording rules](ref:create-recording-rules) allow you to periodically pre-compute frequently used or computationally expensive queries, saving the results as a new time series metric.
|
||||
|
||||
Alert rules and dashboards can then query the new metric resulting from the recording rule. This is faster than querying real-time data and can help to reduce system load.
|
||||
|
||||
Data source-managed recording rules can query Prometheus-based data sources like Mimir or Loki. For more information on recording rules in Prometheus, refer to [Defining recording rules in Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
|
||||
|
||||
Note that in data source-managed groups, the alert rules and recording rules within the same evaluation group are evaluated sequentially. This is useful to ensure that a recording rule is evaluated before any other alert rule queries the pre-computed metric.
|
||||
|
||||
## Before you begin
|
||||
|
||||
- Verify that you have write permission to the Prometheus or Loki data source. Otherwise, you will not be able to create or update Grafana Mimir managed alerting rules.
|
||||
|
||||
- For Grafana Mimir and Loki data sources, enable the ruler API by configuring their respective services.
|
||||
|
||||
- **Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types.
|
||||
|
||||
- **Mimir** - use the `/prometheus` prefix. The Prometheus data source supports both Grafana Mimir and Prometheus, and Grafana expects that both the [Query API](/docs/mimir/latest/operators-guide/reference-http-api/#querier--query-frontend) and [Ruler API](/docs/mimir/latest/operators-guide/reference-http-api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API.
|
||||
|
||||
## Add new recording rule
|
||||
|
||||
To create a new data source-managed recording rule:
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting** -> **Alert rules**.
|
||||
1. Scroll to the **Data source-managed section** and click **+New recording rule**.
|
||||
|
||||
## Enter recording rule name
|
||||
|
||||
The recording rule name must be a Prometheus metric name and contain no whitespace.
|
||||
|
||||
## Define recording rule
|
||||
|
||||
Select your data source and enter a query. The queries used in data source-managed recording rules always run as instant queries.
|
||||
|
||||
## Add namespace and group
|
||||
|
||||
1. From the **Namespace** dropdown, select an existing rule namespace or add a new one.
|
||||
|
||||
Namespaces can contain one or more rule groups and only have an organizational purpose.
|
||||
|
||||
1. From the **Group** dropdown, select an existing group within the selected namespace or add a new one.
|
||||
|
||||
Rules within a group are run sequentially at a regular interval, with the same evaluation time.
|
||||
|
||||
Newly created rules are appended to the end of the group, and you can reorder them from the **Alert rules** page.
|
||||
|
||||
## Add labels
|
||||
|
||||
Optionally, you can add custom labels to the resulting metric by selecting existing key-value pairs from the drop down or entering the new key or value.
|
||||
|
||||
## Query the new metric in dashboards or alert rules
|
||||
|
||||
Click **Save rule** or **Save rule and exit** to save the rule.
|
||||
|
||||
Once saved, the new recording metric is available for use in dashboards and alert rules.
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-recording-rules/create-grafana-managed-recording-rules/
|
||||
description: Recording rules allow you to pre-compute expensive queries in advance and save the results as a new set of time series. Grafana-managed recording rules can create a recording rule for any data source supported by alerting.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- recording rules
|
||||
- configure
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create Grafana-managed recording rules
|
||||
weight: 401
|
||||
refs:
|
||||
expressions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
create-recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/
|
||||
alerting-data-sources:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
configure-grafana-min-interval:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/#min_interval
|
||||
configure-feature-toggles:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/feature-toggles/
|
||||
---
|
||||
|
||||
# Create Grafana-managed recording rules
|
||||
|
||||
[Recording rules](ref:create-recording-rules) allow you to periodically pre-compute frequently used or computationally expensive queries, saving the results as a new time series metric.
|
||||
|
||||
Alert rules and dashboards can then query the new metric resulting from the recording rule. This is faster than querying real-time data and can help to reduce system load.
|
||||
|
||||
Grafana does not contain an embedded time-series database to store recording rule results. You must bring your own Prometheus-compatible database to store the series generated by recording rules.
|
||||
|
||||
Grafana-managed recording rules offer the same Prometheus-like semantics but allow you to query [data sources supported by alerting](ref:alerting-data-sources). Additionally, you can use recording rules to import and map data from other data sources into Prometheus.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
|
||||
Grafana-managed recording rules are enabled by default in Grafana Cloud.
|
||||
|
||||
In Grafana OSS and Enterprise, you must enable them by following the [Before you begin](#before-you-begin) instructions.
|
||||
|
||||
{{< /admonition >}}
|
||||
|
||||
To configure Grafana-managed recording rules, complete the following steps.
|
||||
|
||||
## Before you begin
|
||||
|
||||
This section only applies to Grafana OSS and Grafana Enterprise.
|
||||
|
||||
First, enable the `grafanaManagedRecordingRules` [feature flag](ref:configure-feature-toggles).
|
||||
|
||||
Then, enable the feature by setting `enabled = true` in the `[recording_rules]` section of the Grafana config .ini. Provide the URL of your Prometheus-compatible remote-write endpoint in the `url` field, along with optional credentials or headers.
|
||||
|
||||
```
|
||||
[recording_rules]
|
||||
enabled = true
|
||||
url = http://my-example-prometheus.local:9090/api/prom/push
|
||||
basic_auth_username = my-user
|
||||
basic_auth_password = my-pass
|
||||
|
||||
[recording_rules.custom_headers]
|
||||
X-My-Header = MyValue
|
||||
```
|
||||
|
||||
## Add new recording rule
|
||||
|
||||
To create a new Grafana-managed recording rule:
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting** ->
|
||||
**Alert rules**.
|
||||
1. Scroll to the **Grafana-managed section** and click **+New recording rule**.
|
||||
|
||||
1. Enter the names to identify your recording rule and metric.
|
||||
|
||||
The metric name must be a Prometheus metric name and contain no whitespace. For details, refer to [Prometheus metric names](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels).
|
||||
|
||||
## Define recording rule
|
||||
|
||||
Define a query to get the data you want to measure and set the recording rule output.
|
||||
|
||||
1. Select a data source.
|
||||
1. From the **Options** dropdown, specify a time range.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Grafana Alerting only supports fixed relative time ranges, for example, `now-24hr: now`.
|
||||
|
||||
It does not support absolute time ranges: `2021-12-02 00:00:00 to 2021-12-05 23:59:592` or semi-relative time ranges: `now/d to: now`.
|
||||
{{< /admonition >}}
|
||||
|
||||
1. Add a query.
|
||||
|
||||
To add multiple queries, click **Add query**.
|
||||
|
||||
1. Add one or more [expressions](ref:expressions).
|
||||
|
||||
a. For each expression, select either **Classic condition** to create a single recording rule, or choose from the **Math**, **Reduce**, and **Resample** options.
|
||||
|
||||
When using Prometheus, you can use an instant vector and built-in functions, so you don't need to add additional expressions.
|
||||
|
||||
b. Click **Preview** to verify that the expression is successful.
|
||||
|
||||
1. Click **Set as recording rule output** on the query or expression you want to set as your rule output.
|
||||
|
||||
## Set evaluation behavior
|
||||
|
||||
Use recording rule evaluation to determine how frequently a recording rule should be evaluated.
|
||||
|
||||
To do this, you need to make sure that your recording rule is in the right evaluation group with an interval that works best for your use case.
|
||||
|
||||
1. Select a folder or click **+ New folder**.
|
||||
1. Select an evaluation group or click **+ New evaluation group**.
|
||||
|
||||
If you are creating a new evaluation group, specify the interval for the group.
|
||||
|
||||
All rules within the same group are evaluated concurrently over the same time interval. Every recording rule in a group uses the same evaluation time, meaning that all queries from the same group are always aligned with each other.
|
||||
|
||||
1. Before or after creating a recording rule, you have the option to **Pause evaluation** if necessary.
|
||||
|
||||
### Advanced configuration
|
||||
|
||||
[min_interval](ref:configure-grafana-min-interval) sets the minimum interval to enforce between rule evaluations. The default value is 10s which equals the scheduler interval. Rules are adjusted if they are less than this value or if they are not multiple of the scheduler interval (10s). Higher values can help with resource management as fewer evaluations are scheduled over time.
|
||||
|
||||
This setting has precedence over each individual rule frequency. If a rule frequency is lower than this value, then this value is enforced.
|
||||
|
||||
This setting applies to both Grafana-managed alert and recording rules.
|
||||
|
||||
## Add labels
|
||||
|
||||
Optionally, you can add custom labels to the resulting metric by selecting existing key-value pairs from the drop down or entering the new key or value.
|
||||
|
||||
## Query the new metric in dashboards or alert rules
|
||||
|
||||
Click **Save rule** or **Save rule and exit** to save the rule.
|
||||
|
||||
Once saved, the new recording metric is available for use in dashboards and alert rules.
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/alerting-rules/create-alerts-panels/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-alerts-panels/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
description: Grafana allows you to link alert rules with panels and dashboards. This helps connect alerts with an existing dashboard and informs alert responders on where to investigate.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- panels
|
||||
- create
|
||||
- grafana-managed
|
||||
- data source-managed
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create and link alert rules to panels
|
||||
weight: 300
|
||||
refs:
|
||||
time-series-visualizations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/time-series/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series/
|
||||
annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
view-alert-state-on-panels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/monitor-status/view-alert-state/#view-alert-state-on-panels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/monitor-status/view-alert-state/#view-alert-state-on-panels
|
||||
---
|
||||
|
||||
# Create and link alert rules to panels
|
||||
|
||||
Grafana allows you to link an alert rule to a dashboard panel. This can help you:
|
||||
|
||||
- Inform alert responders about where to investigate and which data to examine.
|
||||
- Visualize the alert state directly from dashboards.
|
||||
|
||||
An alert rule is linked to a panel by setting the [`dashboardUId` and `panelId` annotations](ref:annotations). Both annotations must be set together.
|
||||
|
||||
## Link alert rules to panels
|
||||
|
||||
When configuring the alert rule, you can set the dashboard and panel annotations as shown in this [video](https://youtu.be/ClLp-iSoaSY?si=qKWnvSVaQuvYcuw9&t=170).
|
||||
|
||||
1. Configure the alert rule.
|
||||
1. In the **Configure notification message** section, click **Link dashboard and panel**.
|
||||
1. Select an existing dashboard, then choose a panel from the selected dashboard.
|
||||
1. Complete the alert rule configuration and click **Save rule** to initiate the alert rule.
|
||||
|
||||
You can then [view the alert state on the panel](ref:view-alert-state-on-panels).
|
||||
|
||||
{{< figure src="/media/docs/alerting/panel-displays-alert-state.png" max-width="1200px" caption="A panel displaying the alert status and state changes." >}}
|
||||
|
||||
## Create alert rules from panels
|
||||
|
||||
To streamline alert creation, you can create an alert rule directly from a panel.
|
||||
|
||||
1. Navigate to a dashboard in the **Dashboards** section.
|
||||
1. Hover over the top-right corner of a panel and click the panel menu icon.
|
||||
1. From the dropdown menu, select **More...** > **New alert rule**.
|
||||
1. This opens the **Edit rule** form and pre-fills some values:
|
||||
- Sets the `dashboardUId` and `panelId` annotations to the corresponding dashboard and panel.
|
||||
- Sets the alert rule query using the panel query.
|
||||
1. Complete the alert rule configuration and click **Save rule** to initiate the alert rule.
|
||||
|
||||
You can then [view the alert state on the panel](ref:view-alert-state-on-panels).
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Changes to panel and alert rule queries aren't synchronized. If you change a query, you have to update it in both the panel and the alert rule.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Access linked alert rules from panels
|
||||
|
||||
This option is available only in [time series panels](ref:time-series-visualizations). To access alert rules associated to a time series panel, complete the following steps.
|
||||
|
||||
1. Hover over the top-right corner of the panel and click the panel menu icon.
|
||||
1. Click **Edit**.
|
||||
1. Click the **Alert** tab to view existing alert rules or create a new one.
|
||||
224
docs/sources/alerting/alerting-rules/templates/_index.md
Normal file
224
docs/sources/alerting/alerting-rules/templates/_index.md
Normal file
@@ -0,0 +1,224 @@
|
||||
---
|
||||
aliases:
|
||||
- ../fundamentals/annotation-label/variables-label-annotation/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/annotation-label/variables-label-annotation/
|
||||
- ../alerting-rules/templating-labels-annotations/ # /docs/grafana/<GRAFANA_VERSION>/alerting-rules/templating-labels-annotations/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/templates/
|
||||
description: Learn how to template annotations and labels to include data from queries and expressions in alert messages
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templating
|
||||
- labels
|
||||
- annotations
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Template annotations and labels
|
||||
weight: 500
|
||||
refs:
|
||||
reference-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#labels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#labels
|
||||
reference-values:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#values
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#values
|
||||
annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
explore:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/explore/
|
||||
intro-to-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/templates/
|
||||
alert-rule-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/
|
||||
alert-rule-template-examples:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/examples/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/examples/
|
||||
notification-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
notification-data-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
view-alert-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/monitor-status/view-alert-state/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/monitor-status/view-alert-state/
|
||||
preview-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/manage-notification-templates/#preview-notification-templates
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/manage-notification-templates/#preview-notification-templates
|
||||
labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/latest/alerting/fundamentals/alert-rules/annotation-label/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
---
|
||||
|
||||
# Template annotations and labels
|
||||
|
||||
You can use templates to customize alert and notification messages, including dynamic data from alert rule queries.
|
||||
|
||||
In Grafana Alerting, you can template alert messages in two ways.
|
||||
|
||||
1. **Template annotations and labels**: In the alert rule definition, you can template annotations and labels to include extra information from query data to the alert, adding meaningful details based on the query results.
|
||||
1. **Template notifications**: You can template notifications to control the content and appearance of your notifications.
|
||||
|
||||
## How templating works
|
||||
|
||||
In this diagram, you can see the differences between both types of templates.
|
||||
|
||||
{{< figure src="/media/docs/alerting/how-notification-templates-works.png" max-width="1200px" alt="How templating works" >}}
|
||||
|
||||
Refer to [Templates Introduction](ref:intro-to-templates) for a more detailed explanation of this diagram.
|
||||
|
||||
Both types of templates are written in the Go templating system. However, it's important to understand that variables and functions used in notification templates are different from those used in annotation and label templates.
|
||||
|
||||
1. **Template annotations and labels**: These templates add extra information to individual alert instances. Template variables like [`$labels`](ref:reference-labels) and [`$values`](ref:reference-values) represent alert query data of the individual alert instance.
|
||||
1. **Template notifications**: Notification templates format the notification content for a group of alerts. Variables like [`.Alerts`](ref:notification-data-reference) include all firing and resolved alerts in the notification.
|
||||
|
||||
## Template annotations
|
||||
|
||||
[Annotations](ref:annotations) add additional information to alert instances and are often used to help identify the alert and guide responders on how to address the issue.
|
||||
|
||||
Annotations are key-value pairs defined in the alert rule. They can contain plain text or template code that is evaluated when the alert fires.
|
||||
|
||||
Grafana includes several optional annotations, such as `description`, `summary`, `runbook_url`, `dashboardUId` and `panelId`, which can be edited in the alert rule. You can also create your custom annotations. For example, you might create a new annotation named `location` to report the location of the system that triggered the alert.
|
||||
|
||||
Here’s an example of a `summary` annotation explaining why the alert was triggered, using plain text.
|
||||
|
||||
```
|
||||
CPU usage has exceeded 80% for the last 5 minutes.
|
||||
```
|
||||
|
||||
However, if you want to display dynamic query values in annotations, you need to use template code. Common use cases include:
|
||||
|
||||
- Displaying the query value that triggered the alert.
|
||||
- Highlighting label information that identifies the alert, such as the environment, instance, or region.
|
||||
- Providing specific instructions based on query values.
|
||||
- Customizing runbook links depending on query labels.
|
||||
- Including contact information based on query labels.
|
||||
|
||||
For instance, you can template the previous example to display the specific instance and CPU value that triggered the alert.
|
||||
|
||||
```
|
||||
CPU usage for {{ $labels.instance }} has exceeded 80% ({{ $values.A.Value }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
Alternatively, you can use the `index` function to print query values.
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% ({{ index $values "A" }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
The result of the annotation would be as follows.
|
||||
|
||||
```
|
||||
CPU usage for Instance 1 has exceeded 80% (81.2345) for the last 5 minutes.
|
||||
```
|
||||
|
||||
### How to template an annotation
|
||||
|
||||
Complete the following steps to template an annotation.
|
||||
|
||||
1. Navigate to **Alerts & IRM** -> **Alert rules** -> create or edit an alert rule.
|
||||
1. Scroll down to the **Configure notification message** section.
|
||||
1. Copy in your template in the corresponding annotation field (`summary`, `description`, `runbook_url`, `custom`).
|
||||
|
||||
### Preview annotation templates
|
||||
|
||||
You can template annotations when creating or editing an alert rule.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-rule-using-annotation-template.png" max-width="1200px" alt="An alert rule templating the annotation summary" >}}
|
||||
|
||||
Two common methods are used to test and preview annotation templates:
|
||||
|
||||
1. Trigger the alert and [view the alert instance state in the Grafana UI](ref:view-alert-state), where all annotations of the alert instance are displayed.
|
||||
1. Use a notification template that displays all annotations, then [preview the notification template](ref:preview-notifications) using the alert instance.
|
||||
|
||||
## Template labels
|
||||
|
||||
The set of [labels](ref:labels) for an alert instance is used to uniquely identify that alert among all other alert instances.
|
||||
|
||||
Labels determine how alerts are routed and managed for notifications, making their design key to the effectiveness of your alerting system.
|
||||
|
||||
Labels can be returned from an alert rule query, such as the `pod` label in a Kubernetes Prometheus query. You can also define additional labels in the alert rule to provide extra information for processing alerts.
|
||||
|
||||
Like annotations, labels are key-value pairs that can contain plain text or template code evaluated when the alert fires.
|
||||
|
||||
Template labels when the labels returned by your queries are insufficient. For instance:
|
||||
|
||||
- A new label based on a query value can group a subset of alerts differently, changing how notifications are sent.
|
||||
- A new label based on a query value can be used in a notification policy to alter the notification contact point.
|
||||
|
||||
Here’s an example of templating a `severity` label based on the query value.
|
||||
|
||||
```
|
||||
{{ if (gt $values.A.Value 90.0) -}}
|
||||
critical
|
||||
{{ else if (gt $values.A.Value 80.0) -}}
|
||||
high
|
||||
{{ else if (gt $values.A.Value 60.0) -}}
|
||||
medium
|
||||
{{ else -}}
|
||||
low
|
||||
{{- end }}
|
||||
```
|
||||
|
||||
In this example, the value of the `severity` label is determined by the query value, and the possible options are `critical`, `high`, `medium`, or `low`. You can then use the `severity` label to change their notifications—for instance, sending `critical` alerts immediately or routing `low` alerts to a specific team for further review.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
You should avoid displaying query values in labels, as this may create numerous unique alert instances—one for each distinct label value. Instead, use annotations for query values.
|
||||
{{% /admonition %}}
|
||||
|
||||
### How to template a label
|
||||
|
||||
Complete the following steps to template a label.
|
||||
|
||||
1. Navigate to **Alerts & IRM** -> **Alert rules** -> create or edit an alert rule.
|
||||
1. Scroll down to the **Configure labels and notifications** section.
|
||||
1. Click **+ Add labels**.
|
||||
1. Enter a **key** that identifies the label.
|
||||
1. Copy in your template in the **value** field.
|
||||
|
||||
### Preview label templates
|
||||
|
||||
You can template label values when creating or editing an alert rule.
|
||||
|
||||
To preview label values, select `Use notification policy`, and then click on `Preview routing`.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-instance-routing-preview.png" max-width="1200px" alt="Routing preview displays label values" >}}
|
||||
|
||||
## More information
|
||||
|
||||
For further details on how to template alert rules, refer to:
|
||||
|
||||
- [Annotation and label template reference](ref:alert-rule-template-reference)
|
||||
- [Annotation and label examples](ref:alert-rule-template-examples)
|
||||
|
||||
{{< admonition type="tip" >}}
|
||||
For a practical example of templating, refer to our [Getting Started with Templating tutorial](https://grafana.com/tutorials/alerting-get-started-pt4/).
|
||||
{{< /admonition >}}
|
||||
312
docs/sources/alerting/alerting-rules/templates/examples.md
Normal file
312
docs/sources/alerting/alerting-rules/templates/examples.md
Normal file
@@ -0,0 +1,312 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/templates/examples/
|
||||
description: Examples of templating labels and annotations in Grafana alert rules
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templating
|
||||
- labels
|
||||
- annotations
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Labels and annotations template examples
|
||||
menuTitle: Examples
|
||||
weight: 102
|
||||
refs:
|
||||
labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#labels
|
||||
annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
alert-rule-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/
|
||||
reference-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#labels
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#labels
|
||||
reference-values:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#values
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#values
|
||||
reference-humanize:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#humanize
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#humanize
|
||||
reference-humanizepercentage:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#humanizepercentage
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#humanizepercentage
|
||||
reference-match:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#match
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#match
|
||||
reference-functions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#functions
|
||||
language-functions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language/#functions
|
||||
language-index:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language/#functions
|
||||
language:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language
|
||||
- pattern: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language
|
||||
---
|
||||
|
||||
# Labels and annotations template examples
|
||||
|
||||
Templating allows you to add dynamic data from queries to alert labels and annotations. Dynamic data enhances alert context, making it easier for responders to quickly assess and address the issue.
|
||||
|
||||
This page provides common examples for templating labels and annotations. For more information on templating, refer to:
|
||||
|
||||
- [Template annotations and labels](ref:alert-rule-templates)
|
||||
- [Annotation and label template reference](ref:reference)
|
||||
- [Alerting template language](ref:language)
|
||||
|
||||
## Annotation example
|
||||
|
||||
[Annotations](ref:annotations) add extra details to alert instances and are often used to provide helpful information for identifying the issue and guiding the response.
|
||||
|
||||
A common use case for annotations is to display the specific query value or threshold that triggered the alert.
|
||||
|
||||
For example, you can display the query value from the [`$values`](ref:reference-values) variable to inform about the CPU value that triggered the alert.
|
||||
|
||||
```
|
||||
CPU usage has exceeded 80% ({{ $values.A.value }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
Alternatively, you can use the [`index()`](ref:language-index) function to retrieve the query value as follows.
|
||||
|
||||
```
|
||||
CPU usage has exceeded 80% ({{ index $values "A" }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
```template_output
|
||||
CPU usage has exceeded 80% (81.2345) for the last 5 minutes.
|
||||
```
|
||||
|
||||
### Include labels for extra details
|
||||
|
||||
To provide additional context, you can include labels from the query using the [`$labels`](ref:reference-labels) variable.
|
||||
|
||||
For instance, the previous case could also include the affected `instance`.
|
||||
|
||||
```
|
||||
CPU usage for {{ $labels.instance }} has exceeded 80% ({{ $values.A.Value }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
```template_output
|
||||
CPU usage for Instance 1 has exceeded 80% (81.2345) for the last 5 minutes.
|
||||
```
|
||||
|
||||
You can incorporate any labels returned by the query into the template. For instance, the following template includes information about the environment and region where the alert occurred.
|
||||
|
||||
```
|
||||
Alert triggered in {{ $labels.environment }} on {{ $labels.region }} region.
|
||||
```
|
||||
|
||||
```template_output
|
||||
Alert triggered in production on AMER region.
|
||||
```
|
||||
|
||||
### Print a range query
|
||||
|
||||
To print the value of an instant query you can print its Ref ID using the `index` function or the `$values` variable:
|
||||
|
||||
```
|
||||
{{ $values.A.Value }}
|
||||
```
|
||||
|
||||
For range queries, reduce them from a time series to an instant vector using a reduce expression. You can then print the result by referencing its Ref ID. For example, if the reduce expression averages `A` with the Ref ID `B`, you would then print `$values.B`:
|
||||
|
||||
```
|
||||
{{ $values.B.Value }}
|
||||
```
|
||||
|
||||
### Humanize the value of a query
|
||||
|
||||
To print the humanized value of an instant query, use the [`humanize`](ref:reference-humanize) function:
|
||||
|
||||
```
|
||||
{{ humanize $values.A.Value }}
|
||||
```
|
||||
|
||||
Alternatively:
|
||||
|
||||
```
|
||||
{{ humanize (index $values "A").Value }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
554.9
|
||||
```
|
||||
|
||||
To print the value of an instant query as a percentage, use the [`humanizePercentage`](ref:reference-humanizepercentage) function:
|
||||
|
||||
```
|
||||
{{ humanizePercentage $values.A.Value }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
10%
|
||||
```
|
||||
|
||||
For additional functions to display or format data, refer to:
|
||||
|
||||
- [Annotation and label template functions](ref:reference-functions)
|
||||
- [Template language functions](ref:language-functions)
|
||||
|
||||
## Label example
|
||||
|
||||
[Labels](ref:labels) determine how alerts are routed and managed, ensuring that notifications reach the right teams at the right time. If the labels returned by your queries don’t fully capture the necessary context, you can create a new label and sets its value based on query data.
|
||||
|
||||
### Based on query value
|
||||
|
||||
Here’s an example of creating a `severity` label based on a query value:
|
||||
|
||||
```go
|
||||
{{- if (gt $values.A.Value 90.0) -}}critical
|
||||
{{- else if (gt $values.A.Value 80.0) -}}high
|
||||
{{- else if (gt $values.A.Value 60.0) -}}medium
|
||||
{{- else -}}low
|
||||
{{- end -}}
|
||||
```
|
||||
|
||||
In this example, the `severity` label is determined by the query value:
|
||||
|
||||
- `critical` for values above 90,
|
||||
- `high` for values above 80,
|
||||
- `medium` for values above 60,
|
||||
- and `low` for anything below.
|
||||
|
||||
You can then use the `severity` label to control how alerts are handled. For instance, you could send `critical` alerts immediately, while routing `low` severity alerts to a team for further investigation.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
You should avoid displaying query values in labels, as this may create many alert instances—one for each distinct label value. Instead, use annotations to convey query values.
|
||||
{{% /admonition %}}
|
||||
|
||||
### Based on query label
|
||||
|
||||
You can use labels to differentiate alerts coming from various environments (e.g., production, staging, dev). For example, you may want to add a label that sets the environment based on the instance’s label. Here’s how you can template it:
|
||||
|
||||
```go
|
||||
{{- if eq $labels.instance "prod-server-1" -}}
|
||||
production
|
||||
{{- else if eq $labels.instance "staging-server-1" -}}
|
||||
staging
|
||||
{{- else -}}
|
||||
development
|
||||
{{- end -}}
|
||||
```
|
||||
|
||||
This would print:
|
||||
|
||||
- For instance `prod-server-1`, the label would be `production`.
|
||||
- For `staging-server-1`, the label would be `staging`.
|
||||
- All other instances would be labeled `development`.
|
||||
|
||||
To make this template more flexible, you can use a regular expression that matches the instance name with the instance name prefix using the [`match()`](ref:reference-match) function:
|
||||
|
||||
```go
|
||||
{{- if match "^prod-server-.*" $labels.instance -}}
|
||||
production
|
||||
{{- else if match "^staging-server-.*" $labels.instance -}}
|
||||
staging
|
||||
{{- else -}}
|
||||
development
|
||||
{{- end -}}
|
||||
```
|
||||
|
||||
{{< collapse title="Legacy Alerting templates" >}}
|
||||
|
||||
## Legacy Alerting templates
|
||||
|
||||
For users working with Grafana's legacy alerting system, templates can still be utilized to extract useful information from alert conditions. However, it's important to note that you cannot use `$labels` to print labels from the query if you are using classic conditions, and must use `$values` instead. The reason for this is classic conditions discard these labels to enforce uni-dimensional behavior (at most one alert per alert rule). If classic conditions didn't discard these labels, then queries that returned many time series would cause alerts to flap between firing and resolved constantly as the labels would change every time the alert rule was evaluated.
|
||||
|
||||
Instead, the `$values` variable contains the reduced values of all time series for all conditions that are firing. For example, if you have an alert rule with a query A that returns two time series, and a classic condition B with two conditions, then `$values` would contain `B0`, `B1`, `B2` and `B3`. If the classic condition B had just one condition, then `$values` would contain just `B0` and `B1`.
|
||||
|
||||
#### Print all labels from a classic condition
|
||||
|
||||
To print all labels of all firing time series use the following template (make sure to replace `B` in the regular expression with the Ref ID of the classic condition if it's different):
|
||||
|
||||
```go
|
||||
{{ range $k, $v := $values -}}
|
||||
{{ if (match "B[0-9]+" $k) -}}
|
||||
{{ $k }}: {{ $v.Labels }}{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
For example, a classic condition for two time series exceeding a single condition would print:
|
||||
|
||||
```
|
||||
B0: instance=server1
|
||||
B1: instance=server2
|
||||
```
|
||||
|
||||
If the classic condition has two or more conditions, and a time series exceeds multiple conditions at the same time, then its labels will be duplicated for each condition that is exceeded:
|
||||
|
||||
```
|
||||
B0: instance=server1
|
||||
B1: instance=server2
|
||||
B2: instance=server1
|
||||
B3: instance=server2
|
||||
```
|
||||
|
||||
If you need to print unique labels you should consider changing your alert rules from uni-dimensional to multi-dimensional instead. You can do this by replacing your classic condition with reduce and math expressions.
|
||||
|
||||
#### Print all values from a classic condition
|
||||
|
||||
To print all values from a classic condition take the previous example and replace `$v.Labels` with `$v.Value`:
|
||||
|
||||
```go
|
||||
{{ range $k, $v := $values -}}
|
||||
{{ if (match "B[0-9]+" $k) -}}
|
||||
{{ $k }}: {{ $v.Value }}{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
For example, a classic condition for two time series exceeding a single condition would print:
|
||||
|
||||
```
|
||||
B0: 81.2345
|
||||
B1: 84.5678
|
||||
```
|
||||
|
||||
If the classic condition has two or more conditions, and a time series exceeds multiple conditions at the same time, then `$values` will contain the values of all conditions:
|
||||
|
||||
```
|
||||
B0: 81.2345
|
||||
B1: 92.3456
|
||||
B2: 84.5678
|
||||
B3: 95.6789
|
||||
```
|
||||
|
||||
{{< /collapse >}}
|
||||
81
docs/sources/alerting/alerting-rules/templates/language.md
Normal file
81
docs/sources/alerting/alerting-rules/templates/language.md
Normal file
@@ -0,0 +1,81 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/language/
|
||||
description: Use Go template language to create your notification and alert rule templates
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templates
|
||||
- write templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Alerting template language
|
||||
menuTitle: Template language
|
||||
refs:
|
||||
alert-rule-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/
|
||||
alert-rule-template-reference-variables:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#variables
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#variables
|
||||
notification-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
reference-notificationdata:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
---
|
||||
|
||||
# Alerting template language
|
||||
|
||||
Notification templates and alert rule templates, such as annotations and labels, both use the Go template language, [text/template](https://pkg.go.dev/text/template).
|
||||
|
||||
Both types of templates can use the same keywords, functions, and comparison operators of the Go template language, such as `range`, `if`, `and`, `index`, `eq`, and more.
|
||||
|
||||
However, it's important to note that because notifications and alert rules operate in distinct contexts, some additional variables and functions are only available for either notification or alert rule templates. Refer to:
|
||||
|
||||
- [Annotation and label template reference](ref:alert-rule-template-reference)
|
||||
- [Notification template reference](ref:notification-template-reference)
|
||||
|
||||
This documentation provides an overview of the functions and operators of the Go template language that are available for both notification and alert rule templates.
|
||||
|
||||
## Print
|
||||
|
||||
To print the value of something, use `{{` and `}}`. You can print the value of a [variable](#variables), a field of a variable, the result of a function, or the value of dot.
|
||||
|
||||
```go
|
||||
{{ $values }}
|
||||
{{ $values.A.Value }}
|
||||
{{ humanize 1000.0 }}
|
||||
{{ .Alerts }}
|
||||
```
|
||||
|
||||
## Dot
|
||||
|
||||
In `text/template`, there is a special cursor called dot, written as `.`. You can think of this cursor as a variable whose value changes depending on where in the template it is used.
|
||||
|
||||
At the start of notification templates, dot (`.`) refers to [Notification Data](ref:reference-notificationdata).
|
||||
|
||||
```go
|
||||
{{ .Alerts }}
|
||||
```
|
||||
|
||||
In annotation and label templates, dot (`.`) is initialized with all alert data. It’s recommended to use the [`$labels` and `$values` variables](ref:alert-rule-template-reference-variables) instead to directly access the alert labels and query values.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Dot (`.`) might refer to something else when used in a [range](#range), a [with](#with), or when writing [templates](#templates) used in other templates.
|
||||
{{% /admonition %}}
|
||||
|
||||
[//]: <> (The above section is not included in the shared file because `refs` links are not supported in shared files.)
|
||||
|
||||
{{< docs/shared lookup="alerts/template-language.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
455
docs/sources/alerting/alerting-rules/templates/reference.md
Normal file
455
docs/sources/alerting/alerting-rules/templates/reference.md
Normal file
@@ -0,0 +1,455 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/templates/reference/
|
||||
description: Reference for variables and functions in Grafana alert rule templating.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templating
|
||||
- labels
|
||||
- annotations
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Annotation and label template reference
|
||||
menuTitle: Template reference
|
||||
weight: 101
|
||||
refs:
|
||||
label-types:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#label-types
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#label-types
|
||||
notification-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
language:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language/
|
||||
language-functions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language/#functions
|
||||
language-index:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/language/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/language/#functions
|
||||
print-all-labels-from-a-classic-condition:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/examples/#print-all-labels-from-a-classic-condition
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/examples/#print-all-labels-from-a-classic-condition
|
||||
template-annotations-and-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
explore:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/explore/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/explore/
|
||||
---
|
||||
|
||||
# Annotation and label template reference
|
||||
|
||||
Annotations and labels in alert rules can be defined using plain text. However, you can also define templates to customize their values with dynamic data from alert rule queries.
|
||||
|
||||
For example, you can template the `summary` annotation to include information from query values, providing relevant alert context for responders. Refer to [Template annotations and labels](ref:template-annotations-and-labels) for various use cases.
|
||||
|
||||
In templates, variables represent dynamic values from queries, while functions perform actions to transform or format this data.
|
||||
|
||||
## Variables
|
||||
|
||||
Variables represent dynamic values from alert rule queries that can be displayed or accessed in your templates.
|
||||
|
||||
The `$` and `.` symbols are used to reference variables and their properties. You can reference variables directly in your alert rule definitions using the `$` symbol followed by the variable name. Similarly, you can access properties of variables using the dot (`.`) notation in alert rule templates.
|
||||
|
||||
```
|
||||
{{ $values.A.Value }}
|
||||
```
|
||||
|
||||
Templates are based on the **Go templating system**. Refer to [Template language](ref:language) for additional information.
|
||||
|
||||
The following variables are available when templating annotations and labels:
|
||||
|
||||
| Variables | Description |
|
||||
| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| [$labels](#labels) | Contains all labels from the query, only query labels. |
|
||||
| [$values](#values) | Contains the labels and floating point values of all instant queries and expressions, indexed by their Ref IDs. |
|
||||
| [$value](#value) | A string containing the labels and values of all instant queries; threshold, reduce and math expressions, and classic conditions in the alert rule. |
|
||||
|
||||
### $labels
|
||||
|
||||
The `$labels` variable contains all labels from the query. It excludes [user-configured and reserved labels](ref:label-types), containing only query labels.
|
||||
|
||||
{{< figure src="/media/docs/alerting/query-labels-and-values.png" max-width="1200px" caption="An alert rule displaying labels and value from a query." >}}
|
||||
|
||||
For example, suppose you have a query that returns CPU usage for all of your servers, and you have an alert rule that fires when any of your servers have exceeded 80% CPU usage for the last 5 minutes. You want to add a summary annotation to the alert that tells you which server is experiencing high CPU usage. With the `$labels` variable you can write a template that prints a human-readable sentence such as:
|
||||
|
||||
```
|
||||
CPU usage for {{ $labels.instance }} has exceeded 80% for the last 5 minutes
|
||||
```
|
||||
|
||||
The outcome of this template would be:
|
||||
|
||||
```
|
||||
CPU usage for server1 has exceeded 80% for the last 5 minutes
|
||||
```
|
||||
|
||||
> If you are using a classic condition then `$labels` will not contain any labels from the query. Classic conditions discard these labels in order to enforce uni-dimensional behavior (at most one alert per alert rule). If you want to use labels from the query in your template then use the example [here](ref:print-all-labels-from-a-classic-condition).
|
||||
|
||||
### $values
|
||||
|
||||
The `$values` variable is a table containing the labels and floating point values of all instant queries and expressions, indexed by their Ref IDs (e.g, `A`, `B`, `C`, etc.). It does not contain the results of range queries, as they can return hundreds or thousands of rows.
|
||||
|
||||
Each Ref IDs, such as `$values.A`, has the following properties
|
||||
|
||||
| Property | Type | Description |
|
||||
| -------- | --------------- | ------------------------------------------------------------ |
|
||||
| `Value` | Float | The value returned by the instant query or expression. |
|
||||
| `Labels` | Key/value pairs | The labels associated with the instance query or expression. |
|
||||
|
||||
Here's the previous example printing now the value of the instant query with Ref ID `A`:
|
||||
|
||||
```
|
||||
{{ $values.A.Value }} CPU usage for {{ $labels.instance }} over the last 5 minutes.
|
||||
```
|
||||
|
||||
If the alert has the label `instance=server1` and the query returns `81.2345`, the template would print:
|
||||
|
||||
```
|
||||
81.2345 CPU usage for instance1 over the last 5 minutes.
|
||||
```
|
||||
|
||||
If the query in Ref ID `A` is a range query rather than an instant query then add a reduce expression with Ref ID `B` and replace `$values.A.Value` with `$values.B.Value`:
|
||||
|
||||
```
|
||||
{{ $values.B.Value }} CPU usage for {{ $labels.instance }} over the last 5 minutes.
|
||||
```
|
||||
|
||||
Alternatively, you can use the `index()` function to retrieve the query value:
|
||||
|
||||
```
|
||||
{{ index $values "B" }} CPU usage for {{ index $labels "instance" }} over the last 5 minutes.
|
||||
```
|
||||
|
||||
#### $value
|
||||
|
||||
The `$value` variable is a string containing the labels and values of all instant queries; threshold, reduce and math expressions, and classic conditions in the alert rule.
|
||||
|
||||
This example prints the `$value` variable:
|
||||
|
||||
```
|
||||
{{ $value }}: CPU usage has exceeded 80% for the last 5 minutes.
|
||||
```
|
||||
|
||||
It would display something like this:
|
||||
|
||||
```
|
||||
[ var='A' labels={instance=instance1} value=81.234 ]: CPU usage has exceeded 80% for the last 5 minutes.
|
||||
```
|
||||
|
||||
Instead, we recommend using [$values](#values), which contains the same information as `$value` but is structured in an easier-to-use table format.
|
||||
|
||||
## Functions
|
||||
|
||||
Functions can perform actions in templates such as transforming or formatting data.
|
||||
|
||||
Note that the [functions provided by Go's template language](ref:language-functions), such as `index`, `and`, `printf`, and `len`, are available, along with many others.
|
||||
|
||||
In addition, the following functions are also available for templating annotations and labels:
|
||||
|
||||
**Numbers**
|
||||
|
||||
| Name | Arguments | Returns | Description |
|
||||
| ----------------------------------------- | ---------------- | ------- | ---------------------------------------------------------------- |
|
||||
| [humanize](#humanize) | number or string | string | Humanizes decimal numbers. |
|
||||
| [humanize1024](#humanize1024) | number or string | string | Like `humanize`, but but uses 1024 as the base rather than 1000. |
|
||||
| [humanizeDuration](#humanizeduration) | number or string | string | Humanizes a duration in seconds. |
|
||||
| [humanizePercentage](#humanizepercentage) | number or string | string | Humanizes a ratio value to a percentage. |
|
||||
| [humanizeTimestamp](#humanizetimestamp) | number or string | string | Humanizes a Unix timestamp. |
|
||||
| [toTime](#totime) | number or string | time | Converts a Unix timestamp in seconds to time. |
|
||||
|
||||
**Strings**
|
||||
|
||||
| Name | Arguments | Returns | Description |
|
||||
| ------------------------------- | -------------------------- | ------- | --------------------------------------------------------------------------------------------- |
|
||||
| [title](#title) | string | string | Capitalizes the first character of each word. |
|
||||
| [toUpper](#toupper) | string | string | Returns all text in uppercase. |
|
||||
| [toLower](#tolower) | string | string | Returns all text in lowercase. |
|
||||
| [stripPort](#stripport) | string | string | Returns only host. |
|
||||
| [match](#match) | pattern, text | boolean | Matches the text against a regular expression pattern. |
|
||||
| [reReplaceAll](#rereplaceall) | pattern, replacement, text | string | Replaces text matching the regular expression. |
|
||||
| [graphLink](#graphlink) | expr | string | Returns the path to the graphical view in `Explore` for the given expression and data source. |
|
||||
| [tableLink](#tablelink) | expr | string | Returns the path to the tabular view in `Explore` for the given expression and data source. |
|
||||
| [parseDuration](#parseduration) | string | float | Parses a duration string such as "1h" into the number of seconds it represents. |
|
||||
| [stripDomain](#stripdomain) | string | string | Returns the result of removing the domain part of a FQDN. |
|
||||
|
||||
**Others**
|
||||
|
||||
| Name | Arguments | Returns | Description |
|
||||
| --------------------------- | ------------- | ---------------------- | -------------------------------------------------------------------------------- |
|
||||
| [args](#args) | []interface{} | map[string]interface{} | Translates a list of objects to a map with keys arg0, arg1 etc. |
|
||||
| [safeHtml](#safehtml) | string | string | Marks string as HTML not requiring auto-escaping. |
|
||||
| [externalURL](#externalurl) | none | string | Returns the external URL of the Grafana server as configured in the ini file(s). |
|
||||
| [pathPrefix](#pathprefix) | none | string | Returns the path of the Grafana server as configured in the ini file(s). |
|
||||
|
||||
For further context on these functions, note that templating in Grafana is based on the [Prometheus template implementation](https://prometheus.io/docs/prometheus/latest/configuration/template_reference/), enabling the use of these functions and Prometheus-like templates for formatting alert messages within Grafana.
|
||||
|
||||
#### humanize
|
||||
|
||||
The `humanize` function humanizes decimal numbers:
|
||||
|
||||
```
|
||||
{{ humanize 1000.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1k
|
||||
```
|
||||
|
||||
#### humanize1024
|
||||
|
||||
The `humanize1024` works similar to `humanize` but but uses 1024 as the base rather than 1000:
|
||||
|
||||
```
|
||||
{{ humanize1024 1024.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1ki
|
||||
```
|
||||
|
||||
#### humanizeDuration
|
||||
|
||||
The `humanizeDuration` function humanizes a duration in seconds:
|
||||
|
||||
```
|
||||
{{ humanizeDuration 60.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1m 0s
|
||||
```
|
||||
|
||||
#### humanizePercentage
|
||||
|
||||
The `humanizePercentage` function humanizes a ratio value between 0 and 1 to a percentage:
|
||||
|
||||
```
|
||||
{{ humanizePercentage 0.2 }}
|
||||
```
|
||||
|
||||
```
|
||||
20%
|
||||
```
|
||||
|
||||
#### humanizeTimestamp
|
||||
|
||||
The `humanizeTimestamp` function humanizes a Unix timestamp:
|
||||
|
||||
```
|
||||
{{ humanizeTimestamp 1577836800.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
2020-01-01 00:00:00 +0000 UTC
|
||||
```
|
||||
|
||||
#### toTime
|
||||
|
||||
The `toTime` function converts a Unix timestamp in seconds to time.:
|
||||
|
||||
```
|
||||
{{ toTime 1727802106 }}
|
||||
```
|
||||
|
||||
```
|
||||
2024-10-01 17:01:46 +0000 UTC
|
||||
```
|
||||
|
||||
#### title
|
||||
|
||||
The `title` function capitalizes the first character of each word:
|
||||
|
||||
```
|
||||
{{ title "hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
Hello, World!
|
||||
```
|
||||
|
||||
#### toUpper
|
||||
|
||||
The `toUpper` function returns all text in uppercase:
|
||||
|
||||
```
|
||||
{{ toUpper "Hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
HELLO, WORLD!
|
||||
```
|
||||
|
||||
#### toLower
|
||||
|
||||
The `toLower` function returns all text in lowercase:
|
||||
|
||||
```
|
||||
{{ toLower "Hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
hello, world!
|
||||
```
|
||||
|
||||
#### stripPort
|
||||
|
||||
The `stripPort` splits string into host and port, then returns only host:
|
||||
|
||||
```
|
||||
{{ stripPort "example.com:8080" }}
|
||||
```
|
||||
|
||||
```
|
||||
example.com
|
||||
```
|
||||
|
||||
#### match
|
||||
|
||||
The `match` function matches the text against a regular expression pattern:
|
||||
|
||||
```
|
||||
{{ match "a.*" "abc" }}
|
||||
```
|
||||
|
||||
```
|
||||
true
|
||||
```
|
||||
|
||||
#### reReplaceAll
|
||||
|
||||
The `reReplaceAll` function replaces text matching the regular expression:
|
||||
|
||||
```
|
||||
{{ reReplaceAll "localhost:(.*)" "example.com:$1" "localhost:8080" }}
|
||||
```
|
||||
|
||||
```
|
||||
example.com:8080
|
||||
```
|
||||
|
||||
#### graphLink
|
||||
|
||||
The `graphLink` function returns the path to the graphical view in [Explore](ref:explore) for the given expression and data source:
|
||||
|
||||
```
|
||||
{{ graphLink "{\"expr\": \"up\", \"datasource\": \"gdev-prometheus\"}" }}
|
||||
```
|
||||
|
||||
```
|
||||
/explore?left=["now-1h","now","gdev-prometheus",{"datasource":"gdev-prometheus","expr":"up","instant":false,"range":true}]
|
||||
```
|
||||
|
||||
#### parseDuration
|
||||
|
||||
The `parseDuration` function parses a duration string such as "1h" into the number of seconds it represents.
|
||||
|
||||
```
|
||||
{{ parseDuration "1h" }}
|
||||
```
|
||||
|
||||
```
|
||||
3600
|
||||
```
|
||||
|
||||
#### stripDomain
|
||||
|
||||
The `stripDomain` removes the domain part of a FQDN, leaving port untouched:
|
||||
|
||||
```
|
||||
{{ stripDomain "example.com:8080" }}
|
||||
```
|
||||
|
||||
```
|
||||
example:8080
|
||||
```
|
||||
|
||||
#### tableLink
|
||||
|
||||
The `tableLink` function returns the path to the tabular view in [Explore](ref:explore) for the given expression and data source:
|
||||
|
||||
```
|
||||
{{ tableLink "{\"expr\": \"up\", \"datasource\": \"gdev-prometheus\"}" }}
|
||||
```
|
||||
|
||||
```
|
||||
/explore?left=["now-1h","now","gdev-prometheus",{"datasource":"gdev-prometheus","expr":"up","instant":true,"range":false}]
|
||||
```
|
||||
|
||||
#### args
|
||||
|
||||
The `args` function translates a list of objects to a map with keys arg0, arg1 etc. This is intended to allow multiple arguments to be passed to templates:
|
||||
|
||||
```
|
||||
{{define "x"}}{{.arg0}} {{.arg1}}{{end}}{{template "x" (args 1 "2")}}
|
||||
```
|
||||
|
||||
```
|
||||
1 2
|
||||
```
|
||||
|
||||
#### safeHtml
|
||||
|
||||
The `safeHtml` function marks string as HTML not requiring auto-escaping:
|
||||
|
||||
```
|
||||
{{ safeHtml "<b>Text</b>"}}
|
||||
```
|
||||
|
||||
```
|
||||
<b>Text</b>
|
||||
```
|
||||
|
||||
#### externalURL
|
||||
|
||||
The `externalURL` function returns the external URL of the Grafana server as configured in the ini file(s):
|
||||
|
||||
```
|
||||
{{ externalURL }}
|
||||
```
|
||||
|
||||
```
|
||||
https://example.com/grafana
|
||||
```
|
||||
|
||||
#### pathPrefix
|
||||
|
||||
The `pathPrefix` function returns the path of the Grafana server as configured in the ini file(s):
|
||||
|
||||
```
|
||||
{{ pathPrefix }}
|
||||
```
|
||||
|
||||
```
|
||||
/grafana
|
||||
```
|
||||
|
||||
## Differences with notification templates
|
||||
|
||||
Both notification templates and alert rule templates use the Go templating system. However, the [functions and variables available in notification templates](ref:notification-template-reference) differ from those used in annotations and labels templates, which are described in this documentation.
|
||||
|
||||
Annotation and label templates operate in the context of an individual alert instance, while notification templates apply to a notification that includes a group of alert(s).
|
||||
|
||||
For example, notification templates provide the `.Alerts` variable, which includes the list of all firing and resolved alerts in the notification. This variable is not available in alert rule templates, which operate within the context of a single alert instance.
|
||||
|
||||
Additionally, you cannot reuse templates for labels and annotations as you can with notification templates. Instead, you need to write each template inline within the label or annotation fields and manually copy them wherever you want to reuse the templates.
|
||||
@@ -1,459 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../fundamentals/annotation-label/variables-label-annotation/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/annotation-label/variables-label-annotation/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/templating-labels-annotations/
|
||||
description: Learn about how to template labels and annotations
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templating
|
||||
- labels
|
||||
- annotations
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Template labels and annotations
|
||||
weight: 500
|
||||
refs:
|
||||
explore:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/explore/
|
||||
---
|
||||
|
||||
# Template labels and annotations
|
||||
|
||||
You can use templates to include data from queries and expressions in labels and annotations. For example, you might want to set the severity label for an alert based on the value of the query, or use the instance label from the query in a summary annotation so you know which server is experiencing high CPU usage.
|
||||
|
||||
When using custom labels with templates it is important to make sure that the label value does not change between consecutive evaluations of the alert rule as this will end up creating large numbers of distinct alerts. However, it is OK for the template to produce different label values for different alerts. For example, do not put the value of the query in a custom label as this will end up creating a new set of alerts each time the value changes. Instead use annotations.
|
||||
|
||||
All templates should be written in [text/template](https://pkg.go.dev/text/template). Regardless of whether you are templating a label or an annotation, you should write each template inline inside the label or annotation that you are templating. This means you cannot share templates between labels and annotations, and instead you will need to copy templates wherever you want to use them.
|
||||
|
||||
Each template is evaluated whenever the alert rule is evaluated, and is evaluated for every alert separately. For example, if your alert rule has a templated summary annotation, and the alert rule has 10 firing alerts, then the template will be executed 10 times, once for each alert. You should try to avoid doing expensive computations in your templates as much as possible.
|
||||
|
||||
{{% admonition type="caution" %}}
|
||||
Extra whitespace in label templates can break matches with notification policies.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Variables
|
||||
|
||||
In Grafana templating, the `$` and `.` symbols are used to reference variables and their properties. You can reference variables directly in your alert rule definitions using the `$` symbol followed by the variable name. Similarly, you can access properties of variables using the dot (`.`) notation within alert rule definitions.
|
||||
|
||||
The following variables are available to you when templating labels and annotations:
|
||||
|
||||
### The labels variable
|
||||
|
||||
The `$labels` variable contains all labels from the query. For example, suppose you have a query that returns CPU usage for all of your servers, and you have an alert rule that fires when any of your servers have exceeded 80% CPU usage for the last 5 minutes. You want to add a summary annotation to the alert that tells you which server is experiencing high CPU usage. With the `$labels` variable you can write a template that prints a human-readable sentence such as:
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% for the last 5 minutes
|
||||
```
|
||||
|
||||
> If you are using a classic condition then `$labels` will not contain any labels from the query. Classic conditions discard these labels in order to enforce uni-dimensional behavior (at most one alert per alert rule). If you want to use labels from the query in your template then use the example [here](#print-all-labels-from-a-classic-condition).
|
||||
|
||||
### The value variable
|
||||
|
||||
The `$value` variable is a string containing the labels and values of all instant queries; threshold, reduce and math expressions, and classic conditions in the alert rule. It does not contain the results of range queries, as these can return anywhere from 10s to 10,000s of rows or metrics. If it did, for especially large queries a single alert could use 10s of MBs of memory and Grafana would run out of memory very quickly.
|
||||
|
||||
To print the `$value` variable in the summary you would write something like this:
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% for the last 5 minutes: {{ $value }}
|
||||
```
|
||||
|
||||
And would look something like this:
|
||||
|
||||
```
|
||||
CPU usage for instance1 has exceeded 80% for the last 5 minutes: [ var='A' labels={instance=instance1} value=81.234 ]
|
||||
```
|
||||
|
||||
Here `var='A'` refers to the instant query with Ref ID A, `labels={instance=instance1}` refers to the labels, and `value=81.234` refers to the average CPU usage over the last 5 minutes.
|
||||
|
||||
If you want to print just some of the string instead of the full string then use the `$values` variable. It contains the same information as `$value`, but in a structured table, and is much easier to use then writing a regular expression to match just the text you want.
|
||||
|
||||
### The values variable
|
||||
|
||||
The `$values` variable is a table containing the labels and floating point values of all instant queries and expressions, indexed by their Ref IDs.
|
||||
|
||||
To print the value of the instant query with Ref ID A:
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% for the last 5 minutes: {{ index $values "A" }}
|
||||
```
|
||||
|
||||
For example, given an alert with the labels `instance=server1` and an instant query with the value `81.2345`, this would print:
|
||||
|
||||
```
|
||||
CPU usage for instance1 has exceeded 80% for the last 5 minutes: 81.2345
|
||||
```
|
||||
|
||||
If the query in Ref ID A is a range query rather than an instant query then add a reduce expression with Ref ID B and replace `(index $values "A")` with `(index $values "B")`:
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% for the last 5 minutes: {{ index $values "B" }}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
The following examples attempt to show the most common use-cases we have seen for templates. You can use these examples verbatim, or adapt them as necessary for your use case. For more information on how to write text/template refer see [the beginner's guide to alert notification templates in Grafana](https://grafana.com/blog/2023/04/05/grafana-alerting-a-beginners-guide-to-templating-alert-notifications/).
|
||||
|
||||
### Print all labels, comma separated
|
||||
|
||||
To print all labels, comma separated, print the `$labels` variable:
|
||||
|
||||
```
|
||||
{{ $labels }}
|
||||
```
|
||||
|
||||
For example, given an alert with the labels `alertname=High CPU usage`, `grafana_folder=CPU alerts` and `instance=server1`, this would print:
|
||||
|
||||
```
|
||||
alertname=High CPU usage, grafana_folder=CPU alerts, instance=server1
|
||||
```
|
||||
|
||||
> If you are using classic conditions then `$labels` will not contain any labels from the query. Refer to [the $labels variable](#the-labels-variable) for more information.
|
||||
|
||||
### Print all labels, one per line
|
||||
|
||||
To print all labels, one per line, use a `range` to iterate over each key/value pair and print them individually. Here `$k` refers to the name and `$v` refers to the value of the current label:
|
||||
|
||||
```
|
||||
{{ range $k, $v := $labels -}}
|
||||
{{ $k }}={{ $v }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
For example, given an alert with the labels `alertname=High CPU usage`, `grafana_folder=CPU alerts` and `instance=server1`, this would print:
|
||||
|
||||
```
|
||||
alertname=High CPU usage
|
||||
grafana_folder=CPU alerts
|
||||
instance=server1
|
||||
```
|
||||
|
||||
> If you are using classic conditions then `$labels` will not contain any labels from the query. Refer to [the $labels variable](#the-labels-variable) for more information.
|
||||
|
||||
### Print an individual label
|
||||
|
||||
To print an individual label use the `index` function with the `$labels` variable:
|
||||
|
||||
```
|
||||
The host {{ index $labels "instance" }} has exceeded 80% CPU usage for the last 5 minutes
|
||||
```
|
||||
|
||||
For example, given an alert with the labels `instance=server1`, this would print:
|
||||
|
||||
```
|
||||
The host server1 has exceeded 80% CPU usage for the last 5 minutes
|
||||
```
|
||||
|
||||
> If you are using classic conditions then `$labels` will not contain any labels from the query. Refer to [the $labels variable](#the-labels-variable) for more information.
|
||||
|
||||
### Print the value of a query
|
||||
|
||||
To print the value of an instant query you can print its Ref ID using the `index` function and the `$values` variable:
|
||||
|
||||
```
|
||||
{{ index $values "A" }}
|
||||
```
|
||||
|
||||
For example, given an instant query that returns the value 81.2345, this will print:
|
||||
|
||||
```
|
||||
81.2345
|
||||
```
|
||||
|
||||
To print the value of a range query you must first reduce it from a time series to an instant vector with a reduce expression. You can then print the result of the reduce expression by using its Ref ID instead. For example, if the reduce expression takes the average of A and has the Ref ID B you would write:
|
||||
|
||||
```
|
||||
{{ index $values "B" }}
|
||||
```
|
||||
|
||||
### Print the humanized value of a query
|
||||
|
||||
To print the humanized value of an instant query use the `humanize` function:
|
||||
|
||||
```
|
||||
{{ humanize (index $values "A").Value }}
|
||||
```
|
||||
|
||||
For example, given an instant query that returns the value 81.2345, this will print:
|
||||
|
||||
```
|
||||
81.234
|
||||
```
|
||||
|
||||
To print the humanized value of a range query you must first reduce it from a time series to an instant vector with a reduce expression. You can then print the result of the reduce expression by using its Ref ID instead. For example, if the reduce expression takes the average of A and has the Ref ID B you would write:
|
||||
|
||||
```
|
||||
{{ humanize (index $values "B").Value }}
|
||||
```
|
||||
|
||||
### Print the value of a query as a percentage
|
||||
|
||||
To print the value of an instant query as a percentage use the `humanizePercentage` function:
|
||||
|
||||
```
|
||||
{{ humanizePercentage (index $values "A").Value }}
|
||||
```
|
||||
|
||||
This function expects the value to be a decimal number between 0 and 1. If the value is instead a decimal number between 0 and 100 you can either divide it by 100 in your query or using a math expression. If the query is a range query you must first reduce it from a time series to an instant vector with a reduce expression.
|
||||
|
||||
### Set a severity from the value of a query
|
||||
|
||||
To set a severity label from the value of a query use an if statement and the greater than comparison function. Make sure to use decimals (`80.0`, `50.0`, `0.0`, etc) when doing comparisons against `$values` as text/template does not support type coercion. You can find a list of all the supported comparison functions [here](https://pkg.go.dev/text/template#hdr-Functions).
|
||||
|
||||
```
|
||||
{{ if (gt $values.A.Value 80.0) -}}
|
||||
high
|
||||
{{ else if (gt $values.A.Value 50.0) -}}
|
||||
medium
|
||||
{{ else -}}
|
||||
low
|
||||
{{- end }}
|
||||
```
|
||||
|
||||
### Print all labels from a classic condition
|
||||
|
||||
You cannot use `$labels` to print labels from the query if you are using classic conditions, and must use `$values` instead. The reason for this is classic conditions discard these labels to enforce uni-dimensional behavior (at most one alert per alert rule). If classic conditions didn't discard these labels, then queries that returned many time series would cause alerts to flap between firing and resolved constantly as the labels would change every time the alert rule was evaluated.
|
||||
|
||||
Instead, the `$values` variable contains the reduced values of all time series for all conditions that are firing. For example, if you have an alert rule with a query A that returns two time series, and a classic condition B with two conditions, then `$values` would contain `B0`, `B1`, `B2` and `B3`. If the classic condition B had just one condition, then `$values` would contain just `B0` and `B1`.
|
||||
|
||||
To print all labels of all firing time series use the following template (make sure to replace `B` in the regular expression with the Ref ID of the classic condition if it's different):
|
||||
|
||||
```
|
||||
{{ range $k, $v := $values -}}
|
||||
{{ if (match "B[0-9]+" $k) -}}
|
||||
{{ $k }}: {{ $v.Labels }}{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
For example, a classic condition for two time series exceeding a single condition would print:
|
||||
|
||||
```
|
||||
B0: instance=server1
|
||||
B1: instance=server2
|
||||
```
|
||||
|
||||
If the classic condition has two or more conditions, and a time series exceeds multiple conditions at the same time, then its labels will be duplicated for each condition that is exceeded:
|
||||
|
||||
```
|
||||
B0: instance=server1
|
||||
B1: instance=server2
|
||||
B2: instance=server1
|
||||
B3: instance=server2
|
||||
```
|
||||
|
||||
If you need to print unique labels you should consider changing your alert rules from uni-dimensional to multi-dimensional instead. You can do this by replacing your classic condition with reduce and math expressions.
|
||||
|
||||
### Print all values from a classic condition
|
||||
|
||||
To print all values from a classic condition take the previous example and replace `$v.Labels` with `$v.Value`:
|
||||
|
||||
```
|
||||
{{ range $k, $v := $values -}}
|
||||
{{ if (match "B[0-9]+" $k) -}}
|
||||
{{ $k }}: {{ $v.Value }}{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
For example, a classic condition for two time series exceeding a single condition would print:
|
||||
|
||||
```
|
||||
B0: 81.2345
|
||||
B1: 84.5678
|
||||
```
|
||||
|
||||
If the classic condition has two or more conditions, and a time series exceeds multiple conditions at the same time, then `$values` will contain the values of all conditions:
|
||||
|
||||
```
|
||||
B0: 81.2345
|
||||
B1: 92.3456
|
||||
B2: 84.5678
|
||||
B3: 95.6789
|
||||
```
|
||||
|
||||
## Functions
|
||||
|
||||
The following functions are available to you when templating labels and annotations:
|
||||
|
||||
### args
|
||||
|
||||
The `args` function translates a list of objects to a map with keys arg0, arg1 etc. This is intended to allow multiple arguments to be passed to templates:
|
||||
|
||||
```
|
||||
{{define "x"}}{{.arg0}} {{.arg1}}{{end}}{{template "x" (args 1 "2")}}
|
||||
```
|
||||
|
||||
```
|
||||
1 2
|
||||
```
|
||||
|
||||
### externalURL
|
||||
|
||||
The `externalURL` function returns the external URL of the Grafana server as configured in the ini file(s):
|
||||
|
||||
```
|
||||
{{ externalURL }}
|
||||
```
|
||||
|
||||
```
|
||||
https://example.com/grafana
|
||||
```
|
||||
|
||||
### graphLink
|
||||
|
||||
The `graphLink` function returns the path to the graphical view in [Explore](ref:explore) for the given expression and data source:
|
||||
|
||||
```
|
||||
{{ graphLink "{\"expr\": \"up\", \"datasource\": \"gdev-prometheus\"}" }}
|
||||
```
|
||||
|
||||
```
|
||||
/explore?left=["now-1h","now","gdev-prometheus",{"datasource":"gdev-prometheus","expr":"up","instant":false,"range":true}]
|
||||
```
|
||||
|
||||
### humanize
|
||||
|
||||
The `humanize` function humanizes decimal numbers:
|
||||
|
||||
```
|
||||
{{ humanize 1000.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1k
|
||||
```
|
||||
|
||||
### humanize1024
|
||||
|
||||
The `humanize1024` works similar to `humanize` but but uses 1024 as the base rather than 1000:
|
||||
|
||||
```
|
||||
{{ humanize1024 1024.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1ki
|
||||
```
|
||||
|
||||
### humanizeDuration
|
||||
|
||||
The `humanizeDuration` function humanizes a duration in seconds:
|
||||
|
||||
```
|
||||
{{ humanizeDuration 60.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
1m 0s
|
||||
```
|
||||
|
||||
### humanizePercentage
|
||||
|
||||
The `humanizePercentage` function humanizes a ratio value to a percentage:
|
||||
|
||||
```
|
||||
{{ humanizePercentage 0.2 }}
|
||||
```
|
||||
|
||||
```
|
||||
20%
|
||||
```
|
||||
|
||||
### humanizeTimestamp
|
||||
|
||||
The `humanizeTimestamp` function humanizes a Unix timestamp:
|
||||
|
||||
```
|
||||
{{ humanizeTimestamp 1577836800.0 }}
|
||||
```
|
||||
|
||||
```
|
||||
2020-01-01 00:00:00 +0000 UTC
|
||||
```
|
||||
|
||||
### match
|
||||
|
||||
The `match` function matches the text against a regular expression pattern:
|
||||
|
||||
```
|
||||
{{ match "a.*" "abc" }}
|
||||
```
|
||||
|
||||
```
|
||||
true
|
||||
```
|
||||
|
||||
### pathPrefix
|
||||
|
||||
The `pathPrefix` function returns the path of the Grafana server as configured in the ini file(s):
|
||||
|
||||
```
|
||||
{{ pathPrefix }}
|
||||
```
|
||||
|
||||
```
|
||||
/grafana
|
||||
```
|
||||
|
||||
### tableLink
|
||||
|
||||
The `tableLink` function returns the path to the tabular view in [Explore](ref:explore) for the given expression and data source:
|
||||
|
||||
```
|
||||
{{ tableLink "{\"expr\": \"up\", \"datasource\": \"gdev-prometheus\"}" }}
|
||||
```
|
||||
|
||||
```
|
||||
/explore?left=["now-1h","now","gdev-prometheus",{"datasource":"gdev-prometheus","expr":"up","instant":true,"range":false}]
|
||||
```
|
||||
|
||||
### title
|
||||
|
||||
The `title` function capitalizes the first character of each word:
|
||||
|
||||
```
|
||||
{{ title "hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
Hello, World!
|
||||
```
|
||||
|
||||
### toLower
|
||||
|
||||
The `toLower` function returns all text in lowercase:
|
||||
|
||||
```
|
||||
{{ toLower "Hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
hello, world!
|
||||
```
|
||||
|
||||
### toUpper
|
||||
|
||||
The `toUpper` function returns all text in uppercase:
|
||||
|
||||
```
|
||||
{{ toUpper "Hello, world!" }}
|
||||
```
|
||||
|
||||
```
|
||||
HELLO, WORLD!
|
||||
```
|
||||
|
||||
### reReplaceAll
|
||||
|
||||
The `reReplaceAll` function replaces text matching the regular expression:
|
||||
|
||||
```
|
||||
{{ reReplaceAll "localhost:(.*)" "example.com:$1" "localhost:8080" }}
|
||||
```
|
||||
|
||||
```
|
||||
example.com:8080
|
||||
```
|
||||
@@ -14,29 +14,75 @@ menuTitle: Configure notifications
|
||||
title: Configure notifications
|
||||
weight: 125
|
||||
refs:
|
||||
contact-points:
|
||||
intro-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/contact-points/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/contact-points/
|
||||
notification-policies:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/
|
||||
configure-contact-points:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
templates-page:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/
|
||||
configure-notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-notification-policy/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/create-notification-policy/
|
||||
configure-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
configure-silences:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-silence/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/create-silence/
|
||||
configure-mute-timings:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/mute-timings/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/mute-timings/
|
||||
---
|
||||
|
||||
# Configure notifications
|
||||
|
||||
Choose how, when, and where to send your alert notifications.
|
||||
Configuring how, when, and where to send alert notifications is an essential part of your alerting system.
|
||||
|
||||
As a first step, define your [contact points](ref:contact-points) where to send your alert notifications to. A contact point is a set of one or more integrations that are used to deliver notifications.
|
||||
By default, Grafana Alerting provides default notification messages with relevant alert information, so you don't need to configure messages initially. In the alert rule, you need to configure how to forward alerts:
|
||||
|
||||
Next, create a [notification policy](ref:notification-policies) which is a set of rules for where, when and how your alerts are routed to contact points. In a notification policy, you define where to send your alert notifications by choosing one of the contact points you created.
|
||||
1. Directly to a contact point.
|
||||
2. To a contact point via notification policies (more flexible).
|
||||
|
||||
Optionally, you can add [notification templates](ref:templates-page) to contact points for reuse and consistent messaging in your notifications.
|
||||
{{< figure src="/media/docs/alerting/alerting-configure-notifications-v2.png" max-width="750px" alt="In the alert rule, you can configure alert forwarding directly to a contact point or through notification policies" >}}
|
||||
|
||||
Notification setup is essential for an effective alerting system to scale across multiple teams and services. For a quick overview about the various components involved in handling notifications, refer to the [introduction about notifications](ref:intro-notifications).
|
||||
|
||||
The topics in this section include step-by-step instructions for:
|
||||
|
||||
- [Configuring contact points](ref:configure-contact-points) to specify where to receive alert notifications.
|
||||
- [Configuring notification policies](ref:configure-notification-policies) to determine how alerts are routed to contact points.
|
||||
- [Templating notifications](ref:configure-templates) to customize notification messages.
|
||||
- [Configuring silences](ref:configure-silences) or [mute timings](ref:configure-mute-timings) to stop notifications.
|
||||
|
||||
## Alertmanager architecture
|
||||
|
||||
Grafana Alerting is based on the Prometheus Alerting model, whose architecture decouples rule evaluation from notification handling.
|
||||
|
||||
- The alert rule evaluator, either Grafana or the data source, evaluates alert rules and triggers alerts.
|
||||
- The alert notification manager, known as the **Alertmanager**, receives alerts and manages their notifications.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-alertmanager-architecture.png" max-width="750px" alt="A diagram with the alert generator and alert manager architecture" >}}
|
||||
|
||||
In Grafana, you can use different types of alert rules and configure multiple Alertmanagers.
|
||||
|
||||
By default, Grafana uses its built-in Alertmanager, and Grafana Cloud instances include an additional Alertmanager.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-choose-alertmanager.png" max-width="750px" alt="A screenshot choosing an Alertmanager in the notification policies UI" >}}
|
||||
|
||||
When having multiple Alertmanagers, note that each Alertmanager manages its own independent notification resources, such as contact points, templates, policies, silences, mute timings, and active notifications.
|
||||
|
||||
These notification resources cannot be shared across different Alertmanagers.
|
||||
|
||||
Use the **Choose Alertmanager** dropdown to select the Alertmanager you want to configure.
|
||||
|
||||
@@ -20,26 +20,67 @@ labels:
|
||||
title: Configure notification policies
|
||||
weight: 420
|
||||
refs:
|
||||
notification-policies:
|
||||
alertmanager-architecture:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/#alertmanager-architecture
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/#alertmanager-architecture
|
||||
intro-notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
configure-mute-timings:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/mute-timings/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/mute-timings/
|
||||
configure-contact-points:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/
|
||||
policy-routing:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/#routing
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/#routing
|
||||
policy-inheritance:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/#inheritance
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/#inheritance
|
||||
policy-grouping:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/group-alert-notifications/#group-notifications
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/#group-notifications
|
||||
policy-timing-options:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/group-alert-notifications/#timing-options
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/#timing-options
|
||||
---
|
||||
|
||||
# Configure notification policies
|
||||
|
||||
Notification policies determine how alerts are routed to contact points.
|
||||
|
||||
Policies have a tree structure and each policy can have one or more child policies. Each policy, except for the default policy, can also match specific alert labels.
|
||||
Policies have a tree structure. Each policy can have one or more child policies and a set of label matchers.
|
||||
|
||||
Each alert is evaluated by the default policy and subsequently by each child policy.
|
||||
Each alert (or alert instance) is evaluated by the default policy and subsequently by each child policy. Alerts are routed to the appropriate notification policy by matching alert labels with the policy's label matchers. For further details, refer to [label matching and routing in notification policies](ref:intro-notification-policies).
|
||||
|
||||
If the **Continue matching subsequent sibling nodes** option is enabled for a child policy, evaluation continues even after one or more matches. A parent policy’s configuration settings and contact point information govern the behavior of an alert that does not match any of the child policies. A default policy governs any alert that does not match a child policy.
|
||||
{{< figure src="/media/docs/alerting//get-started-notification-policy-tree-combo.png" max-width="750px" alt="A diagram displaying how the notification policy tree routes alerts" >}}
|
||||
|
||||
You can configure Grafana-managed notification policies as well as notification policies for an external Alertmanager data source.
|
||||
When an alert instance is assigned to a notification policy, the notification policy is responsible for:
|
||||
|
||||
For more information on notification policies, refer to [fundamentals of Notification Policies](ref:notification-policies).
|
||||
- [Grouping similar alerts](ref:policy-grouping) to minimize the number of notifications.
|
||||
- Controlling when notifications are sent using the [timing options](ref:policy-timing-options).
|
||||
- Determining the [contact points](ref:configure-contact-points) that receive the alert notification.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
The default notification policy and its child policies are assigned to a [specific Alertmanager](ref:alertmanager-architecture), and they cannot use contact points or mute timings from other Alertmanagers.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Edit the default notification policy
|
||||
|
||||
@@ -47,12 +88,12 @@ For more information on notification policies, refer to [fundamentals of Notific
|
||||
1. Click **Notification policies**.
|
||||
1. From the **Choose Alertmanager** dropdown, select an external Alertmanager. By default, the **Grafana Alertmanager** is selected.
|
||||
1. In the Default policy section, click **...** -> **Edit**.
|
||||
1. In **Default contact point**, update the contact point for where to send notifications when alert rules do not match any specific policy.
|
||||
1. In **Group by**, choose labels to group alerts by. If multiple alerts are matched for this policy, then they are grouped by these labels. A notification is sent per group. If the field is empty (default), then all notifications are sent in a single group. Use a special label `...` to group alerts by all labels (which effectively disables grouping).
|
||||
1. In **Timing options**, select from the following options:
|
||||
- **Group wait** Time to wait to buffer alerts of the same group before sending an initial notification. Default is 30 seconds.
|
||||
- **Group interval** Minimum time interval between two notifications for a group. Default is 5 minutes.
|
||||
- **Repeat interval** Minimum time interval for re-sending a notification if no new alerts were added to the group. Default is 4 hours.
|
||||
1. In **Default contact point**, update the [contact point](ref:configure-contact-points) for where to send notifications when alert rules do not match any specific policy.
|
||||
1. In **Group by**, choose labels to group alerts. If multiple alerts match this policy, they are grouped by the selected labels, and notifications are sent per group. For more details on using this option, refer to [Group notifications](ref:policy-grouping).
|
||||
1. In **Timing options**, set the [timing options](ref:policy-timing-options) to configure when notifications are sent.
|
||||
- **Group wait**: the time to wait before sending the first notification for a new group of alerts. Default is 30 seconds.
|
||||
- **Group interval**: the time to wait before sending a notification about changes in the alert group. Default is 5 minutes.
|
||||
- **Repeat interval**: the time to wait before sending a notification if the group has not changed since the last notification. Default is 4 hours.
|
||||
1. Click **Save** to save your changes.
|
||||
|
||||
## Add a child policy
|
||||
@@ -62,15 +103,18 @@ You can create a child policy under a default policy or under an existing child
|
||||
If you want to choose where to position your policy, see the section on **Add a sibling policy**.
|
||||
|
||||
1. In the left-side menu, click **Alerts & IRM** and then **Alerting**.
|
||||
2. Click **Notification policies**.
|
||||
3. From the **Choose Alertmanager** dropdown, select an Alertmanager. By default, the **Grafana Alertmanager** is selected.
|
||||
4. Click **+New child policy** from the default policy.
|
||||
5. In the Matching labels section, add one or more rules for matching alert labels.
|
||||
6. In the **Contact point** dropdown, select the contact point to send notification to if alert matches only this specific policy and not any of the child policies.
|
||||
7. Optionally, enable **Continue matching subsequent sibling nodes** to continue matching sibling policies even after the alert matched the current policy. When this option is enabled, you can get more than one notification for one alert.
|
||||
8. Optionally, enable **Override grouping** to specify the same grouping as the default policy. If this option is not enabled, the default policy grouping is used.
|
||||
9. Optionally, enable **Override general timings** to override the timing options configured in the group notification policy.
|
||||
10. Click **Save policy** to save your changes.
|
||||
1. Click **Notification policies**.
|
||||
1. From the **Choose Alertmanager** dropdown, select an Alertmanager. By default, the **Grafana Alertmanager** is selected.
|
||||
1. Click **+New child policy** from the default policy or an existing child policy.
|
||||
1. In the Matching labels section of the child policy, add one or more matching label rules to narrow down a specific case from the parent policy.
|
||||
|
||||
For instance, a child policy of the default policy handles `team=security` alerts, or a child policy handles only the `severity=critical` alerts of a parent policy.
|
||||
|
||||
1. In the **Contact point** dropdown, select the [contact point](ref:configure-contact-points) to send notifications. If left empty, the contact point of the parent policy is [inherited](ref:policy-inheritance).
|
||||
1. Optionally, enable **Continue matching subsequent sibling nodes** to continue matching sibling policies even after the alert matched the current policy. If enabled, multiple policies can handle the same alert.
|
||||
1. Optionally, enable **Override grouping** to set different [grouping](ref:policy-grouping) than the parent policy. If disabled, the grouping of the parent policy is [inherited](ref:policy-inheritance).
|
||||
1. Optionally, enable **Override general timings** to set different [timing options](ref:policy-timing-options) than the parent policy. If disabled, the timing options of the parent policy are [inherited](ref:policy-inheritance).
|
||||
1. Click **Save policy** to save your changes.
|
||||
|
||||
## Add a sibling policy
|
||||
|
||||
@@ -79,9 +123,11 @@ If you want to choose where to position your policy, see the section on **Add a
|
||||
1. Find the child policy you want to create a sibling for.
|
||||
1. Click **Add new policy** -> **New sibling above** or **New sibling below**.
|
||||
|
||||
Notification policies are evaluated from top to bottom, so it is key to be able to choose which notification policy receives alerts first. Adding sibling notification policies instead of always inserting a child policy as well as being able to choose where to insert is key to managing where your alerts go.
|
||||
It's important to determine which policy receives the alert first and to set the correct order of sibling and child policies.
|
||||
|
||||
1. Click **Save policy** to save your changes.
|
||||
Policies are evaluated from top to bottom. If a matching policy is found, the system continues to evaluate its child policies in the order they are displayed. For more details, refer to [notification policies routing](ref:policy-routing).
|
||||
|
||||
1. Follow the instructions from step 5 onward in [adding a child policy](#add-a-child-policy).
|
||||
|
||||
## Search for policies
|
||||
|
||||
@@ -98,7 +144,7 @@ It is important to note that all matched policies are **exact** matches. Grafana
|
||||
|
||||
## Mute timings
|
||||
|
||||
Mute timings are not inherited from a parent notification policy. They have to be configured in full on each level.
|
||||
Mute timings are not inherited from a parent notification policy, and they have to be configured on each level. For instructions, refer to [Configure mute timings](ref:configure-mute-timings).
|
||||
|
||||
## Example
|
||||
|
||||
|
||||
@@ -22,32 +22,52 @@ labels:
|
||||
title: Configure silences
|
||||
weight: 440
|
||||
refs:
|
||||
alert-labels:
|
||||
configure-alertmanager:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
silence-url:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#alert
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#alert
|
||||
shared-alert-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
notification-policies:
|
||||
shared-notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
silences:
|
||||
shared-silences:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-silence/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/create-silence/
|
||||
shared-mute-timings:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/mute-timings/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/mute-timings/
|
||||
alertmanager-architecture:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
---
|
||||
|
||||
# Configure silences
|
||||
|
||||
Silences stop notifications from getting created and last for only a specified window of time.
|
||||
Silences stop notifications from getting created and last for only a specified window of time. Use them to temporarily prevent alert notifications, such as during incident response or a maintenance window.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Silences are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and only suppress notifications for alerts managed by that Alertmanager.
|
||||
{{< /admonition >}}
|
||||
|
||||
- Inhibition rules are not supported in the Grafana Alertmanager.
|
||||
- The preview of silenced alerts only applies to alerts in firing state.
|
||||
{{< /admonition >}}
|
||||
{{< docs/shared lookup="alerts/mute-timings-vs-silences.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
## Add silences
|
||||
|
||||
@@ -59,10 +79,12 @@ To add a silence, complete the following steps.
|
||||
1. Click **Create silence** to open the Create silence page.
|
||||
1. In **Silence start and end**, select the start and end date to indicate when the silence should go into effect and expire.
|
||||
1. Optionally, in **Duration**, specify how long the silence is enforced. This automatically updates the end time in the **Silence start and end** field.
|
||||
1. In the **Label** and **Value** fields, enter one or more _Matching Labels_. Matchers determine which rules the silence will apply to. Any matching alerts (in firing state) will show in the **Silenced alert instances** field
|
||||
1. In the **Label** and **Value** fields, enter one or more _Matching Labels_ to determine which alerts the silence applies to.
|
||||
|
||||
{{< docs/shared lookup="alerts/how_label_matching_works.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
Any matching alerts (in the firing state only) will show under **Affected alert rule instances**.
|
||||
|
||||
1. In **Comment**, add details about the silence.
|
||||
1. Click **Submit**.
|
||||
|
||||
@@ -75,14 +97,6 @@ To edit a silence, complete the following steps.
|
||||
1. Find the silence you want to edit, then click **Edit** (pen icon).
|
||||
1. Make the desired changes, then click **Submit** to save your changes.
|
||||
|
||||
## Create a URL to link to a silence form
|
||||
|
||||
When linking to a silence form, provide the default matching labels and comment via `matcher` and `comment` query parameters. The `matcher` parameter should be in the following format `[label][operator][value]` where the `operator` parameter can be one of the following: `=` (equals, not regular expression), `!=` (not equals, not regular expression), `=~` (equals, regular expression), `!~` (not equals, regular expression).
|
||||
The URL can contain many query parameters with the key `matcher`.
|
||||
For example, to link to silence form with matching labels `severity=critical` & `cluster!~europe-.*` and comment `Silence critical EU alerts`, create a URL `https://mygrafana/alerting/silence/new?matcher=severity%3Dcritical&matcher=cluster!~europe-*&comment=Silence%20critical%20EU%20alert`.
|
||||
|
||||
To link to a new silence page for an external Alertmanager, add a `alertmanager` query parameter with the Alertmanager data source name.
|
||||
|
||||
## Remove silences
|
||||
|
||||
To remove a silence, complete the following steps.
|
||||
@@ -95,13 +109,30 @@ To remove a silence, complete the following steps.
|
||||
|
||||
## Rule-specific silences
|
||||
|
||||
Rule-specific silences are silences that apply only to a specific alert rule.
|
||||
They're created when you silence an alert rule directly using the **Silence notifications** action in the UI.
|
||||
Rule-specific silences are silences that apply only to a specific alert rule. They're created when you silence an alert rule directly using the **Silence notifications** action in the UI.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
As opposed to general silences, rule-specific silence access is tied directly to the alert rule they act on. They can be created manually by including the specific label matcher: `__alert_rule_uid__=<alert rule UID>`.
|
||||
{{< /admonition >}}
|
||||
|
||||
## Useful links
|
||||
## URL link to a silence form
|
||||
|
||||
[Aggregation operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators)
|
||||
Default notification messages often include a link to silence alerts.
|
||||
|
||||
In custom notification templates, you can use [`.Alert.SilenceURL`](ref:silence-url) to redirect users to the UI where they can silence the given alert.
|
||||
|
||||
If [`.Alert.SilenceURL`](ref:silence-url) doesn’t fit your specific use case, you can also create a custom silence link for your custom templates.
|
||||
|
||||
{{< collapse title="Create a custom silence link" >}}
|
||||
|
||||
When linking to a silence form, provide the default matching labels and comment via `matcher` and `comment` query parameters. The `matcher` parameter should be in the following format `[label][operator][value]` where the `operator` parameter can be one of the following: `=` (equals, not regular expression), `!=` (not equals, not regular expression), `=~` (equals, regular expression), `!~` (not equals, regular expression).
|
||||
The URL can contain many query parameters with the key `matcher`.
|
||||
For example, to link to silence form with matching labels `severity=critical` & `cluster!~europe-.*` and comment `Silence critical EU alerts`, create a URL `https://mygrafana/alerting/silence/new?matcher=severity%3Dcritical&matcher=cluster!~europe-*&comment=Silence%20critical%20EU%20alert`.
|
||||
|
||||
To link to a new silence page for an external Alertmanager, add a `alertmanager` query parameter with the Alertmanager data source name.
|
||||
|
||||
{{< /collapse >}}
|
||||
|
||||
## Inhibition rules
|
||||
|
||||
Inhibition rules are supported in the Prometheus Alertmanager. You can [configure a Prometheus Alertmanager](ref:configure-alertmanager) to handle the notification of alerts and suppress notifications via inhibition rules.
|
||||
|
||||
Inhibition rules are not currently supported in the Grafana Alertmanager. For tracking the progress of this feature request, follow [this GitHub issue](https://github.com/grafana/grafana/issues/68822).
|
||||
|
||||
@@ -26,6 +26,16 @@ labels:
|
||||
title: Configure contact points
|
||||
weight: 410
|
||||
refs:
|
||||
sns:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-amazon-sns/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/integrations/configure-amazon-sns/
|
||||
gchat:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-google-chat/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/integrations/configure-google-chat/
|
||||
email:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-email/
|
||||
@@ -71,36 +81,46 @@ refs:
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-teams/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/integrations/configure-teams/
|
||||
external-alertmanager:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
mqtt:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt/
|
||||
alertmanager-architecture:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/#alertmanager-architecture
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/#alertmanager-architecture
|
||||
external-alertmanager:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
manage-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
---
|
||||
|
||||
# Configure contact points
|
||||
|
||||
Use contact points to select your preferred communication channel for receiving notifications when your alert rules are firing. You can add, edit, delete, export, and test a contact point.
|
||||
Use contact points to specify where to receive alert notifications. Contact points contain the configuration for sending alert notifications, including destinations like email, Slack, OnCall, webhooks, and their notification messages.
|
||||
|
||||
Testing a contact point is only available for Grafana Alertmanager.
|
||||
A contact point can have one or multiple destinations, known as [contact point integrations](#list-of-supported-integrations). Alert notifications are sent to each integration within the chosen contact point.
|
||||
|
||||
On the **Contact Points** tab, you can:
|
||||
|
||||
- Add, edit, and view contact points and integrations.
|
||||
- Search for name and type of contact points and integrations.
|
||||
- View all existing contact points and integrations.
|
||||
- View how many notification policies each contact point is being used for and navigate directly to the linked notification policies.
|
||||
- View the status of notification deliveries.
|
||||
- Export individual contact points or all contact points in JSON, YAML, or Terraform format.
|
||||
- Delete contact points. Note that you cannot delete contact points that are in use by a notification policy. To proceed, either delete the notification policy or update it to use another contact point.
|
||||
|
||||
On the **Notification templates** tab, you can:
|
||||
|
||||
- View, edit, copy or delete existing notification templates.
|
||||
{{% admonition type="note" %}}
|
||||
Contact points are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and cannot be used by notification policies in other Alertmanagers.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Add a contact point
|
||||
|
||||
@@ -114,41 +134,23 @@ Complete the following steps to add a contact point.
|
||||
1. From **Integration**, select a type and fill out mandatory fields. For example, if you choose email, enter the email addresses. Or if you choose Slack, enter the Slack channel and users who should be contacted.
|
||||
1. Some contact point integrations, like email or Webhook, have optional settings. In **Optional settings**, specify additional settings for the selected contact point integration.
|
||||
1. In Notification settings, optionally select **Disable resolved message** if you do not want to be notified when an alert resolves.
|
||||
1. To add another contact point integration, click **Add contact point integration** and repeat steps 6 through 8.
|
||||
1. Save your changes.
|
||||
|
||||
## Use notification templates
|
||||
## Add another contact point integration
|
||||
|
||||
Use templates in contact points to customize your notifications.
|
||||
A contact point can have multiple integrations, or destinations for sending notifications.
|
||||
|
||||
Complete the following steps to add templates to your contact point.
|
||||
To add another integration to a contact point, complete the following steps.
|
||||
|
||||
1. Click an existing contact point or create a new one
|
||||
1. In **Optional settings**, click any field that contains templates.
|
||||
|
||||
For example, if you are creating an email contact point integration, click **Message** or **Subject**.
|
||||
|
||||
1. Click **Edit**.
|
||||
A dialog box opens where you can select templates.
|
||||
1. [Optional] Click **Select existing template** to select a template and preview it using the default payload.
|
||||
|
||||
Click **Save** to use just a single template in the field.
|
||||
|
||||
You can also copy the selected template and use it in the custom tab.
|
||||
|
||||
1. [Optional] Click **Enter custom message** to customize and edit the field directly. Note that the title changes depending on the field you are editing.
|
||||
|
||||
Click **Save** to use just a single template in the field.
|
||||
|
||||
1. You can switch between the two tabs to access the list of available templates and copy them across to the customized version.
|
||||
|
||||
1. Click **Save contact point**.
|
||||
1. Add or edit an existing contact point.
|
||||
1. Click **Add contact point integration** and repeat the same steps as [Add a contact point](#add-a-contact-point).
|
||||
- From **Integration**, select a type and fill out mandatory fields.
|
||||
- In **Optional settings**, specify additional settings for the selected contact point integration.
|
||||
1. Save your changes.
|
||||
|
||||
## Test a contact point
|
||||
|
||||
** For Grafana Alertmanager only.**
|
||||
|
||||
Complete the following steps to test a contact point.
|
||||
Testing a contact point is only available for Grafana Alertmanager. Complete the following steps to test a contact point.
|
||||
|
||||
1. In the left-side menu, click **Alerts & IRM** and then **Alerting**.
|
||||
1. Click **Contact points** to view a list of existing contact points.
|
||||
@@ -157,6 +159,16 @@ Complete the following steps to test a contact point.
|
||||
1. Choose whether to send a predefined test notification or choose custom to add your own custom annotations and labels to include in the notification.
|
||||
1. Click **Send test notification** to fire the alert.
|
||||
|
||||
## Customize notification messages
|
||||
|
||||
In contact points, you can also customize notification messages. For example, when setting up an email contact point integration, click **Message** or **Subject** to modify it.
|
||||
|
||||
By default, notification messages include common alert details, which are usually sufficient for most cases.
|
||||
|
||||
If necessary, you can customize the content and format of notification messages. You can create a custom notification template, which can then be applied to one or more contact points.
|
||||
|
||||
On the **Notification templates** tab, you can view, edit, copy or delete notification templates. Refer to [manage notification templates](ref:manage-notification-templates) for instructions on selecting or creating a template for a contact point.
|
||||
|
||||
## List of supported integrations
|
||||
|
||||
Each contact point integration has its own configuration options and setup process. In most cases, this involves providing an API key or a Webhook URL.
|
||||
@@ -166,17 +178,17 @@ The following table lists the contact point integrations supported by Grafana.
|
||||
| Name | Type |
|
||||
| ---------------------------- | ------------------------- |
|
||||
| Alertmanager | `prometheus-alertmanager` |
|
||||
| Amazon SNS | `sns` |
|
||||
| [Amazon SNS](ref:sns) | `sns` |
|
||||
| Cisco Webex Teams | `webex` |
|
||||
| DingDing | `dingding` |
|
||||
| [Discord](ref:discord) | `discord` |
|
||||
| [Email](ref:email) | `email` |
|
||||
| Google Chat | `googlechat` |
|
||||
| [Google Chat](ref:gchat) | `googlechat` |
|
||||
| [Grafana Oncall](ref:oncall) | `oncall` |
|
||||
| Kafka REST Proxy | `kafka` |
|
||||
| [MQTT](ref:mqtt) | `mqtt` |
|
||||
| Line | `line` |
|
||||
| [Microsoft Teams](ref:teams) | `teams` |
|
||||
| [MQTT](ref:mqtt) | `mqtt` |
|
||||
| [Opsgenie](ref:opsgenie) | `opsgenie` |
|
||||
| [Pagerduty](ref:pagerduty) | `pagerduty` |
|
||||
| Pushover | `pushover` |
|
||||
@@ -185,7 +197,7 @@ The following table lists the contact point integrations supported by Grafana.
|
||||
| [Telegram](ref:telegram) | `telegram` |
|
||||
| Threema Gateway | `threema` |
|
||||
| VictorOps | `victorops` |
|
||||
| WeCom | `wecom` |
|
||||
| [Webhook](ref:webhook) | `webhook` |
|
||||
| WeCom | `wecom` |
|
||||
|
||||
Some of these integrations are not compatible with [external Alertmanagers](ref:external-alertmanager). For the list of Prometheus Alertmanager integrations, refer to the [Prometheus Alertmanager receiver settings](https://prometheus.io/docs/alerting/latest/configuration/#receiver-integration-settings).
|
||||
|
||||
@@ -13,7 +13,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Amazon SNS
|
||||
title: Configure Amazon SNS for Alerting
|
||||
weight: 0
|
||||
weight: 100
|
||||
---
|
||||
|
||||
# Configure Amazon SNS for Alerting
|
||||
|
||||
@@ -13,7 +13,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Discord
|
||||
title: Configure Discord for Alerting
|
||||
weight: 0
|
||||
weight: 105
|
||||
---
|
||||
|
||||
# Configure Discord for Alerting
|
||||
|
||||
@@ -13,13 +13,21 @@ labels:
|
||||
- oss
|
||||
menuTitle: Email
|
||||
title: Configure email for Alerting
|
||||
weight: 0
|
||||
weight: 110
|
||||
refs:
|
||||
notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
---
|
||||
|
||||
# Configure email for Alerting
|
||||
|
||||
Use the Grafana Alerting - email integration to send email notifications when your alerts are firing. An email is sent when an alert fires and when an alert gets resolved.
|
||||
|
||||
Note that you can customize the `subject` and `message` of the email using [notification templates](ref:notification-templates). However, you cannot add HTML and CSS to email notifications for visual changes.
|
||||
|
||||
## Before you begin
|
||||
|
||||
{{<admonition type="note">}}
|
||||
|
||||
@@ -13,7 +13,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Google Chat
|
||||
title: Configure Google Chat for Alerting
|
||||
weight: 0
|
||||
weight: 115
|
||||
---
|
||||
|
||||
# Configure Google Chat for Alerting
|
||||
|
||||
@@ -12,9 +12,9 @@ labels:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
menuTitle: MQTT notifier
|
||||
menuTitle: MQTT
|
||||
title: Configure the MQTT notifier for Alerting
|
||||
weight: 0
|
||||
weight: 140
|
||||
---
|
||||
|
||||
# Configure the MQTT notifier for Alerting
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
aliases:
|
||||
- ../../../alerting-rules/manage-contact-points/configure-oncall/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/manage-contact-points/configure-oncall/
|
||||
- ../../../alerting-rules/manage-contact-points/integrations/configure-oncall/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/manage-contact-points/integrations/configure-oncall/
|
||||
- ../configure-oncall/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/manage-contact-points/configure-oncall/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/configure-oncall/
|
||||
description: Configure the Alerting - Grafana OnCall integration to connect alerts generated by Grafana Alerting with Grafana OnCall
|
||||
keywords:
|
||||
@@ -9,8 +10,6 @@ keywords:
|
||||
- alerting
|
||||
- oncall
|
||||
- integration
|
||||
aliases:
|
||||
- ../configure-oncall/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/manage-contact-points/configure-oncall/
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
@@ -18,7 +17,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Grafana OnCall
|
||||
title: Configure Grafana OnCall for Alerting
|
||||
weight: 0
|
||||
weight: 120
|
||||
---
|
||||
|
||||
# Configure Grafana OnCall for Alerting
|
||||
@@ -26,7 +25,7 @@ weight: 0
|
||||
Use the Grafana Alerting - Grafana OnCall integration to effortlessly connect alerts generated by Grafana Alerting with Grafana OnCall, where you can then route them according to defined escalation chains and schedules.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
You can also configure the integration from Grafana OnCall. For more information, refer to [Grafana OnCall documentation](http://grafana.com/docs/oncall/latest/integrations/grafana-alerting/).
|
||||
You can also configure the integration from Grafana OnCall. For more information, refer to [Grafana OnCall documentation](http://grafana.com/docs/oncall/latest/configure/integrations/references/grafana-alerting/).
|
||||
{{< /admonition >}}
|
||||
|
||||
## Before you begin
|
||||
|
||||
@@ -13,7 +13,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Opsgenie
|
||||
title: Configure Opsgenie for Alerting
|
||||
weight: 0
|
||||
weight: 145
|
||||
---
|
||||
|
||||
# Configure Opsgenie for Alerting
|
||||
|
||||
@@ -13,8 +13,13 @@ labels:
|
||||
- oss
|
||||
menuTitle: Slack
|
||||
title: Configure Slack for Alerting
|
||||
weight: 0
|
||||
weight: 155
|
||||
refs:
|
||||
notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
nested-policy:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-notification-policy/#add-new-nested-policy
|
||||
@@ -36,9 +41,9 @@ There are two ways of integrating Slack into Grafana Alerting.
|
||||
|
||||
Webhooks is the simpler way to post messages into Slack. Slack automatically creates a bot user with all the necessary permissions to post messages to one particular channel of your choice.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Grafana Alerting only allows one Slack channel per contact point.
|
||||
{{< /admonition >}}
|
||||
Note that you can only setup one Slack channel per contact point.
|
||||
|
||||
You can customize the `title` and `body` of the Slack message using [notification templates](ref:notification-templates); however, you cannot modify its visual appearance with custom blocks.
|
||||
|
||||
## Before you begin
|
||||
|
||||
@@ -53,17 +58,12 @@ If you are using a Slack API Token, complete the following steps.
|
||||
1. Right click the channel you want to receive notifications in.
|
||||
1. Click View channel details.
|
||||
1. Scroll down and copy the Channel ID.
|
||||
{{< admonition type="note" >}}
|
||||
While going through these steps, Slack may prompt you to Reinstall your app in order for the changes to take effect.
|
||||
{{< /admonition >}}
|
||||
|
||||
Note that while going through these steps, Slack may prompt you to Reinstall your app in order for the changes to take effect.
|
||||
|
||||
### Webhook URL
|
||||
|
||||
If you are using a Webhook URL, follow steps 1 and 5 in the [Slack API Quickstart](https://api.slack.com/start/quickstart).
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Make sure you copy the Slack app Webhook URL. You need this when setting up your contact point integration in Grafana Alerting.
|
||||
{{< /admonition >}}
|
||||
If you are using a Webhook URL, follow steps 1 and 5 in the [Slack API Quickstart](https://api.slack.com/start/quickstart), and copy the Slack app Webhook URL. You need this when setting up your contact point integration in Grafana Alerting.
|
||||
|
||||
## Procedure
|
||||
|
||||
|
||||
@@ -13,13 +13,21 @@ labels:
|
||||
- oss
|
||||
menuTitle: Microsoft Teams
|
||||
title: Configure Microsoft Teams for Alerting
|
||||
weight: 0
|
||||
weight: 135
|
||||
refs:
|
||||
notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
---
|
||||
|
||||
# Configure Microsoft Teams for Alerting
|
||||
|
||||
Use the Grafana Alerting - Microsoft Teams integration to receive notifications in your team’s channel when your alerts are firing.
|
||||
|
||||
Note that you can customize the `title` and `message` of the notification using [notification templates](ref:notification-templates); however, you cannot modify its visual appearance with adaptive cards.
|
||||
|
||||
## Before you begin
|
||||
|
||||
To set up Microsoft Teams for integration with Grafana Alerting, create a new workflow that accepts Webhook requests. This allows Grafana to send alert notifications to Microsoft Teams channels.
|
||||
|
||||
@@ -13,7 +13,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: Telegram
|
||||
title: Configure Telegram for Alerting
|
||||
weight: 0
|
||||
weight: 160
|
||||
---
|
||||
|
||||
# Configure Telegram for Alerting
|
||||
|
||||
@@ -14,7 +14,7 @@ labels:
|
||||
- oss
|
||||
menuTitle: PagerDuty
|
||||
title: Configure PagerDuty for Alerting
|
||||
weight: 0
|
||||
weight: 150
|
||||
---
|
||||
|
||||
# Configure PagerDuty for Alerting
|
||||
|
||||
@@ -19,9 +19,15 @@ labels:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
menuTitle: Webhook notifier
|
||||
menuTitle: Webhook
|
||||
title: Configure the webhook notifier for Alerting
|
||||
weight: 0
|
||||
weight: 165
|
||||
refs:
|
||||
notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
---
|
||||
|
||||
# Configure the webhook notifier for Alerting
|
||||
@@ -139,7 +145,11 @@ The webhook notification is a simple way to send information about a state chang
|
||||
| imageURL | string | URL of a screenshot of a panel assigned to the rule that created this notification |
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
Alert rules are not coupled to dashboards anymore therefore the fields related to dashboards `dashboardId` and `panelId` have been removed.
|
||||
|
||||
You can customize the `title` and `message` fields using [notification templates](ref:notification-templates).
|
||||
|
||||
However, you cannot customize webhook data structure or format, including JSON fields or sending data in XML, nor can you change the webhook HTTP headers.
|
||||
|
||||
{{< /admonition >}}
|
||||
|
||||
## Procedure
|
||||
|
||||
@@ -18,31 +18,34 @@ labels:
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure mute timings
|
||||
weight: 450
|
||||
weight: 430
|
||||
refs:
|
||||
external-alertmanager:
|
||||
alertmanager-architecture:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/#alertmanager-architecture
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/#alertmanager-architecture
|
||||
shared-silences:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-silence/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/create-silence/
|
||||
shared-mute-timings:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/mute-timings/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/mute-timings/
|
||||
---
|
||||
|
||||
# Configure mute timings
|
||||
|
||||
A mute timing is a recurring interval of time when no new notifications for a policy are generated or sent. Use them to prevent alerts from firing a specific and reoccurring period, for example, a regular maintenance period.
|
||||
A mute timing is a recurring interval of time when no new notifications for a policy are generated or sent. Use them to prevent alerts from firing a specific and reoccurring period, for example, a regular maintenance period or weekends.
|
||||
|
||||
Similar to silences, mute timings do not prevent alert rules from being evaluated, nor do they stop alert instances from being shown in the user interface. They only prevent notifications from being created.
|
||||
{{< admonition type="note" >}}
|
||||
Mute timings are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and only suppress notifications for alerts managed by that Alertmanager.
|
||||
{{< /admonition >}}
|
||||
|
||||
You can configure Grafana managed mute timings as well as mute timings for an [external Alertmanager](ref:external-alertmanager).
|
||||
|
||||
## Mute timings vs silences
|
||||
|
||||
The following table highlights the key differences between mute timings and silences.
|
||||
|
||||
| Mute timing | Silence |
|
||||
| -------------------------------------------------- | ---------------------------------------------------------------------------- |
|
||||
| Uses time interval definitions that can reoccur | Has a fixed start and end time |
|
||||
| Is created and then added to notification policies | Uses labels to match against an alert to determine whether to silence or not |
|
||||
{{< docs/shared lookup="alerts/mute-timings-vs-silences.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
## Add mute timings
|
||||
|
||||
@@ -65,6 +68,8 @@ The following table highlights the key differences between mute timings and sile
|
||||
|
||||
A time interval is a specific duration during which alerts are suppressed. The duration typically consists of a specific time range and the days of the week, month, or year.
|
||||
|
||||
A mute timing can contain multiple time intervals.
|
||||
|
||||
Supported time interval options are:
|
||||
|
||||
- Time range: The time inclusive of the start and exclusive of the end time (in UTC if no location has been selected, otherwise local time).
|
||||
@@ -76,9 +81,13 @@ Supported time interval options are:
|
||||
|
||||
All fields are lists; to match the field, at least one list element must be satisfied. Fields also support ranges using `:` (e.g., `monday:thursday`).
|
||||
|
||||
If a field is left blank, any moment of time matches the field. For an instant of time to match a complete time interval, all fields must match. A mute timing can contain multiple time intervals.
|
||||
If a field is left blank, any moment of time matches the field. For an instant of time to match a complete time interval, all fields must match.
|
||||
|
||||
If you want to specify an exact duration, specify all the options. For example, if you wanted to create a time interval for the first Monday of the month, for March, June, September, and December, between the hours of 12:00 and 24:00 UTC your time interval specification would be:
|
||||
If you want to specify an exact duration, specify all the options.
|
||||
|
||||
**Example**
|
||||
|
||||
If you wanted to create a time interval for the first Monday of the month, for March, June, September, and December, between the hours of 12:00 and 24:00 UTC your time interval specification would be:
|
||||
|
||||
- Time range:
|
||||
- Start time: `12:00`
|
||||
|
||||
@@ -13,55 +13,102 @@ labels:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure notification messages
|
||||
weight: 430
|
||||
title: Template notifications
|
||||
weight: 450
|
||||
refs:
|
||||
use-notification-templates:
|
||||
template-annotations-and-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/use-notification-templates/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/use-notification-templates/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
manage-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
using-go-templating-language:
|
||||
examples:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/examples/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/examples/
|
||||
---
|
||||
|
||||
# Configure notification messages
|
||||
# Template notifications
|
||||
|
||||
Customize the content of your notifications with notifications templates.
|
||||
You can use notification templates to change the title, message, and format of notifications.
|
||||
|
||||
You can use notification templates to change the title, message, and format of the message in your notifications.
|
||||
Grafana provides a **default template** for notification titles (`default.title`) and one default template for notification messages (`default.message`). Both templates display common alert details.
|
||||
|
||||
Notification templates are not tied to specific contact point integrations, such as email or Slack. However, you can choose to create separate notification templates for different contact point integrations.
|
||||
You can also create a notification template to customize the content and format of your notification messages. For example:
|
||||
|
||||
You can use notification templates to:
|
||||
|
||||
- Customize content: Personalize the subject of an email or the title of a message. Modify text within notifications, like selecting or omitting certain labels, annotations, and links. Format text with bold and italic styles, and add or remove line breaks.
|
||||
- Personalize the subject of an email or the title of a message.
|
||||
- Modify text within notifications, like selecting or omitting certain labels, annotations, and links.
|
||||
- Format text with bold and italic styles, and add or remove line breaks.
|
||||
|
||||
However, there are limitations. You cannot:
|
||||
|
||||
- Modify Visual Appearance: Add HTML and CSS to email notifications for visual changes. Alter the design of notifications in messaging services like Slack and Microsoft Teams, such as adding custom blocks or adaptive cards.
|
||||
- Manage Media and Data: Adjust the number and size of images or their placement in notifications. Customize webhook data structure or format, including JSON fields or sending data in XML. Modify HTTP headers in webhooks beyond those in the contact point configuration.
|
||||
- Modify Visual Appearance: Add HTML or CSS to email notifications for visual changes. Alter the design of notifications in messaging services like Slack or Microsoft Teams, such as adding custom blocks or adaptive cards.
|
||||
- Manage Media and Data: Customize the data structure or format passed to the templates, like adding new JSON fields or sending XML data for webhooks. Modify HTTP headers in webhooks beyond those defined in the configuration, or adjust the number, size, or placement of images.
|
||||
|
||||
## Learn more
|
||||
Here's an [example](ref:examples) that displays the summary and description annotations for each alert in the notification:
|
||||
|
||||
[Using Go's templating language](ref:using-go-templating-language)
|
||||
```go
|
||||
{{ define "custom.alerts" -}}
|
||||
{{ len .Alerts }} alert(s)
|
||||
{{ range .Alerts -}}
|
||||
{{ template "alert.summary_and_description" . -}}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ define "alert.summary_and_description" }}
|
||||
Summary: {{.Annotations.summary}}
|
||||
Status: {{ .Status }}
|
||||
Description: {{.Annotations.description}}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
Learn how to write the content of your notification templates in Go’s templating language.
|
||||
The notification message would look like this:
|
||||
|
||||
Create reusable notification templates for your contact points.
|
||||
```
|
||||
2 alert(s)
|
||||
|
||||
[Use notification templates](ref:use-notification-templates)
|
||||
Summary: The database server db1 has exceeded 75% of available disk space.
|
||||
Status: firing
|
||||
Description: This alert fires when a database server is at risk of running out of disk space. You should take measures to increase the maximum available disk space as soon as possible to avoid possible corruption.
|
||||
|
||||
Use notification templates to send notifications to your contact points.
|
||||
Summary: The web server web1 has been responding to 5% of HTTP requests with 5xx errors for the last 5 minutes.
|
||||
Status: resolved
|
||||
Description: This alert fires when a web server responds with more 5xx errors than is expected. This could be an issue with the web server or a backend service.
|
||||
```
|
||||
|
||||
[Reference](ref:reference)
|
||||
{{% admonition type="note" %}}
|
||||
Avoid adding extra information about alert instances in notification templates, as this information will only be visible in the notification message.
|
||||
|
||||
Data that is available when writing templates.
|
||||
Instead, you should [use annotations or labels](ref:template-annotations-and-labels) to add information directly to the alert, ensuring it's also visible in the alert state and alert history within Grafana. You can then print the new alert annotation or label in notification templates.
|
||||
{{% /admonition %}}
|
||||
|
||||
#### Select a notification template for a contact point
|
||||
|
||||
Notification templates are not tied to specific contact point integrations, such as email or Slack, and the same template can be shared across multiple contact points.
|
||||
|
||||
The notification template is assigned to the contact point to determine the notification message sent to contact point integrations.
|
||||
|
||||
{{< figure src="/media/docs/alerting/how-notification-templates-works.png" max-width="1200px" caption="A flow of the alert notification process, from querying the alert rule to sending the alert notification message." >}}
|
||||
|
||||
By default, Grafana provides default templates, such as `default.title` and `default.message`, to format notification messages.
|
||||
|
||||
## More information
|
||||
|
||||
For further details on how to write notification templates, refer to:
|
||||
|
||||
- [Select, create, and preview a notification template](ref:manage-notification-templates)
|
||||
- [Notification template reference](ref:reference)
|
||||
- [Notification template examples](ref:examples)
|
||||
|
||||
{{< admonition type="tip" >}}
|
||||
For a practical example of templating, refer to our [Getting Started with Templating tutorial](https://grafana.com/tutorials/alerting-get-started-pt4/).
|
||||
{{< /admonition >}}
|
||||
|
||||
@@ -1,376 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../manage-notifications/template-notifications/create-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/create-notification-templates/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
description: Create notification templates to sent to your contact points
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- notifications
|
||||
- templates
|
||||
- create templates
|
||||
- edit templates
|
||||
- delete templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Create notification templates
|
||||
weight: 200
|
||||
---
|
||||
|
||||
# Create notification templates
|
||||
|
||||
Create reusable notification templates to send to your contact points.
|
||||
|
||||
You can add one or more templates to your notification template.
|
||||
|
||||
Your notification template name must be unique. You cannot have two templates with the same name in the same notification template or in different notification templates. Avoid defining templates with the same name as default templates, such as: `__subject`, `__text_values_list`, `__text_alert_list`, `default.title` and `default.message`.
|
||||
|
||||
To create a notification template, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Contact points**.
|
||||
1. Click the **Notification Templates** tab and then **+ Add notification template**.
|
||||
|
||||
1. Enter a name for the notification template.
|
||||
|
||||
1. Write the content of the template in the content field.
|
||||
|
||||
1. Save your changes.
|
||||
|
||||
`{{ define "email.subject" }}` and `{{ end }}` is automatically added to the start and end of the content:
|
||||
|
||||
To create a notification template that contains more than one template:
|
||||
|
||||
1. Click **+ Add notification template**.
|
||||
|
||||
2. Enter a name for the notification template.
|
||||
|
||||
3. Write each template in the Content field, including `{{ define "name-of-template" }}` and `{{ end }}` at the start and end of each template.
|
||||
|
||||
4. Save your changes.
|
||||
|
||||
## Preview notification templates
|
||||
|
||||
Preview how your notification templates should look before using them in your contact points, helping you understand the result of the template you are creating as well as enabling you to fix any errors before saving it.
|
||||
|
||||
**Note:** This feature is only for Grafana Alertmanager.
|
||||
|
||||
To preview your notification templates:
|
||||
|
||||
1. Navigate to **Alerts&IRM** -> **Alerting** -> **Contact points** -> **Notification Templates**.
|
||||
1. Click **+ Add notification template** or edit an existing template.
|
||||
1. Add or update your template content.
|
||||
|
||||
Default data is provided and you can add or edit alert data to it as well as alert instances. You can add alert data directly in the Payload data window itself or click **Select alert instances** or **Add custom alerts**.
|
||||
|
||||
1. Optional: To add alert data from existing alert instances:
|
||||
|
||||
a. Click **Select alert instances**.
|
||||
|
||||
b. Hover over the alert instances to view more information on each alert instance.
|
||||
|
||||
c. Click **Confirm** to add the alert instance(s) to the payload.
|
||||
|
||||
1. Optional: To add alert data using the Alert data editor, click **Add custom data:**
|
||||
|
||||
a. Add annotations, custom labels and/or set a dashboard or a panel.
|
||||
|
||||
b. Toggle Firing/resolved depending on whether you want to add firing or resolved alerts to your notification.
|
||||
|
||||
c. Click **Add alert data**.
|
||||
|
||||
d. Click **Refresh preview** to see what your template content should look like and the corresponding payload data.
|
||||
|
||||
If there are any errors in your template, they are displayed in the Preview and you can correct them before saving.
|
||||
|
||||
1. Save your changes.
|
||||
|
||||
## Template the subject of an email
|
||||
|
||||
Template the subject of an email to contain the number of firing and resolved alerts:
|
||||
|
||||
```
|
||||
1 firing alert(s), 0 resolved alerts(s)
|
||||
```
|
||||
|
||||
1. Create a template called `email.subject` with the following content:
|
||||
|
||||
```
|
||||
{{ define "email.subject" }}
|
||||
{{ len .Alerts.Firing }} firing alert(s), {{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
2. Execute the template from the subject field in your contact point integration:
|
||||
|
||||
```
|
||||
{{ template "email.subject" . }}
|
||||
```
|
||||
|
||||
## Template the message of an email
|
||||
|
||||
Template the message of an email to contain a summary of all firing and resolved alerts:
|
||||
|
||||
```
|
||||
There are 2 firing alert(s), and 1 resolved alert(s)
|
||||
|
||||
Firing alerts:
|
||||
|
||||
- alertname=Test 1 grafana_folder=GrafanaCloud has value(s) B=1
|
||||
- alertname=Test 2 grafana_folder=GrafanaCloud has value(s) B=2
|
||||
|
||||
Resolved alerts:
|
||||
|
||||
- alertname=Test 3 grafana_folder=GrafanaCloud has value(s) B=0
|
||||
```
|
||||
|
||||
1. Create a notification template called `email` with two templates in the content: `email.message_alert` and `email.message`.
|
||||
|
||||
The `email.message_alert` template is used to print the labels and values for each firing and resolved alert while the `email.message` template contains the structure of the email.
|
||||
|
||||
```
|
||||
{{- define "email.message_alert" -}}
|
||||
{{- range .Labels.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }} has value(s)
|
||||
{{- range $k, $v := .Values }} {{ $k }}={{ $v }}{{ end }}
|
||||
{{- end -}}
|
||||
|
||||
{{ define "email.message" }}
|
||||
There are {{ len .Alerts.Firing }} firing alert(s), and {{ len .Alerts.Resolved }} resolved alert(s)
|
||||
|
||||
{{ if .Alerts.Firing -}}
|
||||
Firing alerts:
|
||||
{{- range .Alerts.Firing }}
|
||||
- {{ template "email.message_alert" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{ if .Alerts.Resolved -}}
|
||||
Resolved alerts:
|
||||
{{- range .Alerts.Resolved }}
|
||||
- {{ template "email.message_alert" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
2. Execute the template from the message field in your contact point integration:
|
||||
|
||||
```
|
||||
{{ template "email.message" . }}
|
||||
```
|
||||
|
||||
## Group multiple alert instances into one email notification
|
||||
|
||||
To make alerts more concise, you can group multiple instances of a firing alert into a single email notification in a table format. This way, you avoid long, repetitive emails and make alerts easier to digest.
|
||||
|
||||
Follow these steps to create a custom notification template that consolidates alert instances into a table.
|
||||
|
||||
1. Modify the alert rule to include an annotation that is referenced in the notification template later on.
|
||||
1. Enter a name for the **custom annotation**: In this example, _ServerInfo_.
|
||||
1. Enter the following code as the value for the annotation. It retrieves the server's instance name and a corresponding metric value, formatted as a table row:
|
||||
|
||||
```
|
||||
{{ index $labels "instance" }}{{- "\t" -}}{{ index $values "A"}}{{- "\n" -}}
|
||||
```
|
||||
|
||||
This line of code returns the labels and their values in the form of a table. Assuming $labels has `{"instance": "node1"}` and $values has `{"A": "123"}`, the output would be:
|
||||
|
||||
```
|
||||
node1 123
|
||||
```
|
||||
|
||||
1. Create a notification template that references the _ServerInfo_ annotation.
|
||||
|
||||
```go
|
||||
{{ define "Table" }}
|
||||
{{- "\nHost\t\tValue\n" -}}
|
||||
{{ range .Alerts -}}
|
||||
{{ range .Annotations.SortedPairs -}}
|
||||
{{ if (eq .Name "ServerInfo") -}}
|
||||
{{ .Value -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
The notification template outputs a list of server information from the "ServerInfo" annotation for each alert instance.
|
||||
|
||||
1. Navigate to your contact point in Grafana
|
||||
1. In the **Message** field, reference the template by name (see **Optional Email settings** section):
|
||||
|
||||
```
|
||||
{{ template "Table" . }}
|
||||
```
|
||||
|
||||
This generates a neatly formatted table in the email, grouping information for all affected servers into a single notification.
|
||||
|
||||
## Conditional notification template
|
||||
|
||||
Template alert notifications based on a label. In this example the label represents a namespace.
|
||||
|
||||
1. Use the following code in your notification template to display different messages based on the namespace:
|
||||
|
||||
```go
|
||||
{{ define "my_conditional_notification" }}
|
||||
{{ if eq .CommonLabels.namespace "namespace-a" }}
|
||||
Alert: CPU limits have reached 80% in namespace-a.
|
||||
{{ else if eq .CommonLabels.namespace "namespace-b" }}
|
||||
Alert: CPU limits have reached 80% in namespace-b.
|
||||
{{ else if eq .CommonLabels.namespace "namespace-c" }}
|
||||
Alert: CPU limits have reached 80% in namespace-c.
|
||||
{{ else }}
|
||||
Alert: CPU limits have reached 80% for {{ .CommonLabels.namespace }} namespace.
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
`.CommonLabels` is a map containing the labels that are common to all the alerts firing.
|
||||
|
||||
Make sure to replace the `.namespace` label with a label that exists in your alert rule.
|
||||
|
||||
1. Replace `namespace-a`, `namespace-b`, and `namespace-c` with your specific namespace values.
|
||||
1. Navigate to your contact point in Grafana
|
||||
1. In the **Message** field, reference the template by name (see **Optional settings** section):
|
||||
|
||||
```
|
||||
{{ template "my_conditional_notification" . }}
|
||||
```
|
||||
|
||||
This template alters the content of alert notifications depending on the namespace value.
|
||||
|
||||
## Template the title of a Slack message
|
||||
|
||||
Template the title of a Slack message to contain the number of firing and resolved alerts:
|
||||
|
||||
```
|
||||
1 firing alert(s), 0 resolved alerts(s)
|
||||
```
|
||||
|
||||
1. Create a template called `slack.title` with the following content:
|
||||
|
||||
```
|
||||
{{ define "slack.title" }}
|
||||
{{ len .Alerts.Firing }} firing alert(s), {{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
2. Execute the template from the title field in your contact point integration:
|
||||
|
||||
```
|
||||
{{ template "slack.title" . }}
|
||||
```
|
||||
|
||||
## Template the content of a Slack message
|
||||
|
||||
Template the content of a Slack message to contain a description of all firing and resolved alerts, including their labels, annotations, Silence URL and Dashboard URL.
|
||||
|
||||
**Note:**
|
||||
|
||||
This template is for Grafana-managed alerts only.
|
||||
To use the template for Grafana Mimir/Loki-managed alerts, delete the references to DashboardURL and SilenceURL.
|
||||
For more information, see the [Prometheus documentation on notifications](https://prometheus.io/docs/alerting/latest/notifications/).
|
||||
|
||||
```
|
||||
1 firing alert(s):
|
||||
|
||||
[firing] Test1
|
||||
Labels:
|
||||
- alertname: Test1
|
||||
- grafana_folder: GrafanaCloud
|
||||
Annotations:
|
||||
- description: This is a test alert
|
||||
Silence: https://example.com/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTest1&matcher=grafana_folder%3DGrafanaCloud
|
||||
Go to dashboard: https://example.com/d/dlhdLqF4z?orgId=1
|
||||
|
||||
1 resolved alert(s):
|
||||
|
||||
[firing] Test2
|
||||
Labels:
|
||||
- alertname: Test2
|
||||
- grafana_folder: GrafanaCloud
|
||||
Annotations:
|
||||
- description: This is another test alert
|
||||
Silence: https://example.com/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTest2&matcher=grafana_folder%3DGrafanaCloud
|
||||
Go to dashboard: https://example.com/d/dlhdLqF4z?orgId=1
|
||||
```
|
||||
|
||||
1. Create a template called `slack` with two templates in the content: `slack.print_alert` and `slack.message`.
|
||||
|
||||
The `slack.print_alert` template is used to print the labels, annotations, SilenceURL and DashboardURL while the `slack.message` template contains the structure of the notification.
|
||||
|
||||
```
|
||||
{{ define "slack.print_alert" -}}
|
||||
[{{.Status}}] {{ .Labels.alertname }}
|
||||
Labels:
|
||||
{{ range .Labels.SortedPairs -}}
|
||||
- {{ .Name }}: {{ .Value }}
|
||||
{{ end -}}
|
||||
{{ if .Annotations -}}
|
||||
Annotations:
|
||||
{{ range .Annotations.SortedPairs -}}
|
||||
- {{ .Name }}: {{ .Value }}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ if .SilenceURL -}}
|
||||
Silence: {{ .SilenceURL }}
|
||||
{{ end -}}
|
||||
{{ if .DashboardURL -}}
|
||||
Go to dashboard: {{ .DashboardURL }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "slack.message" -}}
|
||||
{{ if .Alerts.Firing -}}
|
||||
{{ len .Alerts.Firing }} firing alert(s):
|
||||
{{ range .Alerts.Firing }}
|
||||
{{ template "slack.print_alert" . }}
|
||||
{{ end -}}
|
||||
{{ end }}
|
||||
{{ if .Alerts.Resolved -}}
|
||||
{{ len .Alerts.Resolved }} resolved alert(s):
|
||||
{{ range .Alerts.Resolved }}
|
||||
{{ template "slack.print_alert" .}}
|
||||
{{ end -}}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
```
|
||||
|
||||
2. Execute the template from the text body field in your contact point integration:
|
||||
|
||||
```
|
||||
{{ template "slack.message" . }}
|
||||
```
|
||||
|
||||
## Template both email and Slack with shared templates
|
||||
|
||||
Instead of creating separate notification templates for email and Slack, you can share the same template.
|
||||
|
||||
For example, if you want to send an email with this subject and Slack message with this title:
|
||||
|
||||
```
|
||||
1 firing alert(s), 0 resolved alerts(s)
|
||||
```
|
||||
|
||||
1. Create a template called `common.subject_title` with the following content:
|
||||
|
||||
```
|
||||
{{ define "common.subject_title" }}
|
||||
{{ len .Alerts.Firing }} firing alert(s), {{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
2. For email, execute the template from the subject field in your email contact point integration:
|
||||
|
||||
```
|
||||
{{ template "common.subject_title" . }}
|
||||
```
|
||||
|
||||
3. For Slack, execute the template from the title field in your Slack contact point integration:
|
||||
|
||||
```
|
||||
{{ template "common.subject_title" . }}
|
||||
```
|
||||
@@ -0,0 +1,404 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/alerting-rules/templates/examples/
|
||||
description: Examples of notification templates
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templating
|
||||
- notification templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Notification template examples
|
||||
menuTitle: Examples
|
||||
weight: 103
|
||||
refs:
|
||||
template-annotations-and-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
template-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
manage-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
reference-notification-data:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
reference-alert:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#alert
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#alert
|
||||
language:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/language/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/language/
|
||||
group-alert-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
---
|
||||
|
||||
# Notification template examples
|
||||
|
||||
Notification templates allows you to change the default notification messages.
|
||||
|
||||
You can modify the content and format of notification messages. For example, you can customize the content to show only specific information or adjust the format to suit a particular contact point, such as Slack or Email.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Avoid adding extra information about alert instances in notification templates, as this information is only be visible in the notification message.
|
||||
|
||||
Instead, you should [use annotations or labels](ref:template-annotations-and-labels) to add information directly to the alert, ensuring it's also visible in the alert state and alert history within Grafana. You can then print the new alert annotation or label in notification templates.
|
||||
{{% /admonition %}}
|
||||
|
||||
This page provides various examples illustrating how to template common notification messages. For more details about notification templates, refer to:
|
||||
|
||||
- [Template notifications](ref:template-notifications)
|
||||
- [Select, create, and preview a notification template](ref:manage-notification-templates)
|
||||
- [Notification template reference](ref:reference)
|
||||
|
||||
## Basic examples
|
||||
|
||||
Notification templates can access the [notification data](ref:reference-notification-data) using the dot (`.`). The following examples demonstrate some basic uses of the [template language](ref:language).
|
||||
|
||||
For instance, to check if there are common labels (`.CommonLabels`) for all alerts in the notification, use `if`:
|
||||
|
||||
```go
|
||||
{{ define "custom_message" -}}
|
||||
{{ if .CommonLabels }}
|
||||
Alerts have common labels
|
||||
{{ else }}
|
||||
There are no common labels
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
To iterate on the alerts in the notification and print a specific label, use `range` and `index`:
|
||||
|
||||
```go
|
||||
{{ define "custom_message" -}}
|
||||
{{ range .Alerts }}
|
||||
The name of the alert is {{ index .Labels "alertname" }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
Alternatively, you can use the `.` notation to print the value of the key.
|
||||
|
||||
```go
|
||||
{{ define "custom_message" -}}
|
||||
{{ range .Alerts }}
|
||||
The name of the alert is {{ .Labels.alertname }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
You can then use the template by passing the [notification data (dot `.`)](ref:reference-notification-data):
|
||||
|
||||
```go
|
||||
{{ template "custom_message" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
The name of the alert is InstanceDown
|
||||
|
||||
The name of the alert is CpuOverload
|
||||
```
|
||||
|
||||
## Print alerts with summary and description
|
||||
|
||||
Here's an example that displays the summary and description annotations for each alert in the notification.
|
||||
|
||||
```go
|
||||
{{ define "custom.alerts" -}}
|
||||
{{ len .Alerts }} alert(s)
|
||||
{{ range .Alerts -}}
|
||||
{{ template "alert.summary_and_description" . -}}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ define "alert.summary_and_description" }}
|
||||
Summary: {{.Annotations.summary}}
|
||||
Status: {{ .Status }}
|
||||
Description: {{.Annotations.description}}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
In this example:
|
||||
|
||||
- A template (`alert.summary_and_description`) is defined to print the `summary`, `status`, and `description` of one [alert](ref:reference-alert).
|
||||
- The main template `custom.alerts` iterates the list of alerts (`.Alerts`) in [notification data](ref:reference-notification-data), executing the `alert.summary_and_description` template to print the details of each alert.
|
||||
|
||||
Execute the template by passing the dot (`.`):
|
||||
|
||||
```go
|
||||
{{ template "custom.alerts" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
2 alert(s)
|
||||
|
||||
Summary: The database server db1 has exceeded 75% of available disk space.
|
||||
Status: firing
|
||||
Description: This alert fires when a database server is at risk of running out of disk space. You should take measures to increase the maximum available disk space as soon as possible to avoid possible corruption.
|
||||
|
||||
Summary: The web server web1 has been responding to 5% of HTTP requests with 5xx errors for the last 5 minutes.
|
||||
Status: resolved
|
||||
Description: This alert fires when a web server responds with more 5xx errors than is expected. This could be an issue with the web server or a backend service.
|
||||
```
|
||||
|
||||
## Print firing and resolved alerts
|
||||
|
||||
The following example is similar to the previous one, but it separates firing and resolved alerts.
|
||||
|
||||
```go
|
||||
{{ define "custom.firing_and_resolved_alerts" -}}
|
||||
{{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ range .Alerts.Resolved -}}
|
||||
{{ template "alert.summary_and_description" . -}}
|
||||
{{ end }}
|
||||
{{ len .Alerts.Firing }} firing alert(s)
|
||||
{{ range .Alerts.Firing -}}
|
||||
{{ template "alert.summary_and_description" . -}}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ define "alert.summary_and_description" }}
|
||||
Summary: {{.Annotations.summary}}
|
||||
Status: {{ .Status }}
|
||||
Description: {{.Annotations.description}}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
Instead of `.Alerts`, the template accesses `.Alerts.Firing` and `.Alerts.Resolved` separately to print details for each alert.
|
||||
|
||||
Run the template by passing the dot (`.`):
|
||||
|
||||
```go
|
||||
{{ template "custom.firing_and_resolved_alerts" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
1 resolved alert(s)
|
||||
|
||||
Summary: The database server db1 has exceeded 75% of available disk space.
|
||||
Status: resolved
|
||||
Description: This alert fires when a database server is at risk of running out of disk space. You should take measures to increase the maximum available disk space as soon as possible to avoid possible corruption.
|
||||
|
||||
1 firing alert(s)
|
||||
|
||||
Summary: The web server web1 has been responding to 5% of HTTP requests with 5xx errors for the last 5 minutes.
|
||||
Status: firing
|
||||
Description: This alert fires when a web server responds with more 5xx errors than is expected. This could be an issue with the web server or a backend service.
|
||||
```
|
||||
|
||||
## Print common labels and annotations
|
||||
|
||||
This example displays only the labels and annotations that are common to all alerts in the notification.
|
||||
|
||||
```go
|
||||
{{ define "custom.common_labels_and_annotations" -}}
|
||||
{{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ len .Alerts.Firing }} firing alert(s)
|
||||
Common labels: {{ len .CommonLabels.SortedPairs }}
|
||||
{{ range .CommonLabels.SortedPairs -}}
|
||||
- {{ .Name }} = {{ .Value }}
|
||||
{{ end }}
|
||||
Common annotations: {{ len .CommonAnnotations.SortedPairs }}
|
||||
{{ range .CommonAnnotations.SortedPairs }}
|
||||
- {{ .Name }} = {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
Note that `.CommonAnnotations` and `.CommonLabels` are part of [notification data](ref:reference-notification-data).
|
||||
|
||||
Execute the template by passing the dot (`.`) as argument:
|
||||
|
||||
```go
|
||||
{{ template "custom.common_labels_and_annotations" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
1 resolved alert(s)
|
||||
1 firing alert(s)
|
||||
Common labels: 2
|
||||
- grafana_folder = server_alerts
|
||||
- team = server_admin
|
||||
|
||||
Common annotations: 0
|
||||
```
|
||||
|
||||
## Print individual labels and annotations
|
||||
|
||||
This example displays all labels and annotations for each [alert](ref:reference-alert) in the notification.
|
||||
|
||||
```go
|
||||
{{ define "custom.alert_labels_and_annotations" -}}
|
||||
{{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ range .Alerts.Resolved -}}
|
||||
{{ template "alert.labels_and_annotations" . -}}
|
||||
{{ end }}
|
||||
{{ len .Alerts.Firing }} firing alert(s)
|
||||
{{ range .Alerts.Firing -}}
|
||||
{{ template "alert.labels_and_annotations" . -}}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ define "alert.labels_and_annotations" }}
|
||||
Alert labels: {{ len .Labels.SortedPairs }}
|
||||
{{ range .Labels.SortedPairs -}}
|
||||
- {{ .Name }} = {{ .Value }}
|
||||
{{ end -}}
|
||||
Alert annotations: {{ len .Annotations.SortedPairs }}
|
||||
{{ range .Annotations.SortedPairs -}}
|
||||
- {{ .Name }} = {{ .Value }}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
In this example:
|
||||
|
||||
- The `custom.alert_labels_and_annotations` template iterates over the list of resolved and firing alerts, similar to previous examples. It then executes `alert.labels_and_annotations` for each alert.
|
||||
- The `alert.labels_and_annotations` template prints all the alert labels and annotations by accessing `.Labels.SortedPairs` and `.Annotations.SortedPairs`.
|
||||
|
||||
Run the template by passing the dot (`.`):
|
||||
|
||||
```go
|
||||
{{ template "custom.alert_labels_and_annotations" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
1 resolved alert(s)
|
||||
|
||||
Alert labels: 4
|
||||
- alertname = db_server_disk_space
|
||||
- grafana_folder = server_alerts
|
||||
- server = db1
|
||||
- team = server_admin
|
||||
|
||||
Alert annotations: 2
|
||||
- summary = The database server db1 has exceeded 75% of available disk space.
|
||||
- description = This alert fires when a database server is at risk of running out of disk space. You should take measures to increase the maximum available disk space as soon as possible to avoid possible corruption.
|
||||
|
||||
1 firing alert(s)
|
||||
|
||||
Alert labels: 4
|
||||
- alertname = web_server_http_errors
|
||||
- grafana_folder = server_alerts
|
||||
- server = web1
|
||||
- team = server_admin
|
||||
|
||||
Alert annotations: 2
|
||||
- summary = The web server web1 has been responding to 5% of HTTP requests with 5xx errors for the last 5 minutes.
|
||||
- description = This alert fires when a web server responds with more 5xx errors than is expected. This could be an issue with the web server or a backend service.
|
||||
```
|
||||
|
||||
## Print URLs for runbook and alert data in Grafana
|
||||
|
||||
Note that the following example works only for Grafana-managed alerts. It displays some [alert data](ref:reference-alert) such as `DashboardURL`, `PanelURL`, and `SilenceURL`, which are exclusive to Grafana-managed alerts.
|
||||
|
||||
```go
|
||||
{{ define "custom.alert_additional_details" -}}
|
||||
{{ len .Alerts.Resolved }} resolved alert(s)
|
||||
{{ range .Alerts.Resolved -}}
|
||||
{{ template "alert.additional_details" . -}}
|
||||
{{ end }}
|
||||
{{ len .Alerts.Firing }} firing alert(s)
|
||||
{{ range .Alerts.Firing -}}
|
||||
{{ template "alert.additional_details" . -}}
|
||||
{{ end -}}
|
||||
{{ end -}}
|
||||
{{ define "alert.additional_details" }}
|
||||
- Dashboard: {{ .DashboardURL }}
|
||||
- Panel: {{ .PanelURL }}
|
||||
- AlertGenerator: {{ .GeneratorURL }}
|
||||
- Silence: {{ .SilenceURL }}
|
||||
- RunbookURL: {{ .Annotations.runbook_url}}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
Pass the dot (`.`) to execute the template:
|
||||
|
||||
```go
|
||||
{{ template "custom.alert_additional_details" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
1 resolved alert(s)
|
||||
|
||||
- Dashboard: https://example.com/d/
|
||||
- Panel: https://example.com/d/
|
||||
- AlertGenerator: ?orgId=1
|
||||
- Silence: https://example.com/alerting/silence/new
|
||||
- RunbookURL: https://example.com/on-call/db_server_disk_space
|
||||
|
||||
1 firing alert(s)
|
||||
|
||||
- Dashboard: https://example.com/d/
|
||||
- Panel: https://example.com/d/
|
||||
- AlertGenerator: ?orgId=1
|
||||
- Silence: https://example.com/alerting/silence/new
|
||||
- RunbookURL: https://example.com/on-call/web_server_http_errors
|
||||
```
|
||||
|
||||
## Print a notification title or subject
|
||||
|
||||
A title or subject provides a one-line summary of the notification content.
|
||||
|
||||
Here’s a basic example that displays the number of firing and resolved alerts in the notification.
|
||||
|
||||
```go
|
||||
{{ define "custom_title" -}}
|
||||
{{ if gt (.Alerts.Firing | len) 0 }}🚨 {{ .Alerts.Firing | len }} firing alerts. {{ end }}{{ if gt (.Alerts.Resolved | len) 0 }}✅ {{ .Alerts.Resolved | len }} resolved alerts.{{ end }}
|
||||
{{ end -}}
|
||||
```
|
||||
|
||||
Execute the template by passing the dot (`.`) as argument:
|
||||
|
||||
```go
|
||||
{{ template "custom_title" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
🚨 1 firing alerts. ✅ 1 resolved alerts.
|
||||
```
|
||||
|
||||
The next example is a copy of the default title/subject template used in Grafana.
|
||||
|
||||
```go
|
||||
{{ define "copy_of_default_title" -}}
|
||||
[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
This is a more advanced example:
|
||||
|
||||
- Prints the number of firing and resolved alerts in the notification.
|
||||
- Outputs `.GroupLabels`, the labels used to [group multiple alerts in one notification](ref:group-alert-notifications).
|
||||
- Prints `CommonLabels`, excluding labels in `.GroupLabels`.
|
||||
|
||||
Execute the template by passing the dot (`.`):
|
||||
|
||||
```go
|
||||
{{ template "copy_of_default_title" . }}
|
||||
```
|
||||
|
||||
```template_output
|
||||
[FIRING:1, RESOLVED:1] api warning (sql_db)
|
||||
```
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
aliases:
|
||||
- ../manage-notifications/images-in-notifications/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/images-in-notifications/
|
||||
- ../../manage-notifications/images-in-notifications/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/images-in-notifications/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/images-in-notifications/
|
||||
description: Use images in notifications to help users better understand why alerts are firing or have been resolved
|
||||
keywords:
|
||||
@@ -14,7 +14,8 @@ labels:
|
||||
- enterprise
|
||||
- oss
|
||||
title: Use images in notifications
|
||||
weight: 500
|
||||
menuTitle: Use images
|
||||
weight: 105
|
||||
---
|
||||
|
||||
# Use images in notifications
|
||||
@@ -55,6 +56,8 @@ Refer to the table at the end of this page for a list of contact points and thei
|
||||
|
||||
8. Grafana does not delete screenshots uploaded to its internal web server. To delete screenshots from `static_root_path/images/attachments` after a certain amount of time, we recommend setting up a CRON job.
|
||||
|
||||
9. Note you cannot adjust the number and size of images or their placement in notifications.
|
||||
|
||||
## Configuration
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
@@ -130,7 +133,7 @@ If Grafana has been set up to send images in notifications, however notification
|
||||
5. If images cannot be taken because of mis-configuration or an issue with image rendering there are logs for `Failed to take an image` including the Dashboard UID, Panel ID, and the error message.
|
||||
6. Check that the contact point supports images in notifications and whether it supports uploading images to the receiving service or referencing images that have been uploaded to a cloud storage service.
|
||||
|
||||
## Metrics
|
||||
## Monitor
|
||||
|
||||
Grafana provides the following metrics to observe the performance and failure rate of images in notifications.
|
||||
For example, if a screenshot could not be taken within the expected time (10 seconds) then the counter `grafana_screenshot_failures_total` is updated.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../manage-notifications/template-notifications/using-go-templating-language/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/using-go-templating-language/
|
||||
- ../../configure-notifications/template-notifications/using-go-templating-language/ # /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
weight: 104
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/language/
|
||||
description: Use Go template language to create your notification and alert rule templates
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- templates
|
||||
- write templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Alerting template language
|
||||
menuTitle: Template language
|
||||
refs:
|
||||
alert-rule-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/
|
||||
alert-rule-template-reference-variables:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/#variables
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/#variables
|
||||
notification-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
reference-notificationdata:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#notification-data
|
||||
---
|
||||
|
||||
# Alerting template language
|
||||
|
||||
Notification templates and alert rule templates, such as annotations and labels, both use the Go template language, [text/template](https://pkg.go.dev/text/template).
|
||||
|
||||
Both types of templates can use the same keywords, functions, and comparison operators of the Go template language, such as `range`, `if`, `and`, `index`, `eq`, and more.
|
||||
|
||||
However, it's important to note that because notifications and alert rules operate in distinct context, some additional variables and functions are only available for either notification or alert rule templates. Refer to:
|
||||
|
||||
- [Annotation and label template reference](ref:alert-rule-template-reference)
|
||||
- [Notification template reference](ref:notification-template-reference)
|
||||
|
||||
This documentation provides an overview of the functions and operators of the Go template language that are available for both notification and alert rule templates.
|
||||
|
||||
## Print
|
||||
|
||||
To print the value of something, use `{{` and `}}`. You can print the value of a [variable](#variables), a field of a variable, the result of a function, or the value of dot.
|
||||
|
||||
```
|
||||
{{ $values }}
|
||||
{{ $values.A.Value }}
|
||||
{{ humanize 1000.0 }}
|
||||
{{ .Alerts }}
|
||||
```
|
||||
|
||||
## Dot
|
||||
|
||||
In `text/template`, there is a special cursor called dot, written as `.`. You can think of this cursor as a variable whose value changes depending on where in the template it is used.
|
||||
|
||||
At the start of notification templates, dot (`.`) refers to [Notification Data](ref:reference-notificationdata).
|
||||
|
||||
```
|
||||
{{ .Alerts }}
|
||||
```
|
||||
|
||||
In annotation and label templates, dot (`.`) is initialized with all alert data. It’s recommended to use the [`$labels` and `$values` variables](ref:alert-rule-template-reference-variables) instead to directly access the alert labels and query values.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Dot (`.`) might refer to something else when used in a [range](#range), a [with](#with), or when writing [templates](#templates) used in other templates.
|
||||
{{% /admonition %}}
|
||||
|
||||
[//]: <> (The above section is not included in the shared file because `refs` links are not supported in shared files.)
|
||||
|
||||
{{< docs/shared lookup="alerts/template-language.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
@@ -0,0 +1,136 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../manage-notifications/template-notifications/create-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/create-notification-templates/
|
||||
- ../../manage-notifications/template-notifications/use-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/use-notification-templates/
|
||||
- ../../configure-notifications/template-notifications/use-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/use-notification-templates/
|
||||
- ../../configure-notifications/template-notifications/create-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/create-notification-templates/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/manage-notification-templates/
|
||||
description: Create notification templates to sent to your contact points
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- notifications
|
||||
- templates
|
||||
- create templates
|
||||
- edit templates
|
||||
- delete templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Manage notification templates
|
||||
menuTitle: Manage templates
|
||||
weight: 101
|
||||
refs:
|
||||
notification-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
notification-template-examples:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/examples/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/examples/
|
||||
---
|
||||
|
||||
# Manage notification templates
|
||||
|
||||
In contact points, you can select notification templates to customize the notification messages sent.
|
||||
|
||||
By default, Grafana provides a template for the notification title (`default.title`) and a template for the notification message (`default.message`). Both default templates display common alert details.
|
||||
|
||||
You can also create custom templates to customize the content and format of notification messages, which can then be applied to one or more contact points.
|
||||
|
||||
This documentation provides step-by-step instructions for selecting templates in contact points, previewing templates, and creating custom templates using the Grafana UI.
|
||||
|
||||
## Select a notification template for a contact point
|
||||
|
||||
To add an existing notification template to your contact point, complete the following steps.
|
||||
|
||||
1. Click an existing contact point or create a new one.
|
||||
1. In **Optional settings**, click any field that contains templates.
|
||||
|
||||
For example, if you are creating an email contact point integration, click **Message** or **Subject**.
|
||||
|
||||
1. Click **Edit**.
|
||||
A dialog box opens where you can select templates.
|
||||
1. Click **Select existing template** to select a template and [preview](#preview-a-notification-template) it using the default payload.
|
||||
|
||||
You can also copy the selected template and use it in the custom tab.
|
||||
|
||||
1. Click **Enter custom message** to customize and edit the field directly. Note that the title changes depending on the field you are editing.
|
||||
|
||||
1. You can switch between the two tabs to access the list of available templates and copy them across to the customized version.
|
||||
|
||||
1. Click **Save contact point**.
|
||||
|
||||
## Create a notification template
|
||||
|
||||
Create notification templates to customize notification messages and reuse them in contact points.
|
||||
|
||||
Your notification template name must be unique. You cannot have two templates with the same name in the same notification template or in different notification templates. Avoid defining templates with the same name as default templates, such as: `__subject`, `__text_values_list`, `__text_alert_list`, `default.title` and `default.message`.
|
||||
|
||||
To create a notification template in Grafana, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Contact points**.
|
||||
1. Click the **Notification Templates** tab and then **+ Add notification template**.
|
||||
|
||||
1. Enter a name for the notification template.
|
||||
|
||||
1. Write the content of the template in the content field.
|
||||
|
||||
1. Save your changes.
|
||||
|
||||
If `{{ define }}` is not included in the content, `{{ define "<NOTIFICATION_TEMPLATE_NAME>" }}` and `{{ end }}` is automatically added to the start and end.
|
||||
|
||||
To create a notification template that contains more than one template, complete the following steps.
|
||||
|
||||
1. Click **+ Add notification template**.
|
||||
|
||||
1. Enter a name for the notification template.
|
||||
|
||||
1. Write each template in the Content field, including `{{ define "name-of-template" }}` and `{{ end }}` at the start and end of each template.
|
||||
|
||||
1. Save your changes.
|
||||
|
||||
For more details on how to write notification templates, refer to the [template reference](ref:notification-template-reference) and [examples](ref:notification-template-examples).
|
||||
|
||||
## Preview a notification template
|
||||
|
||||
Preview how your notification templates should look before using them in your contact points, helping you understand the result of the template you are creating as well as enabling you to fix any errors before saving it.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Notification template preview is only for Grafana Alertmanager.
|
||||
{{% /admonition %}}
|
||||
|
||||
To preview your notification templates:
|
||||
|
||||
1. Navigate to **Alerts&IRM** -> **Alerting** -> **Contact points** -> **Notification Templates**.
|
||||
1. Click **+ Add notification template** or edit an existing template.
|
||||
1. Add or update your template content.
|
||||
|
||||
Default data is provided and you can add or edit alert data to it as well as alert instances. You can add alert data directly in the Payload data window itself or click **Select alert instances** or **Add custom alerts**.
|
||||
|
||||
1. Optional: To add alert data from existing alert instances:
|
||||
|
||||
a. Click **Select alert instances**.
|
||||
|
||||
b. Hover over the alert instances to view more information on each alert instance.
|
||||
|
||||
c. Click **Confirm** to add the alert instance(s) to the payload.
|
||||
|
||||
1. Optional: To add alert data using the Alert data editor, click **Add custom data:**
|
||||
|
||||
a. Add annotations, custom labels and/or set a dashboard or a panel.
|
||||
|
||||
b. Toggle Firing/resolved depending on whether you want to add firing or resolved alerts to your notification.
|
||||
|
||||
c. Click **Add alert data**.
|
||||
|
||||
d. Click **Refresh preview** to see what your template content should look like and the corresponding payload data.
|
||||
|
||||
If there are any errors in your template, they are displayed in the Preview and you can correct them before saving.
|
||||
|
||||
1. Save your changes.
|
||||
@@ -13,80 +13,261 @@ labels:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Reference
|
||||
weight: 400
|
||||
title: Notification template reference
|
||||
menuTitle: Template reference
|
||||
weight: 102
|
||||
refs:
|
||||
label-types:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/#label-types
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#label-types
|
||||
alert-rule-template-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/reference/
|
||||
alert-grouping:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
template-language:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/language/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/language/
|
||||
template-language-functions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/language/#functions
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/language/#functions
|
||||
---
|
||||
|
||||
# Reference
|
||||
# Notification template reference
|
||||
|
||||
## Data
|
||||
By default, Grafana provides predefined templates to format notification messages.
|
||||
|
||||
### Alert
|
||||
You can also customize your notifications with custom templates, which are based on the [Go template language](ref:template-language).
|
||||
|
||||
| Name | Kind | Description | Example |
|
||||
| ------------ | -------- | ------------------------------------------------------------------------------------ | --------------------- |
|
||||
| Status | `string` | Firing or resolved | `{{ .Status }}` |
|
||||
| Labels | `KV` | The labels for this alert | `{{ .Labels }}` |
|
||||
| Annotations | `KV` | The annotations for this alert | `{{ .Annotations }}` |
|
||||
| Values | `KV` | The values of all expressions, including Classic Conditions | `{{ .Values }}` |
|
||||
| StartsAt | `Time` | The time the alert fired | `{{ .StartsAt }}` |
|
||||
| EndsAt | `Time` | | `{{ .EndsAt }}` |
|
||||
| GeneratorURL | `string` | A link to Grafana, or the Alertmanager if using an external Alertmanager | `{{ .GeneratorURL }}` |
|
||||
| SilenceURL | `string` | A link to silence the alert | `{{ .SilenceURL }}` |
|
||||
| DashboardURL | `string` | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation | `{{ .DashboardURL }}` |
|
||||
| PanelURL | `string` | A link to the panel if the alert has a Panel ID annotation | `{{ .PanelURL }}` |
|
||||
| Fingerprint | `string` | A unique string that identifies the alert | `{{ .Fingerprint }}` |
|
||||
| ValueString | `string` | A string that contains the labels and value of each reduced expression in the alert. | `{{ .ValueString }}` |
|
||||
This documentation lists the data available for use in notification templates.
|
||||
|
||||
### ExtendedData
|
||||
## Notification Data
|
||||
|
||||
| Name | Kind | Description | Example |
|
||||
| ----------------- | --------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------ |
|
||||
| Receiver | `string` | The name of the contact point sending the notification | `{{ .Receiver }}` |
|
||||
| Status | `string` | The status is `firing` if at least one alert is firing, otherwise `resolved` | `{{ .Status }}` |
|
||||
| Alerts | `[]Alert` | List of all firing and resolved alerts in this notification | `There are {{ len .Alerts }} alerts` |
|
||||
| Firing alerts | `[]Alert` | List of all firing alerts in this notification | `There are {{ len .Alerts.Firing }} firing alerts` |
|
||||
| Resolved alerts | `[]Alert` | List of all resolved alerts in this notification | `There are {{ len .Alerts.Resolved }} resolved alerts` |
|
||||
| GroupLabels | `KV` | The labels that group these alerts in this | `{{ .GroupLabels }}` |
|
||||
| CommonLabels | `KV` | The labels common to all alerts in this notification | `{{ .CommonLabels }}` |
|
||||
| CommonAnnotations | `KV` | The annotations common to all alerts in this notification | `{{ .CommonAnnotations }}` |
|
||||
| ExternalURL | `string` | A link to Grafana, or the Alertmanager that sent this notification if using an external Alertmanager | `{{ .ExternalURL }}` |
|
||||
In notification templates, dot (`.`) is initialized with the following data:
|
||||
|
||||
### KV
|
||||
| Name | Type | Description |
|
||||
| ------------------- | ----------------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| `Receiver` | string | The name of the contact point sending the notification |
|
||||
| `Status` | string | The status is `firing` if at least one alert is firing, otherwise `resolved`. |
|
||||
| `Alerts` | [][Alert](#alert) | List of all firing and resolved alerts in this notification. |
|
||||
| `Alerts.Firing` | [][Alert](#alert) | List of all firing alerts in this notification. |
|
||||
| `Alerts.Resolved` | [][Alert](#alert) | List of all resolved alerts in this notification. |
|
||||
| `GroupLabels` | [KV](#kv) | The labels that group these alerts in this notification based on the `Group by` option. |
|
||||
| `CommonLabels` | [KV](#kv) | The labels common to all alerts in this notification. |
|
||||
| `CommonAnnotations` | [KV](#kv) | The annotations common to all alerts in this notification. |
|
||||
| `ExternalURL` | string | A link to Grafana, or the Alertmanager that sent this notification if using an external Alertmanager. |
|
||||
|
||||
`KV` is a set of key value pairs, where each key and value is a string. If a KV happens to contain numbers or bools then these are string representations of the numeric or boolean value.
|
||||
It's important to remember that [a single notification can group multiple alerts](ref:alert-grouping) to reduce the number of alerts you receive. `Alerts` is an array that includes all the alerts in the notification.
|
||||
|
||||
Here is an example of a KV, the annotations of an alert:
|
||||
Here's an example that prints all available notification data from dot (`.`):
|
||||
|
||||
```yaml
|
||||
summary: 'A summary of the alert'
|
||||
description: 'A description of the alert'
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ .Receiver }}
|
||||
{{ .Status }}
|
||||
There are {{ len .Alerts }} alerts
|
||||
There are {{ len .Alerts.Firing }} firing alerts
|
||||
There are {{ len .Alerts.Resolved }} resolved alerts
|
||||
{{ .GroupLabels }}
|
||||
{{ .CommonLabels }}
|
||||
{{ .CommonAnnotations }}
|
||||
{{ .ExternalURL }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
In addition to iterating over each key value pair, you can sort the pairs, remove keys, and iterate over just the keys or the values.
|
||||
You can execute this template by passing the dot (`.`):
|
||||
|
||||
| Name | Description | Arguments | Returns | Example |
|
||||
| ----------- | ---------------------------------------------- | --------- | ------- | ------------------------------------- |
|
||||
| SortedPairs | Sorts | | | `{{ .Annotations.SortedPairs }}` |
|
||||
| Remove | Returns a copy of the KV with the keys removed | []string | | `{{ .Annotations.Remove "summary" }}` |
|
||||
| Names | A list of the names | | | `{{ .Names }}` |
|
||||
| Values | A list of the values | | | `{{ .Values }}` |
|
||||
|
||||
### Time
|
||||
|
||||
Time is from the Go [`time`](https://pkg.go.dev/time#Time) package.
|
||||
|
||||
You can format a time in a number of different formats using the `date` function.
|
||||
For example, to print the time that an alert fired in the format `15:04:05 MST`:
|
||||
|
||||
```
|
||||
{{ .StartsAt | date "15:04:05 MST" }}
|
||||
```go
|
||||
{{ template "custom_template" . }}
|
||||
```
|
||||
|
||||
You can also use the `tz` function to change the timezone from UTC to a local time. For example:
|
||||
## Alert
|
||||
|
||||
```
|
||||
{{ .StartsAt | tz "Europe/Paris" | date "15:04:05 MST" }}
|
||||
`Alert` contains data for an individual alert:
|
||||
|
||||
| Name | Type | Description |
|
||||
| -------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `Status` | string | Firing or resolved. |
|
||||
| `Labels` | [KV](#kv) | The labels for this alert. It includes all [types of labels](ref:label-types). |
|
||||
| `Annotations` | [KV](#kv) | The annotations for this alert. |
|
||||
| `StartsAt` | [Time](#time) | The time the alert fired |
|
||||
| `EndsAt` | [Time](#time) | Only set if the end time of an alert is known. Otherwise set to a configurable timeout period from the time since the last alert was received. |
|
||||
| `GeneratorURL` | string | A link to Grafana, or the source of the alert if using an external alert generator. |
|
||||
| `Fingerprint` | string | A unique string that identifies the alert. |
|
||||
|
||||
Grafana-managed alerts include these additional properties:
|
||||
|
||||
| Name | Type | Description |
|
||||
| -------------- | --------- | ------------------------------------------------------------------------------------ |
|
||||
| `DashboardURL` | string | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation. |
|
||||
| `PanelURL` | string | A link to the panel if the alert has a Panel ID annotation. |
|
||||
| `SilenceURL` | string | A link to silence the alert. |
|
||||
| `Values` | [KV](#kv) | The values of all expressions, including Classic Conditions. |
|
||||
| `ValueString` | string | A string that contains the labels and value of each reduced expression in the alert. |
|
||||
|
||||
This example iterates over the list of firing and resolved alerts (`.Alerts`) in the notification and prints the data for each alert:
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ range .Alerts }}
|
||||
{{ .Status }}
|
||||
{{ .Labels }}
|
||||
{{ .Annotations }}
|
||||
{{ .StartsAt }}
|
||||
{{ .EndsAt }}
|
||||
{{ .GeneratorURL }}
|
||||
{{ .Fingerprint }}
|
||||
|
||||
{{/* Only available for Grafana-managed alerts */}}
|
||||
{{ .DashboardURL }}
|
||||
{{ .PanelURL }}
|
||||
{{ .SilenceURL }}
|
||||
{{ .Values }}
|
||||
{{ .ValueString }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
You can find a reference for Go's time format [here](https://pkg.go.dev/time#pkg-constants).
|
||||
You can run this template by passing the dot (`.`):
|
||||
|
||||
```go
|
||||
{{ template "custom_template" . }}
|
||||
```
|
||||
|
||||
## KV
|
||||
|
||||
`KV` is a set of key value pairs, where each key and value is a string.
|
||||
|
||||
Similarly to accessing variable properties, you can use `.` to retrieve the value of a value. For example:
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ .CommonLabels.grafana_folder }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
If a KV happens to contain numbers or bools then these are string representations of the numeric or boolean value.
|
||||
|
||||
Additionally, KV provides methods to sort the pairs, remove keys, and iterate over just the keys or values:
|
||||
|
||||
| Method name | Description | Arguments | Returns |
|
||||
| ----------- | ---------------------------------------------- | --------- | --------- |
|
||||
| SortedPairs | Returns a sorted list of key/value pairs. | | Pairs |
|
||||
| Remove | Returns a copy of the KV with the keys removed | []string | [KV](#kv) |
|
||||
| Names | Return the names of the label names | | []string |
|
||||
| Values | Return a list of the values | | []string |
|
||||
|
||||
Here's an example of using these methods:
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ .CommonLabels.SortedPairs }}
|
||||
{{ .CommonLabels.Names }}
|
||||
{{ .CommonLabels.Values }}
|
||||
{{ .CommonLabels.Remove (stringSlice "grafana_folder") }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Time
|
||||
|
||||
Some template functions and properties return a `Time` object, which refers to the [type `Time`](https://pkg.go.dev/time#Time) in Go's time package.
|
||||
|
||||
When accessing a `Time` object, you can use various [`Time` functions](https://pkg.go.dev/time#Time) in your templates as follows.
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ range .Alerts }}
|
||||
{{ .StartsAt }}
|
||||
{{ .StartsAt.Add 6000000000000 }}
|
||||
{{ .StartsAt.Add -6000000000000 }}
|
||||
{{ .StartsAt.AddDate 1 0 0 }}
|
||||
{{ .StartsAt.Year }}/{{ .StartsAt.Month }}/{{ .StartsAt.Day }}
|
||||
{{ .StartsAt.Hour }}:{{ .StartsAt.Minute }}:{{ .StartsAt.Second }}
|
||||
{{ .StartsAt.YearDay }}-{{ .StartsAt.Weekday }}
|
||||
{{ .StartsAt.Unix }} {{ .StartsAt.UnixMilli }}
|
||||
{{ end}}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Functions
|
||||
|
||||
Functions can perform actions in templates such as transforming or formatting data.
|
||||
|
||||
Note that the [functions provided by Go's template language](ref:template-language-functions), such as `index`, `and`, `printf`, and `len`, are available, along with many others.
|
||||
|
||||
In addition, the following functions are also available for templating notifications:
|
||||
|
||||
| Name | Arguments | Returns | Description |
|
||||
| -------------- | -------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `title` | string | string | Capitalizes the first character of each word. |
|
||||
| `toUpper` | string | string | Returns all text in uppercase. |
|
||||
| `toLower` | string | string | Returns all text in lowercase. |
|
||||
| `trimSpace` | string | string | Removes leading and trailing white spaces. |
|
||||
| `match` | pattern, text | boolean | Matches the text against a regular expression pattern. |
|
||||
| `reReplaceAll` | pattern, replacement, text | string | Replaces text matching the regular expression. |
|
||||
| `join` | sep string, s []string | string | Concatenates the elements of s to create a single string. The separator string sep is placed between elements in the resulting string. |
|
||||
| `safeHtml` | string | string | Marks string as HTML not requiring auto-escaping. |
|
||||
| `stringSlice` | ...string | string | Returns the passed strings as a slice of strings. auto-escaping. |
|
||||
| `date` | string, [Time](#time) | string | Format a time in the specified format. For format options, refer to [Go's time format documentation](https://pkg.go.dev/time#pkg-constants). |
|
||||
| `tz` | string, [Time](#time) | [Time](#time) | Returns the time in the specified timezone, such as `Europe/Paris`. For available timezone options, refer to the [list of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). |
|
||||
|
||||
Here's an example using some functions to format text:
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ title "hello, world!" }}
|
||||
{{ toUpper "Hello, world!" }}
|
||||
{{ toLower "Hello, world!" }}
|
||||
{{ trimSpace "Hello, world!" }}
|
||||
{{ match "a.*" "abc" }}
|
||||
{{ reReplaceAll "localhost:(.*)" "example.com:$1" "localhost:8080" }}
|
||||
{{ join "-" (stringSlice "a" "b" "c") }}
|
||||
{{ safeHtml "<b>Text</b>"}}
|
||||
{{ stringSlice "a" "b" "c" }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
`date` and `tz` can format times. For example, to print the time an alert fired in the format `15:04:05 MST`:
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ range .Alerts }}
|
||||
{{ .StartsAt | date "15:04:05 MST" }}
|
||||
{{ end}}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
You can then use `tz` to change the timezone from UTC to local time, such as `Europe/Paris`.
|
||||
|
||||
```go
|
||||
{{ define "custom_template" }}
|
||||
{{ range .Alerts }}
|
||||
{{ .StartsAt | tz "Europe/Paris" }}
|
||||
{{ .StartsAt | tz "Europe/Paris" | date "15:04:05 MST" }}
|
||||
{{ end}}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
```template-output
|
||||
2024-10-30 21:01:45.227 +0100 CET
|
||||
21:01:45 CET
|
||||
```
|
||||
|
||||
## Differences with annotation and label templates
|
||||
|
||||
In the alert rule, you can also template annotations and labels to include additional information. For example, you might add a `summary` annotation that displays the query value triggering the alert.
|
||||
|
||||
Annotation and label templates add relevant information to individual alert instances, while notification templates inform about a group of alert instances.
|
||||
|
||||
Since both types of templates operate in distinct contexts, the [functions and variables available in annotation and label templates](ref:alert-rule-template-reference) differ from those used in notification templates.
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../manage-notifications/template-notifications/use-notification-templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/use-notification-templates/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/use-notification-templates/
|
||||
description: Use notification templates in contact points to customize your notifications
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- notifications
|
||||
- templates
|
||||
- use templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Use notification templates
|
||||
weight: 300
|
||||
refs:
|
||||
create-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
using-go-templating-language:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
---
|
||||
|
||||
# Use notification templates
|
||||
|
||||
Use templates in contact points to customize your notifications.
|
||||
|
||||
Complete the following steps to add templates to your contact point.
|
||||
|
||||
1. Click an existing contact point or create a new one
|
||||
1. In **Optional settings**, click any field that contains templates.
|
||||
|
||||
For example, if you are creating an email contact point integration, click **Message** or **Subject**.
|
||||
|
||||
1. Click **Edit**.
|
||||
A dialog box opens where you can select templates.
|
||||
1. Click **Select existing template** to select a template and preview it using the default payload.
|
||||
|
||||
You can also copy the selected template and use it in the custom tab.
|
||||
|
||||
1. Click **Enter custom message** to customize and edit the field directly. Note that the title changes depending on the field you are editing.
|
||||
|
||||
1. You can switch between the two tabs to access the list of available templates and copy them across to the customized version.
|
||||
|
||||
1. Click **Save contact point**.
|
||||
@@ -1,298 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../manage-notifications/template-notifications/using-go-templating-language/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/template-notifications/using-go-templating-language/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/template-notifications/using-go-templating-language/
|
||||
description: Use Go's templating language to create your own notification templates
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- notifications
|
||||
- templates
|
||||
- write templates
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Using Go's templating language
|
||||
weight: 100
|
||||
refs:
|
||||
create-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
extendeddata:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/#extendeddata
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/#extendeddata
|
||||
reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
---
|
||||
|
||||
# Using Go's templating language
|
||||
|
||||
You write notification templates in Go's templating language, [text/template](https://pkg.go.dev/text/template).
|
||||
|
||||
Before you start creating your own notification templates, we recommend that you read through this topic, which provides you with an overview of Go's templating language and writing templates in text/template.
|
||||
|
||||
## Dot
|
||||
|
||||
In text/template there is a special cursor called dot, and is written as `.`. You can think of this cursor as a variable whose value changes depending where in the template it is used. For example, at the start of a notification template `.` refers to something called [`ExtendedData`](ref:extendeddata) which contains a number of fields including `Alerts`, `Status`, `GroupLabels`, `CommonLabels`, `CommonAnnotations` and `ExternalURL`. However, dot might refer to something else when used in a range over a list, when used inside a `with`, or when writing feature templates to be used in other templates. You can see examples of this in [Create notification templates](ref:create-notification-templates), and all data and functions in the [Reference](ref:reference).
|
||||
|
||||
## Opening and closing tags
|
||||
|
||||
In text/template, templates start with `{{` and end with `}}` irrespective of whether the template prints a variable or executes control structures such as if statements. This is different from other templating languages such as Jinja where printing a variable uses `{{` and `}}` and control structures use `{%` and `%}`.
|
||||
|
||||
## Print
|
||||
|
||||
To print the value of something use `{{` and `}}`. You can print the value of dot, a field of dot, the result of a function, and the value of a [variable](#variables). For example, to print the `Alerts` field where dot refers to `ExtendedData` you would write the following:
|
||||
|
||||
```
|
||||
{{ .Alerts }}
|
||||
```
|
||||
|
||||
## Iterate over alerts
|
||||
|
||||
To print just the labels of each alert, rather than all information about the alert, you can use a `range` to iterate the alerts in `ExtendedData`:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ .Labels }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
Inside the range dot no longer refers to `ExtendedData`, but to an `Alert`. You can use `{{ .Labels }}` to print the labels of each alert. This works because `{{ range .Alerts }}` changes dot to refer to the current alert in the list of alerts. When the range is finished dot is reset to the value it had before the start of the range, which in this example is `ExtendedData`:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ .Labels }}
|
||||
{{ end }}
|
||||
{{/* does not work, .Labels does not exist here */}}
|
||||
{{ .Labels }}
|
||||
{{/* works, cursor was reset */}}
|
||||
{{ .Status }}
|
||||
```
|
||||
|
||||
## Iterate over annotations and labels
|
||||
|
||||
Let's write a template to print the labels of each alert in the format `The name of the label is $name, and the value is $value`, where `$name` and `$value` contain the name and value of each label.
|
||||
|
||||
Like in the previous example, use a range to iterate over the alerts in `.Alerts` such that dot refers to the current alert in the list of alerts, and then use a second range on the sorted labels so dot is updated a second time to refer to the current label. Inside the second range use `.Name` and `.Value` to print the name and value of each label:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ range .Labels.SortedPairs }}
|
||||
The name of the label is {{ .Name }}, and the value is {{ .Value }}
|
||||
{{ end }}
|
||||
{{ range .Annotations.SortedPairs }}
|
||||
The name of the annotation is {{ .Name }}, and the value is {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## The index function
|
||||
|
||||
To print a specific annotation or label use the `index` function.
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
The name of the alert is {{ index .Labels "alertname" }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## If statements
|
||||
|
||||
You can use if statements in templates. For example, to print `There are no alerts` if there are no alerts in `.Alerts` you would write the following:
|
||||
|
||||
```
|
||||
{{ if .Alerts }}
|
||||
There are alerts
|
||||
{{ else }}
|
||||
There are no alerts
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## With
|
||||
|
||||
With is similar to if statements, however unlike if statements, `with` updates dot to refer to the value of the with:
|
||||
|
||||
```
|
||||
{{ with .Alerts }}
|
||||
There are {{ len . }} alert(s)
|
||||
{{ else }}
|
||||
There are no alerts
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Variables
|
||||
|
||||
Variables in text/template must be created within the template. For example, to create a variable called `$variable` with the current value of dot you would write the following:
|
||||
|
||||
```
|
||||
{{ $variable := . }}
|
||||
```
|
||||
|
||||
You can use `$variable` inside a range or `with` and it will refer to the value of dot at the time the variable was defined, not the current value of dot.
|
||||
|
||||
For example, you cannot write a template that use `{{ .Labels }}` in the second range because here dot refers to the current label, not the current alert:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ range .Labels.SortedPairs }}
|
||||
{{ .Name }} = {{ .Value }}
|
||||
{{/* does not work because in the second range . is a label not an alert */}}
|
||||
There are {{ len .Labels }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
You can fix this by defining a variable called `$alert` in the first range and before the second range:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ $alert := . }}
|
||||
{{ range .Labels.SortedPairs }}
|
||||
{{ .Name }} = {{ .Value }}
|
||||
{{/* works because $alert refers to the value of dot inside the first range */}}
|
||||
There are {{ len $alert.Labels }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Range with index
|
||||
|
||||
You can get the index of each alert within a range by defining index and value variables at the start of the range:
|
||||
|
||||
```
|
||||
{{ $num_alerts := len .Alerts }}
|
||||
{{ range $index, $alert := .Alerts }}
|
||||
This is alert {{ $index }} out of {{ $num_alerts }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Define templates
|
||||
|
||||
You can define templates using `define` and the name of the template in double quotes. You should not define templates with the same name as other templates, including default templates such as `__subject`, `__text_values_list`, `__text_alert_list`, `default.title` and `default.message`. Where a template has been created with the same name as a default template, or a template in another notification template, Grafana might use either template. Grafana does not prevent, or show an error message, when there are two or more templates with the same name.
|
||||
|
||||
```
|
||||
{{ define "print_labels" }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Execute templates
|
||||
|
||||
You can execute defined templates using `template`, the name of the template in double quotes, and the cursor that should be passed to the template:
|
||||
|
||||
```
|
||||
{{ template "print_labels" . }}
|
||||
```
|
||||
|
||||
## Pass data to templates
|
||||
|
||||
Within a template dot refers to the value that is passed to the template.
|
||||
|
||||
For example, if a template is passed a list of firing alerts then dot refers to that list of firing alerts:
|
||||
|
||||
```
|
||||
{{ template "print_alerts" .Alerts }}
|
||||
```
|
||||
|
||||
If the template is passed the sorted labels for an alert then dot refers to the list of sorted labels:
|
||||
|
||||
```
|
||||
{{ template "print_labels" .SortedLabels }}
|
||||
```
|
||||
|
||||
This is useful when writing reusable templates. For example, to print all alerts you might write the following:
|
||||
|
||||
```
|
||||
{{ template "print_alerts" .Alerts }}
|
||||
```
|
||||
|
||||
Then to print just the firing alerts you could write this:
|
||||
|
||||
```
|
||||
{{ template "print_alerts" .Alerts.Firing }}
|
||||
```
|
||||
|
||||
This works because both `.Alerts` and `.Alerts.Firing` is a list of alerts.
|
||||
|
||||
```
|
||||
{{ define "print_alerts" }}
|
||||
{{ range . }}
|
||||
{{ template "print_labels" .SortedLabels }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## Comments
|
||||
|
||||
You can add comments with `{{/*` and `*/}}`:
|
||||
|
||||
```
|
||||
{{/* This is a comment */}}
|
||||
```
|
||||
|
||||
To prevent comments from adding line breaks use:
|
||||
|
||||
```
|
||||
{{- /* This is a comment with no leading or trailing line breaks */ -}}
|
||||
```
|
||||
|
||||
## Indentation
|
||||
|
||||
You can use indentation, both tabs and spaces, and line breaks, to make templates more readable:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ range .Labels.SortedPairs }}
|
||||
{{ .Name }} = {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
However, indentation in the template is also present in the text. Next, we look at how to remove it.
|
||||
|
||||
## Remove spaces and line breaks
|
||||
|
||||
In text/template use `{{-` and `-}}` to remove leading and trailing spaces and line breaks.
|
||||
|
||||
For example, when using indentation and line breaks to make a template more readable:
|
||||
|
||||
```
|
||||
{{ range .Alerts }}
|
||||
{{ range .Labels.SortedPairs }}
|
||||
{{ .Name }} = {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
The indentation and line breaks will also be present in the text:
|
||||
|
||||
```
|
||||
alertname = "Test"
|
||||
|
||||
grafana_folder = "Test alerts"
|
||||
```
|
||||
|
||||
You can remove the indentation and line breaks from the text changing `}}` to `-}}` at the start of each range:
|
||||
|
||||
```
|
||||
{{ range .Alerts -}}
|
||||
{{ range .Labels.SortedPairs -}}
|
||||
{{ .Name }} = {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
The indentation and line breaks in the template are now absent from the text:
|
||||
|
||||
```
|
||||
alertname = "Test"
|
||||
grafana_folder = "Test alerts"
|
||||
```
|
||||
@@ -13,6 +13,11 @@ menuTitle: Introduction
|
||||
title: Introduction to Alerting
|
||||
weight: 100
|
||||
refs:
|
||||
notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/
|
||||
notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
@@ -105,11 +110,15 @@ A rule using the PromQL expression above creates as many alert instances as the
|
||||
|
||||
[Contact points](ref:contact-points) determine the notification message and where notifications are sent. For example, you might have a contact point that sends notifications to an email address, to Slack, to an incident management system (IRM) such as Grafana OnCall or Pagerduty, or to a webhook.
|
||||
|
||||
### Notification messages
|
||||
|
||||
By default, notification messages include alert details, such as the number of alerts, their status, and annotations to help responders address alert issues. Notification messages can also be customized.
|
||||
|
||||
In the alert rule, you can choose a contact point to receive the alert notifications or use notification policies instead.
|
||||
|
||||
### Notification policies
|
||||
|
||||
[Notification policies](ref:notification-policies) provide a flexible method to handle alert notifications for larger systems.
|
||||
[Notification policies](ref:notification-policies) is an advanced option to handle alert notifications for larger systems.
|
||||
|
||||
Notification policies routes alerts to contact points via label matching. Each notification policy consists of a set of label matchers (0 or more) that specify which alert instances (identified by their labels) they handle. Notification policies are defined in a tree structure, where the root of the notification policy tree is the **Default notification policy**, which ensures all alert instances are handled.
|
||||
|
||||
@@ -129,12 +138,8 @@ Each notification policy decides where to send the alert (contact point) and whe
|
||||
|
||||
Grafana Alerting is built on the Prometheus model of designing alerting systems. Prometheus-based alerting systems have two main components:
|
||||
|
||||
- An alert generator that evaluates alert rules and sends firing and resolved alerts to the alert receiver.
|
||||
- An alert receiver (also known as Alertmanager) that receives the alerts and is responsible for handling them and sending their notifications.
|
||||
|
||||
Grafana Alerting doesn’t use Prometheus as its default alert generator because it works with many other data sources; not just Prometheus.
|
||||
|
||||
However, Grafana can also use Prometheus as an alert generator (refer to [alert rule types](ref:alert-rules)), as well as external Alertmanagers.
|
||||
- An alert generator that [evaluates alert rules](ref:alert-rule-evaluation) and sends firing and resolved alerts to the alert receiver.
|
||||
- An alert receiver (also known as Alertmanager) that receives the alerts and is responsible for sending their [notifications](ref:notifications).
|
||||
|
||||
## Design your Alerting system
|
||||
|
||||
@@ -147,23 +152,23 @@ Here are some tips on how to create an effective alert management set up for you
|
||||
**Which are the key metrics for your business that you want to monitor and alert on?**
|
||||
|
||||
- Find events that are important to know about and not so trivial or frequent that recipients ignore them.
|
||||
|
||||
- Alerts should only be created for big events that require immediate attention or intervention.
|
||||
|
||||
- Consider quality over quantity.
|
||||
|
||||
**Which type of Alerting do you want to use?**
|
||||
|
||||
- Choose between Grafana-managed Alerting or Grafana Mimir or Loki-managed Alerting; or both.
|
||||
|
||||
**How do you want to organize your alerts and notifications?**
|
||||
|
||||
- Be selective about who you set to receive alerts. Consider sending them to whoever is on call or a specific Slack channel.
|
||||
- Automate as far as possible using the Alerting API or alerts as code (Terraform).
|
||||
- Be selective about who you set to receive alerts. Consider sending them to the right teams, whoever is on call, and the specific channels.
|
||||
- Think carefully about priority and severity levels.
|
||||
- Automate as far as possible provisioning Alerting resources with the API or Terraform.
|
||||
|
||||
**Which information should you include in notifications?**
|
||||
|
||||
- Consider who the alert receivers and responders are.
|
||||
- Share information that helps responders identify and address potential issues.
|
||||
- Link alerts to dashboards to guide responders on which data to investigate.
|
||||
|
||||
**How can you reduce alert fatigue?**
|
||||
|
||||
- Avoid noisy, unnecessary alerts by using silences, mute timings, or pausing alert rule evaluation.
|
||||
- Continually tune your alert rules to review effectiveness. Remove alert rules to avoid duplication or ineffective alerts.
|
||||
- Think carefully about priority and severity levels.
|
||||
- Continually review your thresholds and evaluation rules.
|
||||
|
||||
@@ -43,7 +43,7 @@ Alert rules in different groups can be evaluated simultaneously.
|
||||
|
||||
- **Grafana-managed** alert rules within the same group are evaluated concurrently—they are evaluated at different times over the same evaluation interval but display the same evaluation timestamp.
|
||||
|
||||
- **Data-source managed** alert rules within the same group are evaluated sequentially, one after the other—this is necessary to ensure that recording rules are evaluated before alert rules.
|
||||
- **Data-source managed** alert rules within the same group are evaluated sequentially, one after the other—this is useful to ensure that recording rules are evaluated before alert rules.
|
||||
|
||||
## Pending period
|
||||
|
||||
|
||||
@@ -44,35 +44,58 @@ There are three key components that help you understand how your alerts behave d
|
||||
|
||||
An alert instance can be in either of the following states:
|
||||
|
||||
| State | Description |
|
||||
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Normal** | The state of an alert when the condition (threshold) is not met. |
|
||||
| **Pending** | The state of an alert that has breached the threshold but for less than the [pending period](ref:pending-period). |
|
||||
| **Alerting** | The state of an alert that has breached the threshold for longer than the [pending period](ref:pending-period). |
|
||||
| **NoData** | The state of an alert whose query returns no data or all values are null. You can [change the default behavior of the no data state](#modify-the-no-data-and-error-state). |
|
||||
| **Error** | The state of an alert when an error or timeout occurred evaluating the alert rule. You can [change the default behavior of the error state](#modify-the-no-data-and-error-state). |
|
||||
| State | Description |
|
||||
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Normal** | The state of an alert when the condition (threshold) is not met. |
|
||||
| **Pending** | The state of an alert that has breached the threshold but for less than the [pending period](ref:pending-period). |
|
||||
| **Alerting** | The state of an alert that has breached the threshold for longer than the [pending period](ref:pending-period). |
|
||||
| **No Data<sup>\*</sup>** | The state of an alert whose query returns no data or all values are null. <br/> An alert in this state generates a new [DatasourceNoData alert](#no-data-and-error-alerts). You can [modify the default behavior of the no data state](#modify-the-no-data-or-error-state). |
|
||||
| **Error<sup>\*</sup>** | The state of an alert when an error or timeout occurred evaluating the alert rule. <br/> An alert in this state generates a new [DatasourceError alert](#no-data-and-error-alerts). You can [modify the default behavior of the error state](#modify-the-no-data-or-error-state). |
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
|
||||
`No Data` and `Error` states are supported only for Grafana-managed alert rules.
|
||||
|
||||
{{< /admonition >}}
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-instance-states-v3.png" caption="Alert instance state diagram" alt="A diagram of the distinct alert instance states and transitions." max-width="750px" >}}
|
||||
|
||||
### Notifications
|
||||
### Notification routing
|
||||
|
||||
Alert instances will be routed for [notifications](ref:notifications) when they are in the `Alerting` state or have been `Resolved`, transitioning from `Alerting` to `Normal` state.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-rule-evaluation-overview-statediagram-v2.png" alt="A diagram of the alert instance states and when to route their notifications." max-width="750px" >}}
|
||||
|
||||
### `No Data` and `Error` alerts
|
||||
|
||||
When evaluation of an alert rule produces state `No Data` or `Error`, Grafana Alerting generates a new alert instance that have the following additional labels:
|
||||
|
||||
- `alertname`: Either `DatasourceNoData` or `DatasourceError` depending on the state.
|
||||
- `datasource_uid`: The UID of the data source that caused the state.
|
||||
|
||||
You can manage these alerts like regular ones by using their labels to apply actions such as adding a silence, routing via notification policies, and more.
|
||||
|
||||
### Lifecycle of stale alert instances
|
||||
|
||||
An alert instance is considered stale if its dimension or series has disappeared from the query results entirely for two evaluation intervals.
|
||||
|
||||
Stale alert instances that are in the **Alerting**, **NoData**, or **Error** states transition to the **Normal** state as **Resolved**. Once transitioned, these resolved alert instances are routed for notifications like other resolved alerts.
|
||||
Stale alert instances that are in the **Alerting**, **No Data**, or **Error** states transition to the **Normal** state as **Resolved**. Once transitioned, these resolved alert instances are routed for notifications like other resolved alerts.
|
||||
|
||||
### Modify the no data and error state
|
||||
## Modify the `No Data` or `Error` state
|
||||
|
||||
In [Configure no data and error handling](ref:no-data-and-error-handling), you can change the default behaviour when the evaluation returns no data or an error. You can set the alert instance state to `Alerting`, `Normal`, or keep the last state.
|
||||
|
||||
Note that `No Data` and `Error` states are supported only for Grafana-managed alert rules.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-rule-configure-no-data-and-error.png" alt="A screenshot of the `Configure no data and error handling` option in Grafana Alerting." max-width="500px" >}}
|
||||
|
||||
To reduce the number of **No Data** or **Error** state alerts received, try the following.
|
||||
{{< docs/shared lookup="alerts/table-configure-no-data-and-error.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
Note that when you configure the **No Data** or **Error** behavior to `Alerting` or `Normal`, Grafana attempts to keep a stable set of fields under notification `Values`. If your query returns no data or an error, Grafana reuses the latest known set of fields in `Values`, but will use `-1` in place of the measured value.
|
||||
|
||||
### Reduce `No Data` or `Error` alerts
|
||||
|
||||
To minimize the number of **No Data** or **Error** state alerts received, try the following.
|
||||
|
||||
1. Use the **Keep last state** option. For more information, refer to the section below. This option allows the alert to retain its last known state when there is no data available, rather than switching to a **No Data** state.
|
||||
1. For **No Data** alerts, you can optimize your alert rule by expanding the time range of the query. However, if the time range is too big, it affects the performance of the query and can lead to errors due to timeout.
|
||||
@@ -81,38 +104,29 @@ To reduce the number of **No Data** or **Error** state alerts received, try the
|
||||
|
||||
1. Change the default [evaluation time out](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#evaluation_timeout). The default is set at 30 seconds. To increase the default evaluation timeout, open a support ticket from the [Cloud Portal](https://grafana.com/docs/grafana-cloud/account-management/support/#grafana-cloud-support-options). Note that this should be a last resort, because it may affect the performance of all alert rules and cause missed evaluations if the timeout is too long.
|
||||
|
||||
#### Keep last state
|
||||
### Keep last state
|
||||
|
||||
The "Keep Last State" option helps mitigate temporary data source issues, preventing alerts from unintentionally firing, resolving, and re-firing.
|
||||
|
||||
However, in situations where strict monitoring is critical, relying solely on the "Keep Last State" option may not be appropriate. Instead, consider using an alternative or implementing additional alert rules to ensure that issues with prolonged data source disruptions are detected.
|
||||
|
||||
### `grafana_state_reason` annotation
|
||||
## `grafana_state_reason` for troubleshooting
|
||||
|
||||
Occasionally, an alert instance may be in a state that isn't immediately clear to everyone. For example:
|
||||
|
||||
- Stale alert instances in the `Alerting` state transition to the `Normal` state when the series disappear.
|
||||
- If "no data" handling is configured to transition to a state other than `NoData`.
|
||||
- If "no data" handling is configured to transition to a state other than `No Data`.
|
||||
- If "error" handling is configured to transition to a state other than `Error`.
|
||||
- If the alert rule is deleted, paused, or updated in some cases, the alert instance also transitions to the `Normal` state.
|
||||
|
||||
In these situations, the evaluation state may differ from the alert state, and it might be necessary to understand the reason for being in that state when receiving the notification.
|
||||
|
||||
The `grafana_state_reason` annotation is included in these situations, providing the reason in the notifications that explain why the alert instance transitioned to its current state. For example:
|
||||
The `grafana_state_reason` annotation is included in these situations, providing the reason that explains why the alert instance transitioned to its current state. For example:
|
||||
|
||||
- Stale alert instances in the `Normal` state include the `grafana_state_reason` annotation with the value **MissingSeries**.
|
||||
- If "no data" or "error" handling transitions to the `Normal` state, the `grafana_state_reason` annotation is included with the value **NoData** or **Error**, respectively.
|
||||
- If "no data" or "error" handling transitions to the `Normal` state, the `grafana_state_reason` annotation is included with the value **No Data** or **Error**, respectively.
|
||||
- If the alert rule is deleted or paused, the `grafana_state_reason` is set to **Paused** or **RuleDeleted**. For some updates, it is set to **Updated**.
|
||||
|
||||
### Special alerts for `NoData` and `Error`
|
||||
|
||||
When evaluation of an alert rule produces state `NoData` or `Error`, Grafana Alerting generates a new alert instance that have the following additional labels:
|
||||
|
||||
- `alertname`: Either `DatasourceNoData` or `DatasourceError` depending on the state.
|
||||
- `datasource_uid`: The UID of the data source that caused the state.
|
||||
|
||||
You can manage these alerts like regular ones by using their labels to apply actions such as adding a silence, routing via notification policies, and more.
|
||||
|
||||
## Alert rule state
|
||||
|
||||
The alert rule state is determined by the “worst case” state of the alert instances produced. For example, if one alert instance is `Alerting`, the alert rule state is firing.
|
||||
@@ -133,5 +147,5 @@ An alert rule can have one of the following health statuses:
|
||||
| ---------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||
| **Ok** | No error when evaluating an alerting rule. |
|
||||
| **Error** | An error occurred when evaluating an alerting rule. |
|
||||
| **NoData** | The absence of data in at least one time series returned during a rule evaluation. |
|
||||
| **No Data** | The absence of data in at least one time series returned during a rule evaluation. |
|
||||
| **{status}, KeepLast** | The rule would have received another status but was configured to keep the last state of the alert rule. |
|
||||
|
||||
@@ -3,10 +3,7 @@ aliases:
|
||||
- ../fundamentals/data-source-alerting/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/data-source-alerting/
|
||||
- ../fundamentals/alert-rules/alert-instances/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/alert-instances/
|
||||
- ../fundamentals/alert-rules/organising-alerts/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/organising-alerts/
|
||||
- ../fundamentals/alert-rules/recording-rules/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/recording-rules/
|
||||
- ../fundamentals/alert-rules/alert-rule-types/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/alert-rule-types/
|
||||
- ../unified-alerting/alerting-rules/edit-cortex-loki-namespace-group/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/edit-cortex-loki-namespace-group/
|
||||
- ../unified-alerting/alerting-rules/edit-mimir-loki-namespace-group/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/edit-mimir-loki-namespace-group/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/fundamentals/alert-rules/
|
||||
description: Learn about alert rules
|
||||
keywords:
|
||||
@@ -21,14 +18,16 @@ labels:
|
||||
title: Alert rules
|
||||
weight: 100
|
||||
refs:
|
||||
shared-configure-prometheus-data-source-alerting:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/datasources/prometheus/configure-prometheus-data-source/#alerting
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/prometheus/configure-prometheus-data-source/#alerting
|
||||
queries-and-conditions:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/
|
||||
recorded-queries:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/administration/recorded-queries/
|
||||
notification-images:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/images-in-notifications/
|
||||
@@ -41,14 +40,19 @@ refs:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/
|
||||
create-recording-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-recording-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-mimir-loki-managed-recording-rule/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/
|
||||
expression-queries:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
|
||||
alert-condition:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
|
||||
alert-rule-evaluation:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/rule-evaluation/
|
||||
@@ -69,54 +73,55 @@ Grafana supports two different alert rule types: Grafana-managed alert rules and
|
||||
|
||||
## Grafana-managed alert rules
|
||||
|
||||
Grafana-managed alert rules are the most flexible alert rule type. They allow you to create alert rules that can act on data from any of the [supported data sources](#supported-data-sources), and use multiple data sources in a single alert rule. You can add [expressions to transform your data](ref:expression-queries) and set custom alert conditions. Using [images in alert notifications](ref:notification-images). is also supported.
|
||||
Grafana-managed alert rules are the most flexible alert rule type. They allow you to create alert rules that can act on data from any of the [supported data sources](#supported-data-sources), and use multiple data sources in a single alert rule.
|
||||
|
||||
{{< figure src="/media/docs/alerting/grafana-managed-alerting-architecture.png" max-width="750px" caption="How Grafana-managed alerting works by default" >}}
|
||||
|
||||
1. Alert rules are created within Grafana based on one or more data sources.
|
||||
1. Alert rules are evaluated by the Alert Rule Evaluation Engine from within Grafana.
|
||||
1. Alert rules are created and stored within Grafana.
|
||||
1. Alert rules can query one or more supported data sources.
|
||||
1. Alert rules are evaluated by the Alert Rule Evaluation Engine within Grafana.
|
||||
1. Firing and resolved alert instances are forwarded to [handle their notifications](ref:notifications).
|
||||
|
||||
### Supported data sources
|
||||
|
||||
Grafana-managed alert rules can query backend data sources if Grafana Alerting is enabled by specifying `{"backend": true, "alerting": true}` in the [plugin.json](https://grafana.com/developers/plugin-tools/reference/plugin-json).
|
||||
Grafana-managed alert rules can query backend data sources if Grafana Alerting is enabled by specifying `{"backend": true, "alerting": true}` in the `plugin.json` file.
|
||||
|
||||
Find the public data sources supporting Alerting in the [Grafana Plugins directory](/grafana/plugins/data-source-plugins/?features=alerting).
|
||||
|
||||
## Data source-managed alert rules
|
||||
|
||||
Data source-managed alert rules can be used for Grafana Mimir or Grafana Loki data sources which have been configured to support rule creation.
|
||||
|
||||
They can improve query performance via [recording rules](#recording-rules) and ensure high-availability and fault tolerance when implementing a distributed architecture.
|
||||
|
||||
They are only supported for Grafana Mimir or Grafana Loki data sources with the Ruler API enabled. For more information, refer to the [Loki Ruler API](/docs/loki/<GRAFANA_VERSION>/api/#ruler) or [Mimir Ruler API](/docs/mimir/<GRAFANA_VERSION>/references/http-api/#ruler).
|
||||
Data source-managed alert rules can only be created using Grafana Mimir or Grafana Loki data sources. Both data source backends can provide high availability and fault tolerance, enabling you to scale your alerting setup.
|
||||
|
||||
{{< figure src="/media/docs/alerting/mimir-managed-alerting-architecture-v2.png" max-width="750px" caption="Mimir-managed alerting architecture" >}}
|
||||
|
||||
1. Alert rules are created and stored within the data source itself.
|
||||
1. Alert rules can only query Prometheus-based data. It can use either queries or [recording rules](#recording-rules).
|
||||
1. Alert rules are evaluated by the Alert Rule Evaluation Engine.
|
||||
1. Alert rules are stored within the Mimir or Loki data source.
|
||||
1. Alert rules can query only their specific data source.
|
||||
1. Alert rules are evaluated by the Alert Rule Evaluation Engine within the data source.
|
||||
1. Firing and resolved alert instances are forwarded to [handle their notifications](ref:notifications).
|
||||
|
||||
## Recording rules
|
||||
|
||||
A recording rule allows you to pre-compute frequently needed or computationally expensive expressions and save their result as a new set of time series. This is useful if you want to run alerts on aggregated data or if you have dashboards that query computationally expensive expressions repeatedly.
|
||||
|
||||
Querying this new time series is faster, especially for dashboards since they query the same expression every time the dashboards refresh. For more information, refer to [Create recording rules](ref:create-recording-rules).
|
||||
|
||||
Alternatively, Grafana Enterprise and Grafana Cloud offer [recorded queries](ref:recorded-queries) that can be executed against any data source.
|
||||
{{< docs/shared lookup="alerts/note-prometheus-ds-rules.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
## Comparison between alert rule types
|
||||
|
||||
When choosing which alert rule type to use, consider the following comparison between Grafana-managed and data source-managed alert rules.
|
||||
We recommend using Grafana-managed alert rules whenever possible, and opting for data source-managed alert rules when you need to scale your alerting setup.
|
||||
|
||||
| <div style="width:200px">Feature</div> | <div style="width:200px">Grafana-managed alert rule</div> | <div style="width:200px">Data source-managed alert rule |
|
||||
| ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Create alert rules<wbr /> based on data from any of the supported data sources | Yes | No. You can only create alert rules that are based on Prometheus-based data. |
|
||||
| Mix and match data sources | Yes | No |
|
||||
| Includes support for recording rules | Yes. Only for Grafana OSS users with the `grafanaManagedRecordingRules` feature flag enabled. | Yes |
|
||||
| Add expressions to transform<wbr /> your data and set alert conditions | Yes | No |
|
||||
| Use images in alert notifications | Yes | No |
|
||||
| Organization | Organize and manage access with folders | Use namespaces |
|
||||
| Scaling | More resource intensive, depend on the database, and are likely to suffer from transient errors. They only scale vertically. | Store alert rules within the data source itself and allow for “infinite” scaling. Generate and send alert notifications from the location of your data. |
|
||||
| Alert rule evaluation and delivery | Alert rule evaluation and delivery is done from within Grafana, using an external Alertmanager; or both. | Alert rule evaluation and alert delivery is distributed, meaning there is no single point of failure. |
|
||||
The table below compares Grafana-managed and data source-managed alert rules.
|
||||
|
||||
| <div style="width:200px">Feature</div> | <div style="width:200px">Grafana-managed alert rule</div> | <div style="width:200px">Data source-managed alert rule |
|
||||
| ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
||||
| Create alert rules<wbr /> that query [data sources supporting Alerting](#supported-data-sources) | Yes | Only supports creating rules for Mimir and Loki. |
|
||||
| Mix and match data sources | Yes | No |
|
||||
| Add [expressions](ref:expression-queries) to transform<wbr /> your data and set [alert conditions](ref:alert-condition) | Yes | No |
|
||||
| Use [images in alert notifications](ref:notification-images) | Yes | No |
|
||||
| Support for [recording rules](#recording-rules) | Yes | Yes |
|
||||
| Organization | Organize and manage access with folders | Use namespaces |
|
||||
| Alert rule evaluation and delivery | Alert evaluation is done in Grafana, while delivery can be handled by Grafana or an external Alertmanager. | Alert rule evaluation and alert delivery are distributed. |
|
||||
| Scaling | Alert rules are stored in the Grafana database, which may experience transient errors. It only scales vertically. | Alert rules are stored within the data source and allow for horizontal scaling. |
|
||||
|
||||
## Recording rules
|
||||
|
||||
Similar to alert rules, recording rules are evaluated periodically. A recording rule pre-computes frequently used or computationally expensive queries, and saves the results as a new time series metric.
|
||||
|
||||
The new recording metric can then be used in alert rules and dashboards to optimize their queries.
|
||||
|
||||
For more details, refer to [Create recording rules](ref:create-recording-rules).
|
||||
|
||||
@@ -26,16 +26,16 @@ refs:
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals#alert-instances
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals#alert-instances
|
||||
create-alerts-from-panel:
|
||||
link-alert-rules-to-panels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule#create-alerts-from-panels
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-grafana-managed-rule#create-alerts-from-panels
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/templates/
|
||||
alert-rule-evaluation:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/
|
||||
@@ -57,8 +57,8 @@ refs:
|
||||
|
||||
Labels and annotations add additional information about an alert using key/value pairs:
|
||||
|
||||
- [Labels](#labels) are used to differentiate an alert from all other alerts.
|
||||
- [Annotations](#annotations) are used to provide extra detail on an existing alert.
|
||||
- [Labels](#labels) are used to differentiate an alert from all other alerts and decide how to manage them.
|
||||
- [Annotations](#annotations) provide extra details for alert responders to help them understand and address potential issues.
|
||||
|
||||
## Labels
|
||||
|
||||
@@ -83,24 +83,26 @@ An alert's label set can contain three types of labels:
|
||||
|
||||
**User-configured labels**
|
||||
|
||||
Labels that you manually configure in the alert rule to identify the generated alert instances or group them.
|
||||
Labels that you manually configure in the alert rule to identify the generated alert instances and manage the alerts. Common custom labels, depending on the use case, are: `severity`, `priority`, `team`, and `service`.
|
||||
|
||||
You can also use a [template](ref:templates) to customize the label value and generate dynamic values when the rule is evaluated.
|
||||
Additionally, you can use a [template](ref:templates) to customize the label value and generate dynamic values from query data.
|
||||
|
||||
**Data source query labels**
|
||||
**Query labels**
|
||||
|
||||
For example, if you are monitoring temperature readings and each time series for these readings has a `sensor_id`, and a `location` label, an alert instance might have the labels `{sensor_id="1",location="NY"}`, while another alert instance might have `{sensor_id="2",location="WA"}`.
|
||||
Query labels are labels returned by the data source query.
|
||||
|
||||
Data source query labels labels are also used to generate multiple alert instances from the same alert rule, helping to distinguish alerts from different data.
|
||||
{{< figure src="/media/docs/alerting/query-labels-and-values.png" max-width="1200px" caption="An alert rule query returning labels from the query." >}}
|
||||
|
||||
Query labels can generate multiple alert instances from the same alert rule, helping to distinguish alerts from different data. In this example, the `instance` label generates an alert instance for each server.
|
||||
|
||||
**Reserved labels**
|
||||
|
||||
Reserved labels are automatically added by Grafana:
|
||||
|
||||
- `alert_name`: the name of the alert rule.
|
||||
- `alertname`: the name of the alert rule.
|
||||
- `grafana_folder`: the title of the folder containing the alert.
|
||||
|
||||
Labels prefixed with `grafana_` are reserved by Grafana for special use. To stop Grafana Alerting from adding a reserved label, you can disable it via the `disabled_labels` option in [unified_alerting.reserved_labels](/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana#unified_alertingreserved_labels) configuration.
|
||||
Labels prefixed with `grafana_` are reserved by Grafana for special use. You can disable reserved labels via the [`unified_alerting.reserved_labels`](/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana#unified_alertingreserved_labels) option.
|
||||
|
||||
{{<admonition type="note">}}
|
||||
|
||||
@@ -132,13 +134,29 @@ If multiple label keys are sanitized to the same value, the duplicates have a sh
|
||||
|
||||
## Annotations
|
||||
|
||||
The purpose of annotations is to add additional information to alert instances, such as extra details for notification messages.
|
||||
Annotations add additional information to alert instances, helping responders identify and address potential issues.
|
||||
|
||||
Grafana provides several optional annotations that you can edit for use in notification messages and within Grafana.
|
||||
Annotations are displayed in Grafana and are included by default in notifications. Grafana provides several optional annotations that you can edit:
|
||||
|
||||
- `summary`: A short summary of what the alert has detected and why.
|
||||
- `description`: A detailed description of what happened and what the alert does.
|
||||
- `runbook_url`: The runbook page to guide operators managing a potential incident.
|
||||
- `dashboardUId` and `panelId`: Link the alert to a dashboard and panel. These are automatically set when [creating an alert from panels](ref:create-alerts-from-panel).
|
||||
- `dashboardUId` and `panelId`: [Link the alert to a dashboard and panel](ref:link-alert-rules-to-panels) to facilitate alert investigation.
|
||||
|
||||
Like labels, annotations can use a [template](ref:templates) to customize the label value and generate dynamic values when the rule is evaluated.
|
||||
For example, you can edit the annotation `summary` to explain why the alert was triggered:
|
||||
|
||||
```
|
||||
CPU usage has exceeded 80% for the last 5 minutes.
|
||||
```
|
||||
|
||||
And edit the `description` annotation to provide more context and how to respond:
|
||||
|
||||
```
|
||||
The web server's CPU has exceeded 80% for more than 5 minutes.
|
||||
|
||||
This indicates that the system is under heavy load and may result in an outage.
|
||||
|
||||
Consider scaling the server's resources and investigating bottlenecks.
|
||||
```
|
||||
|
||||
Like labels, annotations can use a [template](ref:templates) to include dynamic data from queries.
|
||||
|
||||
@@ -43,7 +43,7 @@ refs:
|
||||
|
||||
In Grafana, queries fetch and transform data from [data sources,](ref:data-sources) which include databases like MySQL or PostgreSQL, time series databases like Prometheus or InfluxDB, and services like Amazon CloudWatch or Azure Monitor.
|
||||
|
||||
A query specifies the desired data to retrieve from a data source and applies relevant filters or transformations. Query languages or syntax specific to the chosen data source are utilized for constructing these queries.
|
||||
A query specifies the data to extract from a data source, with the syntax varying based on the type of data source used.
|
||||
|
||||
In Alerting, an alert rule defines of one or more queries and expressions that select the data you want to measure and a [condition](#alert-condition) that needs to be met before an alert rule fires.
|
||||
|
||||
|
||||
@@ -27,9 +27,9 @@ refs:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/templates/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/templates/
|
||||
configure-alertmanager:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
@@ -84,7 +84,9 @@ Start defining your [contact points](ref:contact-points) to specify how to recei
|
||||
|
||||
[Contact points](ref:contact-points) contain the configuration for sending alert notifications, specifying destinations like email, Slack, OnCall, webhooks, and their notification messages.
|
||||
|
||||
A contact point is a list of integrations, each sending a message to a specific destination. They allow the customization of notification messages and the use of notification templates.
|
||||
A contact point is a list of integrations, each sending a message to a specific destination.
|
||||
|
||||
By default, notification messages include common alert details, such as the number of alerts, alert names, labels, annotations, and other alert information. You can also customize notification messages and use notification templates.
|
||||
|
||||
First, create the contact point and test the notifications. Then, configure the alert rule to send its notifications to either a contact point or through Notification Policies.
|
||||
|
||||
@@ -132,6 +134,4 @@ Grafana Alerting is based on the Prometheus model for designing alerting systems
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-alertmanager-architecture.png" max-width="750px" alt="A diagram with the alert generator and alert manager architecture" >}}
|
||||
|
||||
Grafana uses a custom Alertmanager to manage and deliver alert notifications, as detailed in this guide. This custom Alertmanager extends the capabilities of the [Prometheus Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/).
|
||||
|
||||
If you already run a Prometheus Alertmanager instance, you can configure Grafana Alerting to forward alerts to your [external Alertmanager for handling notifications](ref:configure-alertmanager).
|
||||
Grafana provides a custom Alertmanager, extending the Prometheus Alertmanager, to manage and deliver alert notifications. If you run a Prometheus or Mimir Alertmanager, you can configure Grafana Alerting to manage them and handle notifications for Grafana-managed alerts. For details, refer to [configure Alertmanagers](ref:configure-alertmanager).
|
||||
|
||||
@@ -21,19 +21,48 @@ labels:
|
||||
title: Contact points
|
||||
weight: 112
|
||||
refs:
|
||||
contact-point-integrations:
|
||||
configure-contact-points:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points#list-of-supported-integrations
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points#list-of-supported-integrations
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points
|
||||
---
|
||||
|
||||
# Contact points
|
||||
|
||||
Contact points contain the configuration for sending alert notifications. You can assign a contact point either in the alert rule or notification policy options.
|
||||
|
||||
A contact point is a [list of integrations](ref:contact-point-integrations), each of which sends a notification to a particular email, webhook, or service such as Slack, Pagerduty, or Grafana OnCall. Each contact point also defines the notification message to be sent, which can use the predefined message, a custom message, or notification templates.
|
||||
A contact point includes one or more contact point integrations for sending alert notifications, such as:
|
||||
|
||||
Contact points can have multiple integrations of the same kind, or a combination of integrations of different kinds. For example, a contact point could contain a Pagerduty integration; an email and Slack integration; or a Pagerduty integration, a Slack integration, and two email integrations.
|
||||
{{< column-list >}}
|
||||
|
||||
You can also configure a contact point with no integrations; in which case no notifications are sent.
|
||||
- Alertmanager
|
||||
- Amazon SNS
|
||||
- Cisco Webex Teams
|
||||
- DingDing
|
||||
- Discord
|
||||
- Email
|
||||
- Google Chat
|
||||
- Grafana Oncall
|
||||
- Kafka REST Proxy
|
||||
- Line
|
||||
- Microsoft Teams
|
||||
- MQTT
|
||||
- Opsgenie
|
||||
- Pagerduty
|
||||
- Pushover
|
||||
- Sensu Go
|
||||
- Slack
|
||||
- Telegram
|
||||
- Threema Gateway
|
||||
- VictorOps
|
||||
- Webhook
|
||||
- WeCom
|
||||
|
||||
{{< /column-list >}}
|
||||
|
||||
For example, a contact point could contain a Pagerduty integration; an email and Slack integration; or a Pagerduty integration, a Slack integration, and two email integrations. You can also configure a contact point with no integrations; in which case no notifications are sent.
|
||||
|
||||
Each contact point integration can also define the notification message to be sent, which can use the predefined message, a custom message, or notification templates.
|
||||
|
||||
For a complete list of supported integrations and more details about contact points, refer to [Configure contact points](ref:configure-contact-points).
|
||||
|
||||
@@ -35,6 +35,10 @@ refs:
|
||||
|
||||
Grouping in Grafana Alerting allows you to batch relevant alerts together into a smaller number of notifications. This is particularly important if notifications are delivered to first-responders, such as engineers on-call, where receiving lots of notifications in a short period of time can be overwhelming. In some cases, it can negatively impact a first-responders ability to respond to an incident. For example, consider a large outage where many of your systems are down. In this case, grouping can be the difference between receiving 1 phone call and 100 phone calls.
|
||||
|
||||
{{< admonition type="tip" >}}
|
||||
For a practical example of grouping, refer to our [Getting Started with Grouping tutorial](https://grafana.com/tutorials/alerting-get-started-pt3/).
|
||||
{{< /admonition >}}
|
||||
|
||||
## Group notifications
|
||||
|
||||
Grouping combines similar alert instances within a specific period into a single notification, reducing alert noise.
|
||||
@@ -105,15 +109,15 @@ Consider a notification policy that:
|
||||
- Groups notifications by the `team` label—one group for each distinct `team`.
|
||||
- Sets the Group wait timer to `30s`.
|
||||
|
||||
| Time | Incoming alert instance | Notification policy group | Number of instances | |
|
||||
| ------------------ | ------------------------------- | ------------------------- | ------------------- | ----------------------------------------------------------------------- |
|
||||
| 00:00 | `alert_name=f1` `team=frontend` | `frontend` | 1 | Starts the group wait timer of the `frontend` group. |
|
||||
| 00:10 | `alert_name=f2` `team=frontend` | `frontend` | 2 | |
|
||||
| 00:20 | `alert_name=b1` `team=backend` | `backend` | 1 | Starts the group wait timer of the `backend` group. |
|
||||
| 00:30<sup>\*</sup> | | `frontend` | 2 | Group wait elapsed. <br/> Send initial notification reporting 2 alerts. |
|
||||
| 00:35 | `alert_name=b2` `team=backend` | `backend` | 2 | |
|
||||
| 00:40 | `alert_name=b3` `team=backend` | `backend` | 3 | |
|
||||
| 00:50<sup>\*</sup> | | `backend` | 3 | Group wait elapsed. <br/> Send initial notification reporting 3 alerts. |
|
||||
| Time | Incoming alert instance | Notification policy group | Number of instances | |
|
||||
| ------------------ | ------------------------------ | ------------------------- | ------------------- | ----------------------------------------------------------------------- |
|
||||
| 00:00 | `alertname=f1` `team=frontend` | `frontend` | 1 | Starts the group wait timer of the `frontend` group. |
|
||||
| 00:10 | `alertname=f2` `team=frontend` | `frontend` | 2 | |
|
||||
| 00:20 | `alertname=b1` `team=backend` | `backend` | 1 | Starts the group wait timer of the `backend` group. |
|
||||
| 00:30<sup>\*</sup> | | `frontend` | 2 | Group wait elapsed. <br/> Send initial notification reporting 2 alerts. |
|
||||
| 00:35 | `alertname=b2` `team=backend` | `backend` | 2 | |
|
||||
| 00:40 | `alertname=b3` `team=backend` | `backend` | 3 | |
|
||||
| 00:50<sup>\*</sup> | | `backend` | 3 | Group wait elapsed. <br/> Send initial notification reporting 3 alerts. |
|
||||
|
||||
### Group interval
|
||||
|
||||
@@ -134,15 +138,15 @@ Here are the related excerpts from the previous example:
|
||||
|
||||
And below is the continuation of the example setting the Group interval timer to 5 minutes:
|
||||
|
||||
| Time | Incoming alert instance | Notification policy group | Number of instances | |
|
||||
| ------------------ | ------------------------------- | ------------------------- | ------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| 01:30 | `alert_name=f3` `team=frontend` | `frontend` | 3 | |
|
||||
| 02:30 | `alert_name=f4` `team=frontend` | `frontend` | 4 | |
|
||||
| 05:30<sup>\*</sup> | | `frontend` | 4 | Group interval elapsed and resets timer. <br/> Send one notification reporting 4 alerts. |
|
||||
| 05:50<sup>\*</sup> | | `backend` | 3 | Group interval elapsed and resets timer. <br/> No group changes, and do not send notification. |
|
||||
| 08:00 | `alert_name=f4` `team=backend` | `backend` | 4 | |
|
||||
| 10:30<sup>\*</sup> | | `frontend` | 4 | Group interval elapsed and resets timer. <br/> No group changes, and do not send notification. |
|
||||
| 10:50<sup>\*</sup> | | `backend` | 4 | Group interval elapsed and resets timer. <br/> Send one notification reporting 4 alerts. |
|
||||
| Time | Incoming alert instance | Notification policy group | Number of instances | |
|
||||
| ------------------ | ------------------------------ | ------------------------- | ------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| 01:30 | `alertname=f3` `team=frontend` | `frontend` | 3 | |
|
||||
| 02:30 | `alertname=f4` `team=frontend` | `frontend` | 4 | |
|
||||
| 05:30<sup>\*</sup> | | `frontend` | 4 | Group interval elapsed and resets timer. <br/> Send one notification reporting 4 alerts. |
|
||||
| 05:50<sup>\*</sup> | | `backend` | 3 | Group interval elapsed and resets timer. <br/> No group changes, and do not send notification. |
|
||||
| 08:00 | `alertname=f4` `team=backend` | `backend` | 4 | |
|
||||
| 10:30<sup>\*</sup> | | `frontend` | 4 | Group interval elapsed and resets timer. <br/> No group changes, and do not send notification. |
|
||||
| 10:50<sup>\*</sup> | | `backend` | 4 | Group interval elapsed and resets timer. <br/> Send one notification reporting 4 alerts. |
|
||||
|
||||
**How it works**
|
||||
|
||||
|
||||
@@ -18,6 +18,21 @@ labels:
|
||||
title: Notification policies
|
||||
weight: 113
|
||||
refs:
|
||||
shared-alert-labels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/annotation-label/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/
|
||||
shared-notification-policies:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/notification-policies/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/notification-policies/
|
||||
shared-silences:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/create-silence/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/create-silence/
|
||||
contact-points:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/contact-points/
|
||||
@@ -60,6 +75,10 @@ Each policy consists of a set of label matchers (0 or more) that specify which a
|
||||
|
||||
{{< figure src="/media/docs/alerting/notification-routing.png" max-width="750px" caption="Matching alert instances with notification policies" alt="Example of a notification policy tree" >}}
|
||||
|
||||
{{< admonition type="tip" >}}
|
||||
For a practical example of routing with notification policies, refer to our [Getting Started with Alert Instances and Notification Routing tutorial](https://grafana.com/tutorials/alerting-get-started-pt4/).
|
||||
{{< /admonition >}}
|
||||
|
||||
## Routing
|
||||
|
||||
To determine which notification policies handle an alert instance, the system looks for matching policies starting from the top of the tree—beginning with the default notification policy.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
---
|
||||
aliases:
|
||||
- ../fundamentals/notifications/templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/templates/
|
||||
- ../fundamentals/notifications/templates/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/templates/
|
||||
- ../contact-points/message-templating/ # /docs/grafana/<GRAFANA_VERSION>/alerting/contact-points/message-templating/
|
||||
- ../alert-rules/message-templating/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alert-rules/message-templating/
|
||||
- ../unified-alerting/message-templating/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/message-templating/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/fundamentals/notifications/templates/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/fundamentals/templates/
|
||||
description: Use templating to customize, format, and reuse alert notification messages. Create more flexible and informative alert notification messages by incorporating dynamic content, such as metric values, labels, and other contextual information.
|
||||
keywords:
|
||||
- grafana
|
||||
@@ -33,24 +33,19 @@ refs:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/annotation-label/#annotations
|
||||
templating-labels-annotations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templating-labels-annotations/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templating-labels-annotations/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/templates/
|
||||
notification-message-reference:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/reference/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/reference/
|
||||
notification-messages:
|
||||
template-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/
|
||||
create-notification-templates:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/create-notification-templates/
|
||||
---
|
||||
|
||||
# Templates
|
||||
@@ -74,6 +69,10 @@ In Grafana, you have various options to template your alert notification message
|
||||
- Template notifications when you want to customize the appearance and information of your notifications.
|
||||
- Avoid using notification templates to add extra information to alert instances—use annotations instead.
|
||||
|
||||
{{< admonition type="tip" >}}
|
||||
For a practical example of templating, refer to our [Getting Started with Templating tutorial](https://grafana.com/tutorials/alerting-get-started-pt4/).
|
||||
{{< /admonition >}}
|
||||
|
||||
This diagram illustrates the entire templating process, from querying labels and templating the alert summary and notification to the final alert notification message.
|
||||
|
||||
{{< figure src="/media/docs/alerting/how-notification-templates-works.png" max-width="1200px" caption="How templating works" >}}
|
||||
@@ -103,7 +102,7 @@ Annotations can contain plain text, but you should template annotations if you n
|
||||
Here’s an example of templating an annotation, which explains where and why the alert was triggered. In this case, the alert triggers when CPU usage exceeds a threshold, and the `summary` annotation provides the relevant details.
|
||||
|
||||
```
|
||||
CPU usage for {{ index $labels "instance" }} has exceeded 80% ({{ index $values "A" }}) for the last 5 minutes.
|
||||
CPU usage for {{ $labels.instance }} has exceeded 80% ({{ $values.A.Value }}) for the last 5 minutes.
|
||||
```
|
||||
|
||||
The outcome of this template would be:
|
||||
@@ -120,11 +119,14 @@ For more details on how to template annotations, refer to [Template annotations
|
||||
|
||||
[Labels](ref:labels) are used to differentiate one alert instance from all other alert instances, as the set of labels uniquely identifies an alert instance. Notification policies and silences use labels to handle alert instances.
|
||||
|
||||
Template labels when you need to improve or change how alerts are uniquely identified. This is helpful if the labels you get from your query aren't detailed enough.
|
||||
You can also template labels based on query results. This is helpful if the labels you get from your query aren't detailed enough. For instance:
|
||||
|
||||
- Add a new label to change how alerts are identified and grouped into different alert groups.
|
||||
- Add a new label used by notification policies or silences to manage how the alert is handled.
|
||||
|
||||
Here’s an example of templating a `severity` label based on the query value:
|
||||
|
||||
```
|
||||
```go
|
||||
{{ if (gt $values.A.Value 90.0) -}}
|
||||
critical
|
||||
{{ else if (gt $values.A.Value 80.0) -}}
|
||||
@@ -136,26 +138,23 @@ low
|
||||
{{- end }}
|
||||
```
|
||||
|
||||
Avoid using query values in labels, as this may result in the creation of numerous alerts when only one is needed. Use annotation to inform about the query value instead.
|
||||
|
||||
For more details on how to template labels, refer to [Template annotations and labels](ref:templating-labels-annotations).
|
||||
|
||||
## Template notifications
|
||||
|
||||
[Notification templates](ref:notification-messages) allow you to customize the content of your notifications, such as the subject of an email or the body of a Slack message.
|
||||
[Notification templates](ref:template-notifications) allow you to customize the content of your notifications, such as the subject of an email or the body of a Slack message.
|
||||
|
||||
Notification templates differ from templating annotations and labels in the following ways:
|
||||
|
||||
- Notification templates are assigned to the **Contact point**, rather than the alert rule.
|
||||
- If not specified, the contact point uses a default template that includes relevant alert information.
|
||||
- You can create reusable notification templates and reference them in other templates.
|
||||
- The same template can be shared across multiple contact points, making it easier to maintain and ensuring consistency.
|
||||
- While both annotation/label templates and notification templates use the same templating language, the available variables and functions differ. For more details, refer to the [notification template reference](ref:notification-message-reference) and [annotation/label template reference](ref:templating-labels-annotations).
|
||||
- Notification templates should not be used to add additional information to individual alerts—use annotations for that purpose.
|
||||
- While both annotation/label templates and notification templates use the same templating language, the available variables and functions differ. For more details, refer to the [notification template reference](ref:notification-message-reference) and [annotation/label template reference](ref:templating-labels-annotations).
|
||||
|
||||
Here is an example of a notification template that summarizes all firing and resolved alerts in a notification group:
|
||||
|
||||
```
|
||||
```go
|
||||
{{ define "alerts.message" -}}
|
||||
{{ if .Alerts.Firing -}}
|
||||
{{ len .Alerts.Firing }} firing alert(s)
|
||||
@@ -184,4 +183,4 @@ The notification message to the contact point would look like this:
|
||||
- The web server web1 has been responding to 5% of HTTP requests with 5xx errors for the last 5 minutes.
|
||||
```
|
||||
|
||||
For instructions on creating and using notification templates, refer to [Create notification templates.](ref:create-notification-templates)
|
||||
For more details, refer to [Template notifications](ref:template-notifications).
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/manage-notifications/
|
||||
description: Detect and respond for day-to-day triage and analysis of what’s going on and action you need to take
|
||||
keywords:
|
||||
- grafana
|
||||
- detect
|
||||
- respond
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
menuTitle: Detect and respond
|
||||
title: Detect and respond
|
||||
weight: 130
|
||||
---
|
||||
|
||||
# Detect and respond
|
||||
|
||||
Use Grafana Alerting to track and generate alerts and send notifications, providing an efficient way for engineers to monitor, respond, and triage issues within their services.
|
||||
|
||||
Alerts and alert notifications provide a lot of value as key indicators to issues during the triage process, providing engineers with the information they need to understand what is going on in their system or service.
|
||||
@@ -1,56 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/alerting-rules/declare-incident-from-alert/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/declare-incident-from-alert/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/manage-notifications/declare-incident-from-alert/
|
||||
description: Declare an incident from a firing alert
|
||||
keywords:
|
||||
- grafana
|
||||
- alert rules
|
||||
- incident
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Declare incidents from firing alerts
|
||||
weight: 1010
|
||||
---
|
||||
|
||||
# Declare incidents from firing alerts
|
||||
|
||||
Declare an incident from a firing alert to streamline your alert to incident workflow.
|
||||
|
||||
## Before you begin
|
||||
|
||||
- Ensure you have Grafana Incident installed
|
||||
- You must have a firing alert
|
||||
|
||||
## Procedure
|
||||
|
||||
To declare an incident from a firing alert, complete the following steps.
|
||||
|
||||
1. Navigate to Alerts & Incidents -> Alerting -> Alert rules.
|
||||
2. From the Alert rules list view, click the firing alert that you want to declare an incident for.
|
||||
|
||||
**Note:**
|
||||
|
||||
You can also access **Declare Incident** from the alert details page.
|
||||
|
||||
3. Click **Declare Incident**.
|
||||
The **Declare Incident** pop-up opens in the Grafana Incident application.
|
||||
4. In the **Declare Incident** pop-up, enter what's going on.
|
||||
|
||||
**Note**: This field is pre-filled with the name of the alert rule, but you can edit it as required.
|
||||
|
||||
The alert rule is also linked to the incident.
|
||||
|
||||
5. Select a severity.
|
||||
6. Add labels, as required.
|
||||
7. Click **More options** to include a channel prefix and status.
|
||||
8. Click **Declare Incident**.
|
||||
|
||||
## Next steps
|
||||
|
||||
View and track the incident in the Grafana Incident application.
|
||||
|
||||
For more information, refer to [Grafana Incident documentation](/docs/grafana-cloud/incident/configure-settings/).
|
||||
@@ -1,67 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/unified-alerting/alerting-rules/rule-list/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/rule-list
|
||||
- ../../alerting/alerting-rules/view-alert-rules/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/view-alert-rules
|
||||
- ../../alerting/alerting-rules/rule-list/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/rule-list
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/manage-notifications/view-alert-rules/
|
||||
description: View alert rules
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- view
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: View alert rules
|
||||
weight: 410
|
||||
---
|
||||
|
||||
# View alert rules
|
||||
|
||||
The Alert rules list view page lists all existing alert rules. By default, alert rules are grouped by alert rule type: Grafana-managed (Grafana) or data source-managed (Mimir/Cortex/Loki). The Grafana section also contains alert rules for Prometheus-compatible data sources. You can view alert rules for Prometheus compatible data sources, but you cannot edit them.
|
||||
|
||||
When managing large volumes of alerts, you can use extended alert rule search capabilities to filter on folders, evaluation groups, and rules. Additionally, you can filter alert rules by their properties like labels, state, type, and health.
|
||||
|
||||
From the Alert rule list page, you can duplicate alert rules, silence notifications and pause or resume evaluation. If you pause evaluation, the alert rule state shows as **Paused**.
|
||||
|
||||
### Grouped view
|
||||
|
||||
Grouped view shows Grafana alert rules grouped by folder and Loki or Prometheus alert rules grouped by `namespace` + `group`. This is the default rule list view, intended for managing alert rules. You can expand each group to view a list of rules in this group.
|
||||
|
||||
### State view
|
||||
|
||||
State view shows alert rules grouped by state. Use this view to get an overview of which rules are in which state. You can expand each group to view more details.
|
||||
|
||||
## View alert rule details
|
||||
|
||||
To view alert rule details, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alert rules**.
|
||||
1. Click to expand an alert rule.
|
||||
1. In **Actions**, click **View** (the eye icon).
|
||||
|
||||
The namespace and group are shown in the breadcrumb navigation. They are interactive and can be used to filter rules by namespace or group.
|
||||
|
||||
The rest of the alert detail content is split up into tabs:
|
||||
|
||||
**Query and conditions**
|
||||
|
||||
View the details of the query that is used for the alert rule, including the expressions and intermediate values for each step of the expression pipeline. A graph view is included for range queries and data sources that return time series-like data frames.
|
||||
|
||||
**Instances**
|
||||
|
||||
Explore each alert instance, its status, labels and various other metadata for multi-dimensional alert rules.
|
||||
|
||||
Use **Search by label** to enter search criteria using label selectors. For example, `environment=production,region=~US|EU,severity!=warning`.
|
||||
|
||||
**History**
|
||||
|
||||
Explore the recorded history for an alert rule. You can also filter by alert state.
|
||||
|
||||
**Details**
|
||||
|
||||
Debug or audit using the alert rule metadata and view the alert rule annotations.
|
||||
42
docs/sources/alerting/monitor-status/_index.md
Normal file
42
docs/sources/alerting/monitor-status/_index.md
Normal file
@@ -0,0 +1,42 @@
|
||||
---
|
||||
aliases:
|
||||
- ./manage-notifications/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/
|
||||
description: Detect and respond for day-to-day triage and analysis of what’s going on and action you need to take
|
||||
keywords:
|
||||
- grafana
|
||||
- detect
|
||||
- respond
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Monitor alerts
|
||||
weight: 130
|
||||
refs:
|
||||
configure-alert-rules:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/
|
||||
configure-notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/
|
||||
---
|
||||
|
||||
# Monitor alerts
|
||||
|
||||
Alerts and alert notifications should provide key information to help alert responders and incident participants understand what happened in their system and how to respond.
|
||||
|
||||
Grafana Alerting offers the ability to monitor your alerts and manage your alerting setup. You can get an overview of your alerts, track the history of alert states, and monitor notification statuses. These can help you start investigating alert issues within Grafana and improve the reliability of your alerting implementation.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-history-page.png" max-width="750px" alt="History page in Grafana Alerting" >}}
|
||||
|
||||
The previous sections explain how to [configure alert rules](ref:configure-alert-rules) and [configure notifications](ref:configure-notifications) to generate alerts and send their notifications.
|
||||
|
||||
This section focuses on finding and understanding the state of your alert rules, alert instances, and their notifications. For more details, refer to:
|
||||
|
||||
{{< section >}}
|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/alerting-rules/declare-incident-from-alert/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/declare-incident-from-alert/
|
||||
- ../../alerting/manage-notifications/declare-incident-from-alert/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/declare-incident-from-alert/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/declare-incident-from-alert/
|
||||
description: Declare an incident from a firing alert
|
||||
keywords:
|
||||
- grafana
|
||||
- alert rules
|
||||
- incident
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
title: Declare incidents from firing alerts
|
||||
weight: 1010
|
||||
---
|
||||
|
||||
# Declare incidents from firing alerts
|
||||
|
||||
Declare an incident from a firing alert to streamline your alert to incident workflow.
|
||||
|
||||
## Before you begin
|
||||
|
||||
- Ensure you have [Grafana Incident](/docs/grafana-cloud/incident/) installed.
|
||||
- You must have a firing alert.
|
||||
|
||||
## Procedure
|
||||
|
||||
To declare an incident from a firing alert, complete the following steps.
|
||||
|
||||
1. Navigate to **Alerts & IRM** -> **Alerting** -> **Alert rules**.
|
||||
1. From the Alert rules page, find the firing alert that you want to declare an incident for.
|
||||
1. Click **More** -> **Declare Incident**.
|
||||
|
||||
Alternatively, you can declare an incident from the Alert details page.
|
||||
|
||||
The **Declare Incident** pop-up opens in the Grafana Incident application.
|
||||
|
||||
1. In the **Declare Incident** pop-up, enter the **What's going on?** field.
|
||||
|
||||
This field is pre-filled with the name of the alert rule, but you can edit it as required.
|
||||
|
||||
The alert rule is also linked to the incident.
|
||||
|
||||
1. Select a severity.
|
||||
1. Add labels, as required.
|
||||
1. Click **More options** to include a channel prefix and status.
|
||||
1. Click **Automated options** to enable automated actions configured by admins.
|
||||
1. Click **Declare Incident**.
|
||||
|
||||
## Next steps
|
||||
|
||||
View and manage the incident in the **Grafana Incident** application.
|
||||
|
||||
{{< figure src="/media/docs/alerting/incident-response-management-timeline-ui.png" max-width="750px" alt="Incident timeline view in Grafana Incident" >}}
|
||||
|
||||
For more information, refer to the [Grafana Incident documentation](/docs/grafana-cloud/incident/configure-settings/).
|
||||
@@ -5,8 +5,9 @@ aliases:
|
||||
- ../../alerting/alert-groups/view-alert-grouping/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alert-groups/view-alert-grouping/
|
||||
- ../../alerting/unified-alerting/alert-groups/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alert-groups/
|
||||
- ../../alerting/manage-notifications/view-notification-errors/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-notification-errors/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/manage-notifications/view-alert-groups/
|
||||
description: The Groups view lists grouped alerts that are actively triggering notifications.
|
||||
- ../../alerting/manage-notifications/view-alert-groups/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-alert-groups/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/view-active-notifications/
|
||||
description: The Active notifications view lists grouped alerts that are actively triggering notifications.
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
@@ -19,7 +20,7 @@ labels:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: View the status of notifications
|
||||
title: View active notifications
|
||||
weight: 800
|
||||
refs:
|
||||
alertmanager:
|
||||
@@ -34,20 +35,27 @@ refs:
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/group-alert-notifications/
|
||||
---
|
||||
|
||||
# View the status of notifications
|
||||
# View active notifications
|
||||
|
||||
The Groups view page lists grouped alerts that are actively triggering notifications.
|
||||
The Active notifications page lists groups of alerts (or alert instances) that are actively triggering notifications.
|
||||
|
||||
By default, Grafana Alerting groups similar firing alerts (or alert instances) to prevent notification overload. For details on how notification grouping works, refer to [Group alert notifications](ref:grouping).
|
||||
By default, Grafana Alerting [groups similar alerts into a single notification](ref:grouping).
|
||||
|
||||
In the Groups view, you can see alert groups, check the state of their notifications, and also filter for alert instances that match specific criteria. This view is useful for debugging and verifying your grouping settings of notification policies.
|
||||
In this view, you can:
|
||||
|
||||
- Find alert groups and the state of their notifications.
|
||||
- Filter for alert instances that match specific criteria.
|
||||
|
||||
The Active notifications view is useful for debugging and verifying how notifications are grouped based on your notification policy settings.
|
||||
|
||||
## View alert groups and notification state
|
||||
|
||||
To view alert groups, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting**.
|
||||
1. Click **Groups** to view the list of groups firing notifications.
|
||||
1. Click **Active notifications** to view the list of groups firing notifications.
|
||||
|
||||
{{< figure src="/media/docs/alerting/active-notifications-view2.png" max-width="750px" alt="Active notifications view in Grafana Alerting" >}}
|
||||
|
||||
By default, alert groups are grouped by the notification policies grouping.
|
||||
|
||||
58
docs/sources/alerting/monitor-status/view-alert-rules.md
Normal file
58
docs/sources/alerting/monitor-status/view-alert-rules.md
Normal file
@@ -0,0 +1,58 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/unified-alerting/alerting-rules/rule-list/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/alerting-rules/rule-list
|
||||
- ../../alerting/alerting-rules/view-alert-rules/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/view-alert-rules
|
||||
- ../../alerting/alerting-rules/rule-list/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/rule-list
|
||||
- ../../alerting/manage-notifications/view-alert-rules/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-alert-rules/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/view-alert-rules/
|
||||
description: View alert rules
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- guide
|
||||
- rules
|
||||
- view
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: View alert rules
|
||||
weight: 410
|
||||
refs:
|
||||
view-alert-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/monitor-status/view-alert-state/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/monitor-status/view-alert-state/
|
||||
---
|
||||
|
||||
# View alert rules
|
||||
|
||||
The Alert rules list view page lists all existing recording and alert rules, including those created in Grafana and those available in configured data sources.
|
||||
|
||||
To access the Alert rules page, click **Alerts & IRM** -> **Alerting** -> **Alert rules**.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-rules-page.png" max-width="750px" alt="Alert rule view page in Grafana Alerting" >}}
|
||||
|
||||
By default, alert rules are grouped by alert rule type: Grafana-managed or data source-managed.
|
||||
|
||||
In this view, you can find and edit rules created in Grafana. However, rules created in Prometheus-compatible data sources are displayed but cannot be edited.
|
||||
|
||||
This view includes filters to simplify managing large volumes of alerts.
|
||||
|
||||
You can filter by data sources, dashboards, and alert rule properties such as state, type, health, and contact points. The **Search** input allows you to filter by additional parameters like folders, evaluation groups, labels, and more.
|
||||
|
||||
You can also change how the rule list is displayed using the **View as** option.
|
||||
|
||||
- **Grouped**: Displays Grafana rules grouped by folder and evaluation group, and data-source rules by namespace and evaluation group. This is the default view.
|
||||
|
||||
- **List**: Displays Grafana rules grouped only by folder.
|
||||
|
||||
- **State**: Displays rules grouped by state, providing an overview for each state.
|
||||
|
||||
Select a group to expand it and view the list of alert rules within that group.
|
||||
|
||||
{{< figure src="/media/docs/alerting/view-alert-rule-list-with-actions.png" max-width="750px" alt="View alert rule state and alert rule health in Grafana Alerting" >}}
|
||||
|
||||
For details on how rule states and alert instance states are displayed, refer to [View alert state](ref:view-alert-state).
|
||||
@@ -1,60 +1,29 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/alerting-rules/view-state-health/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/view-state-health
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/manage-notifications/view-state-health/
|
||||
description: View the state and health of alert rules
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/view-alert-state-history/
|
||||
description: View alert rules
|
||||
keywords:
|
||||
- grafana
|
||||
- alert rules
|
||||
- keep last state
|
||||
- alerting
|
||||
- guide
|
||||
- state
|
||||
- health
|
||||
- rules
|
||||
- view
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: View alert state and history
|
||||
weight: 420
|
||||
refs:
|
||||
alert-rule-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-rule-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-rule-state
|
||||
alert-rule-evaluation:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/
|
||||
alert-instance-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
title: View alert state history
|
||||
weight: 440
|
||||
---
|
||||
|
||||
# View alert state and history
|
||||
|
||||
An alert rule and its corresponding alert instances can transition through distinct states during their [evaluation](ref:alert-rule-evaluation). There are three key components that helps us understand the behavior of our alerts:
|
||||
|
||||
- [Alert Instance State](ref:alert-instance-state): Refers to the state of the individual alert instances.
|
||||
- [Alert Rule State](ref:alert-rule-state): Determined by the "worst state" among its alert instances.
|
||||
- [Alert Rule Health](ref:alert-rule-health): Indicates the status in cases of `Error` or `NoData` events.
|
||||
|
||||
To view the state and health of your alert rules:
|
||||
|
||||
1. In the left-side menu, click **Alerts & IRM** and then **Alerting**.
|
||||
1. Click **Alert rules** to view the list of existing alerts.
|
||||
1. Click an alert rule to view its state, health, and state history.
|
||||
|
||||
## View state history using the History page
|
||||
# View alert state history
|
||||
|
||||
View a history of all alert events generated by your Grafana-managed alert rules from one centralized page. This helps you see patterns in your alerts over time, observe trends, make predictions, and even debug alerts that might be firing too often.
|
||||
|
||||
An alert event is displayed each time an alert instance changes its state over a period of time. All alert events are displayed regardless of whether silences or mute timings are set, so you’ll see a complete set of your data history even if you’re not necessarily being notified.
|
||||
|
||||
## View from the History page
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
For Grafana Enterprise and OSS users:
|
||||
The feature is available starting with Grafana 11.2.
|
||||
@@ -65,22 +34,26 @@ Users can only see the history and transitions of alert rules they have access t
|
||||
|
||||
To access the History view, complete the following steps.
|
||||
|
||||
1. Navigate to **Alerts & IRM** -> **Alerting** -> History.
|
||||
1. Navigate to **Alerts & IRM** -> **Alerting** -> **History**.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-history-page.png" max-width="750px" alt="History page in Grafana Alerting" >}}
|
||||
|
||||
An Alert Events chart shows your alert activity within a specific time frame. Each alert event represents an alert instance that changed its state at a particular point in time. The history of the data is displayed over a period of time.
|
||||
|
||||
1. Filter by labels either by entering a label or clicking a label of an instance in the list of events.
|
||||
1. Filter by current state and previous state by selecting a state from the drop-down or by clicking the states from the list of events.
|
||||
2. Filter by labels either by entering a label or clicking a label of an instance in the list of events.
|
||||
3. Filter by current state and previous state by selecting a state from the drop-down or by clicking the states from the list of events.
|
||||
Zoom in by dragging on the chart or use the time picker.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
If you exceed the 5000 alerts limit, you may see data missing from the chart. To see complete results, narrow the time frame.
|
||||
{{% /admonition %}}
|
||||
|
||||
1. Under the chart, there is a list of events. Each event represents a state change on an alert instance. Expand a row to see the number of transitions for the alert instance, a state graph, and the value in the transition.
|
||||
1. Click the alert rule name to jump to the History tab in the Alert Rule view.
|
||||
4. Under the chart, there is a list of events. Each event represents a state change on an alert instance. Expand a row to see the number of transitions for the alert instance, a state graph, and the value in the transition.
|
||||
5. Click the alert rule name to jump to the History tab in the Alert Rule view.
|
||||
|
||||
## View state history using the State history view
|
||||
{{< figure src="/media/docs/alerting/alerting-alert-history-tab.png" max-width="750px" alt="Alert History tab in Grafana Alerting" >}}
|
||||
|
||||
## View from the State history view
|
||||
|
||||
Use the State history view to get insight into how your individual alert instances behave over time.
|
||||
|
||||
@@ -96,7 +69,7 @@ To access the State history view, complete the following steps.
|
||||
1. Click an alert rule.
|
||||
1. Select **Show state history**.
|
||||
|
||||
The State history view opens.
|
||||
{{< figure src="/media/docs/alerting/alerting-state-history-view.png" max-width="750px" alt="State history view in Grafana Alerting" >}}
|
||||
|
||||
The timeline view at the top displays a timeline of changes for the past hour, so you can track how your alert instances are behaving over time.
|
||||
|
||||
138
docs/sources/alerting/monitor-status/view-alert-state.md
Normal file
138
docs/sources/alerting/monitor-status/view-alert-state.md
Normal file
@@ -0,0 +1,138 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../alerting/alerting-rules/view-state-health/ # /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/view-state-health
|
||||
- ../../alerting/manage-notifications/view-state-health/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-state-health/
|
||||
- ../../alerting/manage-notifications/view-alert-state-on-panels/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-alert-state-on-panels/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor-status/view-alert-state/
|
||||
description: View the state and health of alert rules
|
||||
keywords:
|
||||
- grafana
|
||||
- alert rules
|
||||
- keep last state
|
||||
- guide
|
||||
- state
|
||||
- health
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: View alert state
|
||||
weight: 420
|
||||
refs:
|
||||
time-series-visualizations:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/time-series/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series/
|
||||
alert-list-panel:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/visualizations/alert-list/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/alert-list/
|
||||
link-alert-rules-to-panels:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/link-alert-rules-to-panels/
|
||||
alert-rule-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-rule-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-rule-state
|
||||
alert-instance-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
alert-instance-state:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/#alert-instance-state
|
||||
alert-rule-health:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/state-and-health/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/state-and-health/
|
||||
---
|
||||
|
||||
# View alert state
|
||||
|
||||
An alert rule and its corresponding alert instances can transition through distinct states during the alert rule evaluation.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-instance-states-v3.png" alt="A diagram of the distinct alert instance states and transitions." max-width="750px" >}}
|
||||
|
||||
There are three key components that helps us understand the behavior of our alerts:
|
||||
|
||||
- [Alert Instance State](ref:alert-instance-state): Refers to the state of the individual alert instances.
|
||||
- [Alert Rule State](ref:alert-rule-state): Determined by the "worst state" among its alert instances.
|
||||
- [Alert Rule Health](ref:alert-rule-health): Indicates the status in cases of `Error` or `NoData` events.
|
||||
|
||||
## View alert rule and instance states
|
||||
|
||||
To view the state and health of your alert rules and the status of alert instances:
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting**.
|
||||
1. Click **Alert rules** to view the list of existing alert rules.
|
||||
|
||||
{{< figure src="/media/docs/alerting/view-alert-rule-list-with-actions.png" max-width="750px" alt="View alert rule state and alert rule health in Grafana Alerting" >}}
|
||||
|
||||
Each alert rule shows its state, health, summary, next evaluation time, and available actions such as **Pause evaluation**, **Silence notifications**, **Export**, **Delete**, and more.
|
||||
|
||||
1. Click on an alert rule to view additional details and its resulting alert instances.
|
||||
|
||||
{{< figure src="/media/docs/alerting/view-alert-instance-state.png" max-width="750px" alt="View alert rule state and alert rule health in Grafana Alerting" >}}
|
||||
|
||||
### View from the alert rule details page
|
||||
|
||||
To view more alert rule details, complete the following steps.
|
||||
|
||||
1. Click **Alerts & IRM** -> **Alerting** -> **Alert rules**.
|
||||
1. Click to expand an alert rule.
|
||||
1. In **Actions**, click **View** (the eye icon).
|
||||
|
||||
{{< figure src="/media/docs/alerting/alert-rule-view-page-with-breadcrumb.png" max-width="750px" alt="Alert rule view page in Grafana Alerting" >}}
|
||||
|
||||
The namespace and group are shown in the breadcrumb navigation. They are interactive and can be used to filter rules by namespace or group.
|
||||
|
||||
The rest of the alert detail content is split up into tabs:
|
||||
|
||||
**Query and conditions**
|
||||
|
||||
View the details of the query that is used for the alert rule, including the expressions and intermediate values for each step of the expression pipeline. A graph view is included for range queries and data sources that return time series-like data frames.
|
||||
|
||||
**Instances**
|
||||
|
||||
Explore each alert instance, its status, labels and various other metadata for multi-dimensional alert rules.
|
||||
|
||||
Use **Search by label** to enter search criteria using label selectors. For example, `environment=production,region=~US|EU,severity!=warning`.
|
||||
|
||||
**History**
|
||||
|
||||
Explore the recorded history for an alert rule. You can also filter by alert state.
|
||||
|
||||
**Details**
|
||||
|
||||
Debug or audit using the alert rule metadata and view the alert rule annotations.
|
||||
|
||||
## View alert state on panels
|
||||
|
||||
When an [alert rule is linked to a time series panel](ref:link-alert-rules-to-panels), the time series panel displays the alert state and alert events.
|
||||
|
||||
A heart icon near the panel title shows the current alert state:
|
||||
|
||||
- A broken red heart when the alert is in `Alerting` state.
|
||||
- A green heart when the alert is in `Normal` state.
|
||||
|
||||
Colored annotations indicate changes in alert state, such as pending, alerting, and resolving.
|
||||
|
||||
{{< figure src="/media/docs/alerting/panel-displays-alert-state.png" max-width="1200px" alt="A panel with a firing alert and annotations that display the pending and alerting state changes." >}}
|
||||
|
||||
Additionally, Grafana provides an [alert list panel](ref:alert-list-panel) that you can add to a dashboard to display a list of alerts and their states.
|
||||
|
||||
{{< figure src="/static/img/docs/alert-list-panel/alert-list-panel.png" max-width="850px" alt="Two alert list panels displaying distinct lists of alerts." >}}
|
||||
|
||||
You can configure the alert list panel with various visualization options and filters to control how alerts are displayed. For more details, refer to the [Alert list documentation](ref:alert-list-panel).
|
||||
|
||||
{{< docs/play title="this demo dashboard with alert list panels and linked alert rules" url="https://play.grafana.org/d/000000074/" >}}
|
||||
@@ -2,90 +2,21 @@
|
||||
aliases:
|
||||
- unified-alerting/set-up/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/set-up/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/set-up/
|
||||
description: Set up or upgrade your implementation of Grafana Alerting
|
||||
description: Additional configuration for Grafana Alerting
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
menuTitle: Set up
|
||||
title: Set up Alerting
|
||||
weight: 110
|
||||
refs:
|
||||
terraform-provisioning:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/provision-alerting-resources/terraform-provisioning/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/provision-alerting-resources/terraform-provisioning/
|
||||
configure-high-availability:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-high-availability/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-high-availability/
|
||||
configure-alertmanager:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/set-up/configure-alertmanager/
|
||||
data-source-management:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/administration/data-source-management/
|
||||
data-source-alerting:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/#supported-data-sources
|
||||
file-provisioning:
|
||||
- pattern: /docs/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/provision-alerting-resources/file-provisioning/
|
||||
menuTitle: Additional configuration
|
||||
title: Additional configuration
|
||||
weight: 160
|
||||
---
|
||||
|
||||
# Set up Alerting
|
||||
# Additional configuration
|
||||
|
||||
Set up or upgrade your implementation of Grafana Alerting.
|
||||
|
||||
**Note:**
|
||||
|
||||
These are set-up instructions for Grafana Alerting Open Source.
|
||||
|
||||
## Before you begin
|
||||
|
||||
- Configure your [data sources](ref:data-source-management)
|
||||
- Check which data sources are compatible with and supported by [Grafana Alerting](ref:data-source-alerting)
|
||||
|
||||
Watch this short video to get started, or follow the [written tutorial](http://grafana.com/tutorials/alerting-get-started/).
|
||||
|
||||
{{< youtube id="6W8Nu4b_PXM" >}}
|
||||
|
||||
## Set up Alerting
|
||||
|
||||
To set up Alerting, you need to:
|
||||
|
||||
1. Configure alert rules
|
||||
|
||||
- Create Grafana-managed or Mimir/Loki-managed alert rules and recording rules
|
||||
|
||||
1. Configure contact points
|
||||
|
||||
- Check the default contact point and update the email address
|
||||
|
||||
- Optional: Add new contact points and integrations
|
||||
|
||||
1. Configure notification policies
|
||||
|
||||
- Check the default notification policy
|
||||
|
||||
- Optional: Add additional nested policies
|
||||
|
||||
- Optional: Add labels and label matchers to control alert routing
|
||||
|
||||
1. Optional: Integrate with [Grafana OnCall](/docs/oncall/latest/integrations/grafana-alerting)
|
||||
|
||||
## Advanced set up options
|
||||
|
||||
Grafana Alerting supports many additional configuration options, from configuring external Alertmanagers to routing Grafana-managed alerts outside of Grafana, to defining your alerting setup as code.
|
||||
Grafana Alerting offers a variety of advanced configuration options to further tailor your alerting setup. These optional features include configuring up permissions and role-based access control, adding external Alertmanagers, or defining your alerting setup as code. While not essential for basic alerting, these options can enhance security, scalability, and automation in complex environments.
|
||||
|
||||
The following topics provide you with advanced configuration options for Grafana Alerting.
|
||||
|
||||
- [Provision alert rules using file provisioning](ref:file-provisioning)
|
||||
- [Provision alert rules using Terraform](ref:terraform-provisioning)
|
||||
- [Add an external Alertmanager](ref:configure-alertmanager)
|
||||
- [Configure high availability](ref:configure-high-availability)
|
||||
{{< section >}}
|
||||
|
||||
108
docs/sources/alerting/set-up/configure-alertmanager/_index.md
Normal file
108
docs/sources/alerting/set-up/configure-alertmanager/_index.md
Normal file
@@ -0,0 +1,108 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../configure-alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/configure-alertmanager/
|
||||
- ../unified-alerting/fundamentals/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/fundamentals/alertmanager/
|
||||
- ../manage-notifications/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/alertmanager/
|
||||
- ../fundamentals/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alertmanager/
|
||||
- ../fundamentals/notifications/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/alertmanager
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/set-up/configure-alertmanager/
|
||||
description: Learn about Alertmanagers and set up Alerting to use an external Alertmanager
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- set up
|
||||
- configure
|
||||
- external Alertmanager
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure Alertmanagers
|
||||
weight: 200
|
||||
refs:
|
||||
alertmanager-data-source:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/datasources/alertmanager/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/alertmanager/
|
||||
notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/
|
||||
---
|
||||
|
||||
# Configure Alertmanagers
|
||||
|
||||
Grafana Alerting is based on the architecture of the Prometheus alerting system. Grafana sends firing and resolved alerts to an Alertmanager, which is responsible for [handling notifications](ref:notifications). This architecture decouples alert rule evaluation from notification handling, improving scalability.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-alertmanager-architecture.png" max-width="750px" alt="A diagram with the alert generator and alert manager architecture" >}}
|
||||
|
||||
Grafana can enable one or more Alertmanagers to receive Grafana-managed alerts for notification handling. It’s important to note that each Alertmanager manages its own independent alerting resources, such as:
|
||||
|
||||
- Contact points and notification templates
|
||||
- Notification policies and mute timings
|
||||
- Silences
|
||||
- Active notifications
|
||||
|
||||
Use the `Choose Alertmanager` dropdown on these pages to switch between Alertmanagers and view or manage their resources.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-choose-alertmanager.png" max-width="750px" alt="A screenshot choosing an Alertmanager in the notification policies UI" >}}
|
||||
|
||||
## Types of Alertmanagers in Grafana
|
||||
|
||||
Grafana can be configured to handle alert notifications using various Alertmanagers.
|
||||
|
||||
- **Grafana Alertmanager**: Grafana includes a built-in Alertmanager that extends the [Prometheus Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/). This is the default Alertmanager and is referred to as "Grafana" in the user interface. It can only handle Grafana-managed alerts.
|
||||
|
||||
- **Cloud Alertmanager**: Each Grafana Cloud instance comes preconfigured with an additional Alertmanager (`grafanacloud-STACK_NAME-ngalertmanager`) from the Mimir (Prometheus) instance running in the Grafana Cloud Stack.
|
||||
|
||||
The Cloud Alertmanager is available exclusively in Grafana Cloud and can handle both Grafana-managed and data source-managed alerts.
|
||||
|
||||
Some Grafana Cloud services, such as **Kubernetes Monitoring** and **Synthetic Monitoring** use the Cloud Alertmanager to create and manage alerts.
|
||||
|
||||
- **Other Alertmanagers**: Grafana Alerting also supports sending alerts to other Alertmanagers, such as the [Prometheus Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/), which can handle both Grafana-managed and data source-managed alerts.
|
||||
|
||||
Grafana Alerting supports using a combination of Alertmanagers and can [enable other Alertmanagers to receive Grafana-managed alerts](#enable-an-alertmanager-to-receive-grafana-managed-alerts). The decision often depends on your alerting setup and where your alerts are generated.
|
||||
|
||||
For example, if you already have an Alertmanager running in your on-premises or cloud infrastructure to handle Prometheus alerts, you can forward Grafana-managed alerts to the same Alertmanager for unified notification handling.
|
||||
|
||||
## Add an Alertmanager
|
||||
|
||||
Alertmanagers should be configured as data sources using Grafana Configuration from the main Grafana navigation menu. To add an Alertmanager, complete the following steps.
|
||||
|
||||
{{< docs/shared lookup="alerts/add-alertmanager-ds.md" source="grafana" version="<GRAFANA_VERSION>" >}}
|
||||
|
||||
For provisioning instructions, refer to the [Alertmanager data source documentation](ref:alertmanager-data-source).
|
||||
|
||||
After adding an Alertmanager, you can use the Grafana Alerting UI to manage notification policies, contact points, silences, and other alerting resources from within Grafana.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
When using Prometheus, you can manage silences in the Grafana Alerting UI. However, other Alertmanager resources such as contact points, notification policies, and templates are read-only because the Prometheus Alertmanager HTTP API does not support updates for these resources.
|
||||
{{% /admonition %}}
|
||||
|
||||
When using multiple Alertmanagers, use the `Choose Alertmanager` dropdown to switch between Alertmanagers.
|
||||
|
||||
## Enable an Alertmanager to receive Grafana-managed alerts
|
||||
|
||||
After enabling **Receive Grafana Alerts** in the Data Source Settings, you must also configure the Alertmanager in the Alerting Settings page. Grafana supports enabling one or multiple Alertmanagers to receive all generated Grafana-managed alerts.
|
||||
|
||||
1. In the left-side menu, click **Alerts & IRM** and then **Alerting**.
|
||||
1. Click **Settings** to view the list of configured Alertmanagers.
|
||||
1. For the selected Alertmanager, click the **Enable/Disable** button to toggle receiving Grafana-managed alerts. When activated, the Alertmanager displays `Receiving Grafana-managed alerts`.
|
||||
|
||||
{{< figure src="/media/docs/alerting/grafana-alerting-settings.png" max-width="750px" alt="Grafana Alerting Settings page" >}}
|
||||
|
||||
All Grafana-managed alerts are forwarded to Alertmanagers marked as `Receiving Grafana-managed alerts`.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Grafana Alerting does not support forwarding Grafana-managed alerts to the AlertManager in Amazon Managed Service for Prometheus. For more details, refer to [this GitHub issue](https://github.com/grafana/grafana/issues/64064).
|
||||
{{% /admonition %}}
|
||||
|
||||
## Manage Alertmanager configurations
|
||||
|
||||
On the Settings page, you can also manage your Alertmanager configurations.
|
||||
|
||||
- Manage version snapshots for the built-in Alertmanager, which allows administrators to roll back unintentional changes or mistakes in the Alertmanager configuration.
|
||||
- Compare the historical snapshot with the latest configuration to see which changes were made.
|
||||
@@ -1,102 +0,0 @@
|
||||
---
|
||||
aliases:
|
||||
- ../../configure-alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/configure-alertmanager/
|
||||
- ../unified-alerting/fundamentals/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/unified-alerting/fundamentals/alertmanager/
|
||||
- ../manage-notifications/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/alertmanager/
|
||||
- ../fundamentals/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alertmanager/
|
||||
- ../fundamentals/notifications/alertmanager/ # /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/alertmanager
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/set-up/configure-alertmanager/
|
||||
description: Learn about Alertmanagers and set up Alerting to use an external Alertmanager
|
||||
keywords:
|
||||
- grafana
|
||||
- alerting
|
||||
- set up
|
||||
- configure
|
||||
- external Alertmanager
|
||||
labels:
|
||||
products:
|
||||
- cloud
|
||||
- enterprise
|
||||
- oss
|
||||
title: Configure Alertmanagers
|
||||
weight: 200
|
||||
refs:
|
||||
notifications:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/notifications/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/notifications/
|
||||
---
|
||||
|
||||
# Configure Alertmanagers
|
||||
|
||||
Grafana Alerting is based on the architecture of the Prometheus alerting system. Grafana sends firing and resolved alerts to an Alertmanager, which is responsible for [handling notifications](ref:notifications).
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-alertmanager-architecture.png" max-width="750px" alt="A diagram with the alert generator and alert manager architecture" >}}
|
||||
|
||||
This architecture decouples alert rule evaluation from notification handling, allowing alerts to be forwarded to other Alertmanagers.
|
||||
|
||||
Grafana can use different Alertmanagers. It’s important to note that each Alertmanager manages its own independent alerting resources, such as:
|
||||
|
||||
- Contact points and notification templates
|
||||
- Notification policies and mute timings
|
||||
- Silences
|
||||
- Active notifications
|
||||
|
||||
Use the `Choose Alertmanager` on these pages to switch between Alertmanagers.
|
||||
|
||||
{{< figure src="/media/docs/alerting/alerting-choose-alertmanager.png" max-width="750px" alt="A screenshot choosing an Alertmanager in the notification policies UI" >}}
|
||||
|
||||
## Types of Alertmanagers in Grafana
|
||||
|
||||
Grafana can be configured to manage both Grafana-managed and data source-managed alerts using various Alertmanagers, depending on your infrastructure and alerting requirements.
|
||||
|
||||
- **Grafana Alertmanager**: Grafana has its own built-in Alertmanager, referred to as "Grafana" in the user interface. It is the default Alertmanager and can only handle Grafana-managed alerts.
|
||||
|
||||
- **Cloud Alertmanager**: Each Grafana Cloud instance comes preconfigured with an additional Alertmanager (`grafanacloud-STACK_NAME-ngalertmanager`) from the Mimir (Prometheus) instance running in the Grafana Cloud Stack.
|
||||
|
||||
The Cloud Alertmanager is available exclusively in Grafana Cloud and can handle both Grafana-managed and data source-managed alerts.
|
||||
|
||||
Some Grafana Cloud services, such as **Kubernetes Monitoring** and **Synthetic Monitoring** use the Cloud Alertmanager to create and manage alerts.
|
||||
|
||||
- **Other Alertmanagers**: Grafana Alerting also supports sending alerts to other Alertmanagers, such as the [Prometheus Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/), which can handle Grafana-managed alerts and data sources-managed alerts such as alerts from Loki, Mimir, and Prometheus.
|
||||
|
||||
You can use a combination of Alertmanagers. The decision often depends on your alerting setup and where your alerts are being generated. Here are two examples of when you may want to add an Alertmanager and send your alerts there instead of using the built-in Grafana Alertmanager.
|
||||
|
||||
1. You may already have Alertmanagers on-premises in your own Cloud infrastructure that you still want to use because you have other alert generators, such as Prometheus.
|
||||
|
||||
2. You want to use both Prometheus on-premises and hosted Grafana to send alerts to the same Alertmanager that runs in your Cloud infrastructure.
|
||||
|
||||
## Add an Alertmanager
|
||||
|
||||
From Grafana, you can configure and administer your own Alertmanager to receive Grafana alerts.
|
||||
|
||||
After adding an Alertmanager, you can use the Grafana Alerting UI to manage notification policies, contact points, and other alerting resources from within Grafana, with support for HTTP basic authentication credentials.
|
||||
|
||||
Alertmanagers should be configured as data sources using Grafana Configuration from the main Grafana navigation menu. To add an Alertmanager, complete the following steps.
|
||||
|
||||
1. Click **Connections** in the left-side menu.
|
||||
1. On the Connections page, search for `Alertmanager`.
|
||||
1. Click the **Create a new data source** button.
|
||||
|
||||
If you don't see this button, you may need to install the plugin, relaunch your Cloud instance, and then repeat steps 1 and 2.
|
||||
|
||||
1. Fill out the fields on the page, as required.
|
||||
|
||||
If you are provisioning your data source, set the flag `handleGrafanaManagedAlerts` in the `jsonData` field to `true` to send Grafana-managed alerts to this Alertmanager.
|
||||
|
||||
**Note:** Prometheus, Grafana Mimir, and Cortex implementations of Alertmanager are supported. For Prometheus, contact points and notification policies are read-only in the Grafana Alerting UI.
|
||||
|
||||
1. Click **Save & test**.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
Grafana Alerting does not support sending alerts to the AWS Managed Service for Prometheus due to the lack of sigv4 support in Prometheus.
|
||||
{{% /admonition %}}
|
||||
|
||||
## Manage Alertmanager configurations
|
||||
|
||||
On the Settings page, you can manage your Alertmanager configurations and configure where Grafana-managed alert instances are forwarded.
|
||||
|
||||
- Manage which Alertmanagers receive alert instances from Grafana-managed rules without navigating and editing data sources.
|
||||
- Manage version snapshots for the built-in Alertmanager, which allows administrators to roll back unintentional changes or mistakes in the Alertmanager configuration.
|
||||
- Compare the historical snapshot with the latest configuration to see which changes were made.
|
||||
@@ -21,9 +21,9 @@ weight: 600
|
||||
refs:
|
||||
state-history:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/manage-notifications/view-state-health/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/monitor-status/view-alert-state-history/
|
||||
- pattern: /docs/grafana-cloud/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/manage-notifications/view-state-health/
|
||||
destination: /docs/grafana-cloud/alerting-and-irm/alerting/monitor-status/view-alert-state-history/
|
||||
meta-monitoring:
|
||||
- pattern: /docs/grafana/
|
||||
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/monitor/
|
||||
|
||||
@@ -52,4 +52,32 @@ Grafana Alerting has the following permissions.
|
||||
| `alert.provisioning:write` | n/a | Update all Grafana alert rules, notification policies, etc via provisioning API. Permissions to folders and data source are not required. |
|
||||
| `alert.provisioning.provenance:write` | n/a | Set provisioning status for alerting resources. Cannot be used alone. Requires user to have permissions to access resources |
|
||||
|
||||
Contact point permissions. To enable these permissions, enable the `alertingApiServer` feature toggle.
|
||||
|
||||
| Action | Applicable scope | Description |
|
||||
| -------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------- |
|
||||
| `alert.notifications.receivers:read` | `receivers:*`<br>`receivers:uid:*` | Read contact points. |
|
||||
| `alert.notifications.receivers.secrets:read` | `receivers:*`<br>`receivers:uid:*` | Export contact points with decrypted secrets. |
|
||||
| `alert.notifications.receivers:create` | n/a | Create a new contact points. The creator is automatically granted full access to the created contact point. |
|
||||
| `alert.notifications.receivers:write` | `receivers:*`<br>`receivers:uid:*` | Update existing contact points. |
|
||||
| `alert.notifications.receivers:delete` | `receivers:*`<br>`receivers:uid:*` | Update and delete existing contact points. |
|
||||
| `receivers.permissions:read` | `receivers:*`<br>`receivers:uid:*` | Read permissions for contact points. |
|
||||
| `receivers.permissions:write` | `receivers:*`<br>`receivers:uid:*` | Manage permissions for contact points. |
|
||||
|
||||
Mute time interval permissions. To enable these permissions, enable the `alertingApiServer` feature toggle.
|
||||
|
||||
| Action | Applicable scope | Description |
|
||||
| ------------------------------------------- | ---------------- | -------------------------------------------------- |
|
||||
| `alert.notifications.time-intervals:read` | n/a | Read mute time intervals. |
|
||||
| `alert.notifications.time-intervals:write` | n/a | Create new or update existing mute time intervals. |
|
||||
| `alert.notifications.time-intervals:delete` | n/a | Delete existing time intervals. |
|
||||
|
||||
Notification template permissions. To enable these permissions, enable the `alertingApiServer` feature toggle.
|
||||
|
||||
| Action | Applicable scope | Description |
|
||||
| -------------------------------------- | ---------------- | ---------------------------------------- |
|
||||
| `alert.notifications.templates:read` | n/a | Read templates. |
|
||||
| `alert.notifications.templates:write` | n/a | Create new or update existing templates. |
|
||||
| `alert.notifications.templates:delete` | n/a | Delete existing templates. |
|
||||
|
||||
To help plan your RBAC rollout strategy, refer to [Plan your RBAC rollout strategy](https://grafana.com/docs/grafana/next/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/).
|
||||
|
||||
@@ -57,12 +57,28 @@ Details of the fixed roles and the access they provide for Grafana Alerting are
|
||||
| Access to alert rules provisioning API: `fixed:alerting.provisioning:writer` | `alert.provisioning:read` and `alert.provisioning:write` | Manage all alert rules, notification policies, contact points, templates, in the organization using the provisioning API. |
|
||||
| Set provisioning status: `fixed:alerting.provisioning.status:writer` | `alert.provisioning.provenance:write` | Set provisioning rules for Alerting resources. Should be used together with other regular roles (Notifications Writer and/or Rules Writer.) |
|
||||
|
||||
If you have enabled the `alertingApiServer` feature toggle, an additional set of fixed roles is available.
|
||||
|
||||
| Display name in UI / Fixed role | Permissions | Description |
|
||||
| ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- |
|
||||
| Contact Point Reader: `fixed:alerting.receivers:reader` | `alert.notifications.receivers:read` for scope `receivers:*` | Read all contact points. |
|
||||
| Contact Point Creator: `fixed:alerting.receivers:creator` | `alert.notifications.receivers:create` | Create a new contact point. The user is automatically granted full access to the created contact point. |
|
||||
| Contact Point Writer: `fixed:alerting.receivers:writer` | `alert.notifications.receivers:read`, `alert.notifications.receivers:write`, `alert.notifications.receivers:delete` for scope `receivers:*` and <br> `alert.notifications.receivers:create` | Create a new contact point and manage all existing contact points. |
|
||||
| Templates Reader: `fixed:alerting.templates:reader` | `alert.notifications.templates:read` | Read all notification templates. |
|
||||
| Templates Writer: `fixed:alerting.templates:writer` | `alert.notifications.templates:read`, `alert.notifications.templates:write`, `alert.notifications.templates:delete` | Create new and manage existing notification templates. |
|
||||
| Time Intervals Reader: `fixed:alerting.time-intervals:reader` | `alert.notifications.time-intervals:read` | Read all time intervals. |
|
||||
| Time Intervals Writer: `fixed:alerting.time-intervals:writer` | `alert.notifications.time-intervals:read`, `alert.notifications.time-intervals:write`, `alert.notifications.time-intervals:delete` | Create new and manage existing time intervals. |
|
||||
|
||||
## Create custom roles
|
||||
|
||||
Create custom roles of your own to manage permissions. Custom roles contain unique combinations of permissions, actions and scopes. Create a custom role when basic roles and fixed roles do not meet your permissions requirements.
|
||||
|
||||
For more information on creating custom roles, refer to [Create custom roles](https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/manage-rbac-roles/#create-custom-roles).
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
It is not recommended to create custom roles that include `alerting.notifications.receiver` actions with a scope other than `receivers:*`. The UID used in the scope is not stable and changes whenever a contact point is renamed.
|
||||
{{< /admonition >}}
|
||||
|
||||
### Examples
|
||||
|
||||
The following examples give you an idea of how you can combine permissions for Grafana Alerting.
|
||||
|
||||
@@ -64,3 +64,30 @@ To manage folder permissions, complete the following steps.
|
||||
1. Hover your mouse cursor over a folder and click **Go to folder**.
|
||||
1. Click **Manage permissions** from the Folder actions menu.
|
||||
1. Update or add permissions as required.
|
||||
|
||||
## Manage access using contact point permissions
|
||||
|
||||
### Before you begin
|
||||
|
||||
- Enable the `alertingApiServer` feature toggle.
|
||||
|
||||
Extend or limit the access provided by a role to contact points by assigning permissions to individual contact point.
|
||||
|
||||
This allows different users, teams, or service accounts to have customized access to read or modify specific contact points.
|
||||
|
||||
Refer to the following table for details on the additional access provided by contact point permissions.
|
||||
|
||||
| Folder permission | Additional Access |
|
||||
| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| View | View and export contact point as well as select it on the Alert rule edit page |
|
||||
| Edit | Update or delete the contact point |
|
||||
| Admin | Same additional access as Edit and manage permissions for the contact point. User should have additional permissions to read users and teams. |
|
||||
|
||||
### Steps
|
||||
|
||||
To contact point permissions, complete the following steps.
|
||||
|
||||
1. In the left-side menu, click **Contact points**.
|
||||
1. Hover your mouse cursor over a contact point and click **More**.
|
||||
1. Click **Manage permissions** from the actions menu.
|
||||
1. Update or add permissions as required.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
---
|
||||
aliases:
|
||||
- ./meta-monitoring/ # /docs/grafana/<GRAFANA_VERSION>/alerting/meta-monitoring/
|
||||
- ./set-up/meta-monitoring/ # /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/meta-monitoring/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/monitor/
|
||||
- ../meta-monitoring/ # /docs/grafana/<GRAFANA_VERSION>/alerting/meta-monitoring/
|
||||
- ../monitoring/ # /docs/grafana/<GRAFANA_VERSION>/alerting/monitoring/
|
||||
canonical: https://grafana.com/docs/grafana/latest/alerting/set-up/meta-monitoring
|
||||
description: Monitor your alerting metrics to ensure you identify potential issues before they become critical.
|
||||
keywords:
|
||||
- grafana
|
||||
@@ -12,9 +12,8 @@ labels:
|
||||
products:
|
||||
- enterprise
|
||||
- oss
|
||||
menuTitle: Monitor
|
||||
title: Meta monitoring
|
||||
weight: 140
|
||||
weight: 700
|
||||
---
|
||||
|
||||
# Meta monitoring
|
||||
@@ -15,7 +15,7 @@ labels:
|
||||
- enterprise
|
||||
- oss
|
||||
title: Performance considerations and limitations
|
||||
weight: 600
|
||||
weight: 800
|
||||
---
|
||||
|
||||
# Performance considerations and limitations
|
||||
|
||||
@@ -118,7 +118,7 @@ Ensure you have a public dashboard footer logo or footer text set if you don't w
|
||||
|
||||
#### Learn more
|
||||
|
||||
[Configure custom branding documentation](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/configure-custom-branding/#custom-branding-for-shared-dashboards) for public dashboards
|
||||
[Configure custom branding documentation](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-grafana/configure-custom-branding/#custom-branding-for-public-dashboards) for public dashboards
|
||||
|
||||
### Subfolders cause very rare issues with folders that have forward slashes in their names
|
||||
|
||||
|
||||
@@ -31,17 +31,17 @@ cards:
|
||||
href: ./variables/
|
||||
description: Add variables to metric queries and panel titles to create interactive and dynamic dashboards.
|
||||
height: 24
|
||||
- title: Public dashboards
|
||||
href: ./dashboard-public/
|
||||
description: Make your Grafana dashboards public and share them with anyone without requiring access to your Grafana organization.
|
||||
height: 24
|
||||
- title: Reporting
|
||||
href: ./create-reports/
|
||||
description: Automatically generate and share PDF reports from your Grafana dashboards.
|
||||
height: 24
|
||||
- title: Sharing
|
||||
href: ./share-dashboards-panels/
|
||||
description: Share Grafana dashboards and panels using links, snapshots, embeds, and exports.
|
||||
height: 24
|
||||
- title: Shared dashboards
|
||||
href: ./share-dashboards-panels/shared-dashboards/
|
||||
description: Share your dashboards with anyone without requiring access to your Grafana organization.
|
||||
description: Share Grafana dashboards and panels within your organization using links, snapshots, and JSON exports.
|
||||
height: 24
|
||||
refs:
|
||||
panels:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user