{ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "metadata": { "name": "Hayami DTM, Azure Monitor alerting", "description": "Creates Azure Monitor metric alert rules on the custom metrics each DTM node\npublishes to its OWN virtual machine (namespace DTM), one set per node VM, and\nattaches every rule to an action group YOU already have. How you are notified\n(email, SMS, Teams, a webhook, or nothing) is the action group's business, not\nthis sample's.\n\nThe rule set, its thresholds and the prerequisites (turn the exporter on with a\nblank target on every node) are described in ../README.md. The Terraform and\nBicep flavours beside this one create the same rules from the same parameters.\n\nDeploy into the resource group that holds the DTM node VMs:\n\n az deployment group create \\\n --resource-group rg-dtm \\\n --template-file azuredeploy.json \\\n --parameters actionGroupId= \\\n nodeVmNames='[\"\",\"\",\"\"]'\n\nOr in the Azure portal: Deploy a custom template, Build your own template in\nthe editor, and paste this file.\n\nSelf-contained: this file is the whole sample. There is nothing else to fetch.\n" }, "parameters": { "actionGroupId": { "type": "string", "defaultValue": "", "metadata": { "description": "Resource ID of your EXISTING Azure Monitor action group. Every rule notifies it. Leave empty only with createExampleActionGroup = true." } }, "nodeVmNames": { "type": "array", "minLength": 1, "metadata": { "description": "Names of the DTM node VMs in this resource group. Every rule is created once per VM, scoped to that VM, which is where each node publishes its own metrics. If your nodes span resource groups, deploy once per group and set expectedNodeCount to the whole cluster's size." } }, "expectedNodeCount": { "type": "int", "defaultValue": 0, "minValue": 0, "maxValue": 100, "metadata": { "description": "Cluster size the \"cluster undersized\" rule compares each node's view of its membership against, 1 to 100. 0 means the number of nodeVmNames." } }, "expectedDnsListeners": { "type": "int", "defaultValue": 2, "minValue": 1, "maxValue": 16, "metadata": { "description": "DNS listeners a healthy node runs, 1 to 16: UDP and TCP, plus one if DNS-over-TLS is enabled (DNS-over-HTTPS is not counted). Read dtm_dns_listeners_total from a healthy node's /metrics to be sure." } }, "dnsLatencyP99ThresholdSeconds": { "type": "string", "defaultValue": "0.010", "metadata": { "description": "Authoritative DNS p99 latency, in SECONDS, above which \"DNS latency above SLO\" fires: the p99 over 15 minutes of the answers this node gives from its own zones. A decimal string, because template parameters have no decimal type, so nothing checks it: write seconds, not milliseconds ('0.010' is 10 ms, '10' is 10 seconds), and note that '0' fires whenever the 100-answer gate is met. The default is the 10 ms objective in deploy/grafana/dtm-alerts.yml." } }, "dnsLatencyP95ThresholdSeconds": { "type": "string", "defaultValue": "0.050", "metadata": { "description": "Authoritative DNS p95 latency, in SECONDS, above which \"DNS latency high\" fires. A decimal string in seconds, unchecked like the p99 one ('0.050' is 50 ms; '0' fires whenever the 100-answer gate is met). The default is the 50 ms tier in deploy/grafana/dtm-alerts.yml." } }, "grantMetricsPublisher": { "type": "bool", "defaultValue": true, "metadata": { "description": "Grant each node VM's system-assigned identity the Monitoring Metrics Publisher role on its own VM, which it needs to publish its metrics. Set false if roles are assigned elsewhere, or if the nodes run on a user-assigned identity: grant that identity the role on each VM yourself." } }, "ruleNamePrefix": { "type": "string", "defaultValue": "dtm", "metadata": { "description": "Prefix for every alert rule name." } }, "createExampleActionGroup": { "type": "bool", "defaultValue": false, "metadata": { "description": "EXAMPLE ONLY. Create an email-only action group and use it in place of actionGroupId, to try the rules out. Replace it with your own action group for real use." } }, "exampleActionGroupEmail": { "type": "string", "defaultValue": "", "metadata": { "description": "EXAMPLE ONLY. The address the example action group emails, when createExampleActionGroup = true." } } }, "variables": { "notifyActionGroupId": "[if(parameters('createExampleActionGroup'), resourceId('Microsoft.Insights/actionGroups', format('{0}-example-email', parameters('ruleNamePrefix'))), parameters('actionGroupId'))]", "clusterSize": "[if(greater(parameters('expectedNodeCount'), 0), parameters('expectedNodeCount'), length(parameters('nodeVmNames')))]", "rules": [ { "key": "heartbeat-missing", "description": "This DTM node has stopped publishing its heartbeat. Whether Azure Monitor fires on a custom metric that stops arriving is not guaranteed; cluster-undersized on the other nodes is the dependable node-down signal.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_heartbeat", "aggregation": "Count", "operator": "LessThan", "threshold": 1, "dimensions": [] } ] }, { "key": "cluster-undersized", "description": "This DTM node sees fewer cluster members than expected: a peer has stopped, or can no longer exchange replication traffic with it.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_cluster_nodes", "aggregation": "Maximum", "operator": "LessThan", "threshold": "[variables('clusterSize')]", "dimensions": [] } ] }, { "key": "not-serving-dns", "description": "Fewer DNS listeners are alive on this DTM node than it should run.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_dns_listeners_alive", "aggregation": "Maximum", "operator": "LessThan", "threshold": "[parameters('expectedDnsListeners')]", "dimensions": [] } ] }, { "key": "stored-certificate-refused", "description": "This DTM node cannot serve its stored API certificate bundle (the certificate and its key do not pair, or the key is missing).", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_tls_stored_pair_refused", "aggregation": "Minimum", "operator": "GreaterThanOrEqual", "threshold": 1, "dimensions": [] } ] }, { "key": "tls-certificate-expiring", "description": "A TLS certificate this DTM node presents (the API, or DNS-over-TLS and DNS-over-HTTPS) expires within 21 days.", "severity": 2, "frequency": "PT1H", "window": "PT6H", "criteria": [ { "metric": "dtm_tls_certificate_expires_in_seconds", "aggregation": "Minimum", "operator": "LessThan", "threshold": 1814400, "dimensions": [] } ] }, { "key": "member-unreachable-day", "description": "A cluster member has been unreachable from this DTM node for over a day.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_replication_longest_unreachable_member_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 86400, "dimensions": [] } ] }, { "key": "member-unreachable-week", "description": "A cluster member has been unreachable from this DTM node for over a week.", "severity": 0, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_replication_longest_unreachable_member_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 604800, "dimensions": [] } ] }, { "key": "member-fenced", "description": "This DTM node is fencing at least one cluster member.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_replication_fenced_members", "aggregation": "Maximum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "stale-replica-blocked", "description": "This DTM node is blocking a stale replica from rejoining.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_replication_stale_replica_blocked", "aggregation": "Maximum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "swap-stable-ip-unrestored", "description": "A failed DTM swap left its stable IP with no running VM confirmed holding it.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_lifecycle_swap_stable_ip_unrestored", "aggregation": "Maximum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "security-advisory-critical", "description": "A critical security advisory is active for this DTM version.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_advisory_active", "aggregation": "Maximum", "operator": "GreaterThanOrEqual", "threshold": 1, "dimensions": [ { "name": "severity", "operator": "Include", "values": [ "critical" ] } ] } ] }, { "key": "security-advisory", "description": "A security advisory is active for this DTM version.", "severity": 2, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_advisory_active", "aggregation": "Maximum", "operator": "GreaterThanOrEqual", "threshold": 1, "dimensions": [ { "name": "severity", "operator": "Exclude", "values": [ "critical" ] } ] } ] }, { "key": "tag-registration-unclaimed", "description": "Resources are tagged for registration with DTM, but the record they name does not exist.", "severity": 2, "frequency": "PT5M", "window": "PT30M", "criteria": [ { "metric": "dtm_tag_registration_unclaimed_endpoints", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "management-group-access-revoked", "description": "DTM region discovery has lost access to a management group it was reading.", "severity": 2, "frequency": "PT15M", "window": "PT1H", "criteria": [ { "metric": "dtm_region_discovery_membership_lost_access", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "region-discovery-incomplete", "description": "DTM region discovery has run, but its last cycle did not complete.", "severity": 2, "frequency": "PT1H", "window": "PT6H", "criteria": [ { "metric": "dtm_region_discovery_last_cycle_complete", "aggregation": "Maximum", "operator": "LessThan", "threshold": 1, "dimensions": [] }, { "metric": "dtm_region_discovery_last_run_timestamp_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "tag-registration-incomplete", "description": "DTM tag registration has run, but its last cycle did not complete.", "severity": 2, "frequency": "PT1H", "window": "PT6H", "criteria": [ { "metric": "dtm_tag_registration_last_cycle_complete", "aggregation": "Maximum", "operator": "LessThan", "threshold": 1, "dimensions": [] }, { "metric": "dtm_tag_registration_last_run_timestamp_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "data-disk-critical", "description": "Less than 5% of this DTM node's data volume is free.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_storage_free_ratio", "aggregation": "Maximum", "operator": "LessThan", "threshold": "[json('0.05')]", "dimensions": [] } ] }, { "key": "data-disk-low", "description": "Less than 10% of this DTM node's data volume is free.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_storage_free_ratio", "aggregation": "Maximum", "operator": "LessThan", "threshold": "[json('0.10')]", "dimensions": [] } ] }, { "key": "restart-loop", "description": "This DTM node has started more than twice in 15 minutes: something is killing the process shortly after boot.", "severity": 0, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_process_starts", "aggregation": "Total", "operator": "GreaterThan", "threshold": 2, "dimensions": [] } ] }, { "key": "bbolt-stale-lock-loop", "description": "This DTM node keeps recovering a stale database lock at startup: a disk or process problem, not a one-off.", "severity": 2, "frequency": "PT5M", "window": "PT30M", "criteria": [ { "metric": "dtm_storage_bbolt_stale_lock_recovered_total", "aggregation": "Total", "operator": "GreaterThanOrEqual", "threshold": 2, "dimensions": [] } ] }, { "key": "memory-self-restart", "description": "This DTM node restarted itself because its memory stayed over the configured cap.", "severity": 2, "frequency": "PT15M", "window": "PT1H", "criteria": [ { "metric": "dtm_memory_self_restart_total", "aggregation": "Total", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "health-check-drops", "description": "This DTM node keeps dropping health checks it has no capacity to run.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_health_check_drops_increase_5m", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "query-error-rate", "description": "More than 5% of the DNS queries this DTM node answered in 5 minutes ended in an error (not NOERROR or NXDOMAIN).", "severity": 2, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_dns_query_error_ratio_5m", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": "[json('0.05')]", "dimensions": [] } ] }, { "key": "dns-latency-above-slo", "description": "The p99 latency of this DTM node's authoritative DNS answers over 15 minutes is above the objective, with at least 100 such answers.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_dns_authoritative_latency_p99_15m_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": "[json(parameters('dnsLatencyP99ThresholdSeconds'))]", "dimensions": [] }, { "metric": "dtm_dns_authoritative_queries_increase_15m", "aggregation": "Minimum", "operator": "GreaterThanOrEqual", "threshold": 100, "dimensions": [] } ] }, { "key": "dns-latency-high", "description": "The p95 latency of this DTM node's authoritative DNS answers over 15 minutes is high, with at least 100 such answers.", "severity": 0, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_dns_authoritative_latency_p95_15m_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": "[json(parameters('dnsLatencyP95ThresholdSeconds'))]", "dimensions": [] }, { "metric": "dtm_dns_authoritative_queries_increase_15m", "aggregation": "Minimum", "operator": "GreaterThanOrEqual", "threshold": 100, "dimensions": [] } ] }, { "key": "bbolt-freelist-bloat", "description": "More than half of this DTM node's database pages have been free for 6 hours: compact it.", "severity": 2, "frequency": "PT1H", "window": "PT6H", "criteria": [ { "metric": "dtm_storage_bbolt_free_page_ratio", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": "[json('0.5')]", "dimensions": [] } ] }, { "key": "replication-lag-high", "description": "This DTM node has not completed a replication sync for over 2 minutes while it has peers.", "severity": 2, "frequency": "PT1M", "window": "PT5M", "criteria": [ { "metric": "dtm_replication_last_sync_age_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 120, "dimensions": [] }, { "metric": "dtm_cluster_nodes", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 1, "dimensions": [] } ] }, { "key": "replication-queue-backlog", "description": "The replication broadcast queue on this DTM node has held 5000 or more messages for 15 minutes while it has peers.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_replication_broadcast_queue_depth", "aggregation": "Minimum", "operator": "GreaterThanOrEqual", "threshold": 5000, "dimensions": [] }, { "metric": "dtm_cluster_nodes", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 1, "dimensions": [] } ] }, { "key": "region-discovery-stalled", "description": "DTM region discovery has run before, but not for over 3 hours.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_region_discovery_last_run_age_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 10800, "dimensions": [] } ] }, { "key": "tag-registration-stalled", "description": "DTM tag registration has run before, but not for over 3 hours.", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_tag_registration_last_run_age_seconds", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 10800, "dimensions": [] } ] }, { "key": "region-discovery-failing", "description": "DTM region discovery runs have kept failing for an hour.", "severity": 2, "frequency": "PT15M", "window": "PT1H", "criteria": [ { "metric": "dtm_region_discovery_run_errors_increase_30m", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "tag-registration-failing", "description": "DTM tag registration runs have kept failing for an hour.", "severity": 2, "frequency": "PT15M", "window": "PT1H", "criteria": [ { "metric": "dtm_tag_registration_run_errors_increase_30m", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [] } ] }, { "key": "azure-auth-denied", "description": "Azure keeps denying a DTM node's identity access it needs (a role assignment is missing or was removed).", "severity": 2, "frequency": "PT5M", "window": "PT15M", "criteria": [ { "metric": "dtm_azure_auth_denials_increase_10m", "aggregation": "Minimum", "operator": "GreaterThan", "threshold": 0, "dimensions": [ { "name": "backend", "operator": "Include", "values": [ "*" ] } ] } ] } ], "byNode": { "name": "node", "operator": "Include", "values": [ "*" ] }, "azureCriteria": "[map(variables('rules'), lambda('rule', map(range(0, length(lambdaVariables('rule').criteria)), lambda('j', createObject('criterionType', 'StaticThresholdCriterion', 'name', format('c{0}', add(lambdaVariables('j'), 1)), 'metricNamespace', 'DTM', 'metricName', lambdaVariables('rule').criteria[lambdaVariables('j')].metric, 'timeAggregation', lambdaVariables('rule').criteria[lambdaVariables('j')].aggregation, 'operator', lambdaVariables('rule').criteria[lambdaVariables('j')].operator, 'threshold', lambdaVariables('rule').criteria[lambdaVariables('j')].threshold, 'dimensions', concat(if(equals(length(lambdaVariables('rule').criteria), 1), createArray(variables('byNode')), createArray()), lambdaVariables('rule').criteria[lambdaVariables('j')].dimensions), 'skipMetricValidation', true())))))]", "monitoringMetricsPublisherRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb')]" }, "resources": [ { "condition": "[parameters('createExampleActionGroup')]", "type": "Microsoft.Insights/actionGroups", "apiVersion": "2023-01-01", "name": "[format('{0}-example-email', parameters('ruleNamePrefix'))]", "location": "global", "properties": { "groupShortName": "dtmexample", "enabled": true, "emailReceivers": [ { "name": "example", "emailAddress": "[parameters('exampleActionGroupEmail')]", "useCommonAlertSchema": true } ] } }, { "copy": { "name": "alerts", "count": "[mul(length(parameters('nodeVmNames')), length(variables('rules')))]" }, "type": "Microsoft.Insights/metricAlerts", "apiVersion": "2018-03-01", "name": "[format('{0}-{1}-{2}', parameters('ruleNamePrefix'), parameters('nodeVmNames')[div(copyIndex(), length(variables('rules')))], variables('rules')[mod(copyIndex(), length(variables('rules')))].key)]", "location": "global", "properties": { "description": "[variables('rules')[mod(copyIndex(), length(variables('rules')))].description]", "severity": "[variables('rules')[mod(copyIndex(), length(variables('rules')))].severity]", "enabled": true, "scopes": [ "[resourceId('Microsoft.Compute/virtualMachines', parameters('nodeVmNames')[div(copyIndex(), length(variables('rules')))])]" ], "evaluationFrequency": "[variables('rules')[mod(copyIndex(), length(variables('rules')))].frequency]", "windowSize": "[variables('rules')[mod(copyIndex(), length(variables('rules')))].window]", "autoMitigate": true, "criteria": { "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", "allOf": "[variables('azureCriteria')[mod(copyIndex(), length(variables('rules')))]]" }, "actions": [ { "actionGroupId": "[variables('notifyActionGroupId')]" } ] }, "dependsOn": [ "[resourceId('Microsoft.Insights/actionGroups', format('{0}-example-email', parameters('ruleNamePrefix')))]" ] }, { "copy": { "name": "metricsPublisher", "count": "[length(parameters('nodeVmNames'))]" }, "condition": "[parameters('grantMetricsPublisher')]", "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", "scope": "[resourceId('Microsoft.Compute/virtualMachines', parameters('nodeVmNames')[copyIndex()])]", "name": "[guid(resourceId('Microsoft.Compute/virtualMachines', parameters('nodeVmNames')[copyIndex()]), variables('monitoringMetricsPublisherRoleId'))]", "properties": { "roleDefinitionId": "[variables('monitoringMetricsPublisherRoleId')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('nodeVmNames')[copyIndex()]), '2024-07-01', 'full').identity.principalId]", "principalType": "ServicePrincipal" } } ], "outputs": { "alertRuleIds": { "type": "array", "metadata": { "description": "Resource IDs of the alert rules created." }, "copy": { "count": "[mul(length(parameters('nodeVmNames')), length(variables('rules')))]", "input": "[resourceId('Microsoft.Insights/metricAlerts', format('{0}-{1}-{2}', parameters('ruleNamePrefix'), parameters('nodeVmNames')[div(copyIndex(), length(variables('rules')))], variables('rules')[mod(copyIndex(), length(variables('rules')))].key))]" } }, "notifiedActionGroupId": { "type": "string", "metadata": { "description": "The action group every rule notifies." }, "value": "[variables('notifyActionGroupId')]" } } }