Implementing Best Practices

# Correct naming conventions
http_requests_total
process_resident_memory_bytes
http_request_duration_seconds_sum
cache_hits_total

# Avoid these
HTTPRequestTotal
processResidentMemoryBytes
httpRequestDurationSecondsSum

http_requests_total{user_id="1234567890", endpoint="/api/v1/users"}

http_requests_total{job="api-server", endpoint="/api/v1/users", status_code="200"}

package main

import (
    "log"
    "net/http"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpRequestCount = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total number of HTTP requests.",
    }, []string{"method", "code"})
)

func main() {
    http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
        promhttp.Handler().ServeHTTP(w, r)
    })

    http.HandleFunc("/hello", func(w http.ResponseWriter, r *http.Request) {
        httpRequestCount.WithLabelValues(r.Method, "200").Inc()
        w.WriteHeader(http.StatusOK)
        w.Write([]byte("Hello, World!"))
    })

    log.Fatal(http.ListenAndServe(":8080", nil))
}

from prometheus_client import Counter, start_http_server
import time

# Create a counter metric
http_requests = Counter("http_requests_total", "Total number of HTTP requests.", [ "method", "code" ])

def handle_request():
    # Increment the counter when handling a request
    http_requests.labels(method="GET", code="200").inc()
    return "Hello, World!"

if __name__ == "__main__":
    start_http_server(8000)
    while True:
        handle_request()
        time.sleep(1)

groups:
- name: http-errors
  rules:
  - alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5xx"}[1m]) / rate(http_requests_total[1m]) > 0.01
    for: 5m
    labels:
      severity: page
    annotations:
      summary: "High error rate detected"
      description: "Error rate is above 1% for more than 5 minutes."

# Prometheus recording rule to ensure all metrics have required labels
groups:
- name: example_metrics
  rules:
  - record: instance_job:up:rate1m
    expr: up{job!=""} == 1

const client = require('prom-client');
const express = require('express');

// Create metrics
const httpRequestCounter = new client.Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'status']
});

const httpRequestDuration = new client.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  buckets: [0.05, 0.1, 0.2, 0.5, 1, 2, 5]
});

const app = express();

app.use((req, res, next) => {
  const end = httpRequestDuration.startTimer();
  res.on('finish', () => {
    end({ method: req.method, status: res.statusCode });
    httpRequestCounter.inc({ method: req.method, status: res.statusCode });
  });
  next();
});

app.get('/', (req, res) => {
  res.send('Hello World');
});

app.listen(3000);

const histogram = new Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  buckets: [0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10]
});

// Record a sample
histogram.observe(0.3);

const counter = new Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests'
});

// Increment the counter
counter.inc();

groups:
- name: cpu_rules
  rules:
  - record: node_cpu:usage
    expr: rate(node_cpu_seconds_total[5m]) * 100

silences:
  - matchers:
      - name: alertname
        value: InstanceDown
    startsAt: "2026-01-01T00:00:00Z"
    endsAt: "2026-01-01T06:00:00Z"
    comment: "Scheduled maintenance window"

name: HighErrorRate
expr: rate(errors[5m]) > 10
annotations:
  summary: "High error rate detected"
  description: "The service is experiencing a high number of errors. See the runbook for troubleshooting steps."
  runbook_url: "https://internal.docs.example.com/troubleshooting/service-errors"
  runbook_text: "View Troubleshooting Guide"

global:
  external_labels:
    environment: production
    region: us-west-1

scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

promtool check config /path/to/prometheus.yml