0

目标是使用从 kube-prometheus-stack helm chart 部署的 Prometheus/Grafana 为 Kubernetes 上的 ActiveMQ 部署实现 JMX 监控。

后续步骤：

  1. 创建了一个包含以下内容的 dockerfile 以启用容器上的 javaagent 来监控 JMX

    FROM rmohr/activemq:5.15.9
ENV ACTIVEMQ_TCP=61616 ACTIVEMQ_AMQP=5672 ACTIVEMQ_STOMP=61613 ACTIVEMQ_MQTT=1883 ACTIVEMQ_WS=61614 ACTIVEMQ_UI=8161
ENV ACTIVEMQ_HOME /opt/activemq
ENV AGENT_VERSION=0.3.0
ENV AGENT_SHA512=af1abc5e9412e62f767d2ece00930b08e749278030ff26a690d721c707b526ca1f99731e75a4f23f5080e7f57b4d61c3e3f8d510b7c0e998e1eb397b1e1ac08c
ENV AGENT_BASE=jmx_prometheus_javaagent
ENV AGENT=$AGENT_BASE-$AGENT_VERSION.jar
ENV AGENT_CONFIG=config.yaml
ENV AGENT_PORT=8080
ENV AGENT_OPTS="-javaagent:/$AGENT=$AGENT_PORT:/config.yaml"
ADD https://repo1.maven.org/maven2/io/prometheus/jmx/$AGENT_BASE/$AGENT_VERSION/$AGENT /
USER root
RUN chmod 0644 /$AGENT && if [ "$AGENT_SHA512" != "$(cd / && sha512sum $AGENT | awk '{print($1)}')" ]; then \
        echo "Agent sha512 value doesn't match! exiting."  && exit 1; \
    fi
COPY $AGENT_CONFIG /
USER activemq
WORKDIR $ACTIVEMQ_HOME
EXPOSE $ACTIVEMQ_TCP $ACTIVEMQ_AMQP $ACTIVEMQ_STOMP $ACTIVEMQ_MQTT $ACTIVEMQ_WS $ACTIVEMQ_UI $AGENT_PORT
ENV ACTIVEMQ_OPTS_MEMORY="-Xms64M -Xmx1G"
ENV ACTIVEMQ_CONF=$ACTIVEMQ_HOME/conf
ENV ACTIVEMQ_OPTS="$AGENT_OPTS $ACTIVEMQ_OPTS_MEMORY -Djava.util.logging.config.file=logging.properties -Djava.security.auth.login.config=$ACTIVEMQ_CONF/login.config"
CMD ["/bin/sh", "-c", "bin/activemq console"]

  2. 构建上述镜像后，将其部署到我们的 kubernetes 集群中。

  3. 登录到 ActiveMQ 容器并运行curl http://localhost:8080/metrics，它正确返回了一些 JMX 指标

  4. 使用 helm 图表安装 kube-prometheus-stack：

    helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm install prometheus prometheus-community/kube-prometheus-stack

  5. 它已成功部署，我们可以开始使用 Prometheus/Grafana 监控 Kubernetes 集群中的其他资源

  6. 接下来，我想将 ActiveMQ pod JMX 指标放到 Prometheus 上。为此，我创建了带有标签的 service 和 serviceMonitor 文件release: prometheus并在集群中进行了配置。

  7. 期望是 Prometheus 发现了在端口公开的 ActiveMQ JMX 服务8080，但它没有发现该服务

  8. 作为第二种选择，我尝试通过additionalScrapeConfigs在自定义值文件中提供以下内容来升级 kube-prometheus-stack 安装并执行helm upgrade. 最后提供配置。

以下是使用的部署、服务和服务监控文件：

Deployment:

    apiVersion: apps/v1
    kind: Deployment
    metadata:
      annotations:
        meta.helm.sh/release-name: prometheus
        meta.helm.sh/release-namespace: monitoring
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
      name: activemq
      labels:
        app.kubernetes.io/instance: prometheus
        app.kubernetes.io/name: activemq    
        release: prometheus
      namespace: monitoring  
    spec:
      replicas: 1
      selector:
        matchLabels:
          app.kubernetes.io/instance: prometheus
          app.kubernetes.io/name: activemq
          app: activemq      
      template:
        metadata:
          labels:
            app.kubernetes.io/instance: prometheus
            app.kubernetes.io/name: activemq
            app: activemq         
        
        spec:
          containers:
            - name: activemq
              image: prashantunnikrishnan/activemq-jmx:5.15.9
              imagePullPolicy: IfNotPresent
              ports:
              - containerPort: 8161
                name: admin
              - containerPort: 61616
                name: transport-port
              - containerPort: 61613
                name: stomp-port
              - containerPort: 8080
                name: jmx-metrics

Service:

    apiVersion: v1
    kind: Service
    metadata:
      annotations:
        meta.helm.sh/release-name: prometheus
        meta.helm.sh/release-namespace: monitoring
        prometheus.io/scrape: "true"
        prometheus.io/scrape: "8080"
        prometheuses.monitoring.coreos.com/non-namespaced: "true"
        servicemonitors.monitoring.coreos.com/non-namespaced: "true"
      name: activemq
      labels:
        app: activemq
        app.kubernetes.io/instance: prometheus
        app.kubernetes.io/name: activemq 
        release: prometheus    
    spec:
      selector:
        app: activemq
        app.kubernetes.io/instance: prometheus
        app.kubernetes.io/name: activemq    
      ports:
        - protocol: TCP
          port: 8161
          targetPort: 8161
          name: admin
        - protocol: TCP
          port: 61616
          targetPort: 61616
          name: transport-port
        - protocol: TCP
          port: 61613
          targetPort: 61613
          name: stomp-port     
        - protocol: TCP
          port: 8080
          targetPort: 8080
          name: jmx-metrics

ServiceMonitor:

    apiVersion: monitoring.coreos.com/v1
    kind: ServiceMonitor
    metadata:
      annotations:
        meta.helm.sh/release-name: prometheus
        meta.helm.sh/release-namespace: monitoring  
      generation: 1
      labels:
        app.kubernetes.io/instance: prometheus  
        app: activemq
        release: prometheus
      name: activemq
      namespace: monitoring
    spec:
      endpoints:
      - path: /metrics
        port: jmx-metrics
      selector:
        matchLabels:
          app.kubernetes.io/instance: prometheus
          app.kubernetes.io/name: activemq     
          app: activemq
          release: prometheus

Values.yaml：

additionalScrapeConfigs:   
  - job_name: jmx_exporter
    scrape_interval: 15s
    kubernetes_sd_configs:
    - role: pod
      namespaces:
        names:
          - monitoring
    relabel_configs:
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: pod
    - source_labels: [__address__]
      action: replace
      regex: ([^:]+)(?::\d+)?
      replacement: ${1}:8080
      target_label: __address__
    - source_labels: [__meta_kubernetes_pod_label_app]
      action: keep
      regex: activemq-*
4

0 回答 0