我已经使用官方helm图表设置了 Loki Distributed 。
nameOverride: null
# -- Overrides the chart's computed full name
fullnameOverride: null
# -- Image pull secrets for Docker images
imagePullSecrets: []
loki:
# Configures the readiness probe for all of the Loki pods
readinessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 30
timeoutSeconds: 1
image:
# -- The Docker registry
registry: docker.io
# -- Docker image repository
repository: grafana/loki
# -- Overrides the image tag whose default is the chart's appVersion
tag: null
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Common annotations for all pods
podAnnotations: {}
# -- The number of old ReplicaSets to retain to allow rollback
revisionHistoryLimit: 10
# -- The SecurityContext for Loki pods
podSecurityContext:
fsGroup: 10001
runAsGroup: 10001
runAsNonRoot: true
runAsUser: 10001
# -- The SecurityContext for Loki containers
containerSecurityContext:
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
allowPrivilegeEscalation: false
# -- Specify an existing secret containing loki configuration. If non-empty, overrides `loki.config`
existingSecretForConfig: ""
# -- Config file contents for Loki
# @default -- See values.yaml
config: |
auth_enabled: false
server:
log_level: info
http_listen_port: 3100
distributor:
ring:
kvstore:
store: memberlist
ingester:
max_transfer_retries: 0
chunk_idle_period: 1h
chunk_target_size: 1536000
max_chunk_age: 1h
lifecycler:
join_after: 0s
ring:
kvstore:
store: memberlist
memberlist:
join_members:
- {{ include "loki.fullname" . }}-memberlist
limits_config:
ingestion_rate_mb: 10
ingestion_burst_size_mb: 20
max_concurrent_tail_requests: 20
max_cache_freshness_per_query: 10m
schema_config:
configs:
- from: 2020-09-07
store: boltdb-shipper
object_store: aws
schema: v11
index:
prefix: loki_index_
period: 24h
storage_config:
aws:
s3: s3://eu-west-1
bucketnames: loki-dev-v1
boltdb_shipper:
active_index_directory: /var/loki/index
shared_store: s3
cache_location: /var/loki/cache
index_gateway_client:
server_address: dns:///{{ include "loki.indexGatewayFullname" . }}:9095
query_range:
align_queries_with_step: true
max_retries: 5
split_queries_by_interval: 15m
cache_results: true
results_cache:
cache:
enable_fifocache: true
fifocache:
max_size_items: 1024
validity: 24h
frontend_worker:
frontend_address: {{ include "loki.queryFrontendFullname" . }}:9095
frontend:
log_queries_longer_than: 5s
compress_responses: true
tail_proxy_url: http://{{ include "loki.querierFullname" . }}:3100
serviceAccount:
# -- Specifies whether a ServiceAccount should be created
create: true
# -- The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the fullname template
name: null
# -- Image pull secrets for the service account
imagePullSecrets: []
# -- Annotations for the service account
annotations: {}
# -- Set this toggle to false to opt out of automounting API credentials for the service account
automountServiceAccountToken: true
# RBAC configuration
rbac:
# -- If enabled, a PodSecurityPolicy is created
pspEnabled: false
# ServiceMonitor configuration
serviceMonitor:
# -- If enabled, ServiceMonitor resources for Prometheus Operator are created
enabled: false
# -- Alternative namespace for ServiceMonitor resources
namespace: null
# -- Namespace selector for ServiceMonitor resources
namespaceSelector: {}
# -- ServiceMonitor annotations
annotations: {}
# -- Additional ServiceMonitor labels
labels: {}
# -- ServiceMonitor scrape interval
interval: null
# -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
scrapeTimeout: null
# -- ServiceMonitor relabel configs to apply to samples before scraping
# https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
relabelings: []
# -- ServiceMonitor will use http by default, but you can pick https as well
scheme: http
# -- ServiceMonitor will use these tlsConfig settings to make the health check requests
tlsConfig: null
# Rules for the Prometheus Operator
prometheusRule:
# -- If enabled, a PrometheusRule resource for Prometheus Operator is created
enabled: false
# -- Alternative namespace for the PrometheusRule resource
namespace: null
# -- PrometheusRule annotations
annotations: {}
# -- Additional PrometheusRule labels
labels: {}
# -- Contents of Prometheus rules file
groups: []
# - name: loki-rules
# rules:
# - record: job:loki_request_duration_seconds_bucket:sum_rate
# expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job)
# - record: job_route:loki_request_duration_seconds_bucket:sum_rate
# expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route)
# - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate
# expr: sum(rate(container_cpu_usage_seconds_total[1m])) by (node, namespace, pod, container)
# Configuration for the ingester
ingester:
# -- Number of replicas for the ingester
replicas: 3
image:
# -- The Docker registry for the ingester image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the ingester image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the ingester image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for ingester pods
priorityClassName: null
# -- Annotations for ingester pods
podAnnotations: {}
# -- Labels for ingestor service
serviceLabels: {}
# -- Additional CLI args for the ingester
extraArgs: []
# -- Environment variables to add to the ingester pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the ingester pods
extraEnvFrom: []
# -- Volume mounts to add to the ingester pods
extraVolumeMounts: []
# -- Volumes to add to the ingester pods
extraVolumes: []
# -- Resource requests and limits for the ingester
resources: {}
# -- Grace period to allow the ingester to shutdown before it is killed. Especially for the ingestor,
# this must be increased. It must be long enough so ingesters can be gracefully shutdown flushing/transferring
# all data and to successfully leave the member ring on shutdown.
terminationGracePeriodSeconds: 300
# -- Affinity for ingester pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.ingesterSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.ingesterSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for ingester pods
nodeSelector: {}
# -- Tolerations for ingester pods
tolerations: []
persistence:
# Vaib - This si used by the boltdb shipper to save the indexes before flushing it to s3
# Vaib - https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/boltdb-shipper.md
# -- Enable creating PVCs which is required when using boltdb-shipper
enabled: true
# -- Size of persistent disk
size: 50Gi
# -- Storage class to be used.
# If defined, storageClassName: <storageClass>.
# If set to "-", storageClassName: "", which disables dynamic provisioning.
# If empty or set to null, no storageClassName spec is
# set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
storageClass: null
# Configuration for the distributor
distributor:
# -- Number of replicas for the distributor
replicas: 1
autoscaling:
# -- Enable autoscaling for the distributor
enabled: true
# -- Minimum autoscaling replicas for the distributor
minReplicas: 3
# -- Maximum autoscaling replicas for the distributor
maxReplicas: 5
# -- Target CPU utilisation percentage for the distributor
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the distributor
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the distributor image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the distributor image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the distributor image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for distributor pods
priorityClassName: null
# -- Annotations for distributor pods
podAnnotations: {}
# -- Labels for distributor service
serviceLabels: {}
# -- Additional CLI args for the distributor
extraArgs: []
# -- Environment variables to add to the distributor pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the distributor pods
extraEnvFrom: []
# -- Volume mounts to add to the distributor pods
extraVolumeMounts: []
# -- Volumes to add to the distributor pods
extraVolumes: []
# -- Resource requests and limits for the distributor
resources: {}
# -- Grace period to allow the distributor to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for distributor pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.distributorSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.distributorSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for distributor pods
nodeSelector: {}
# -- Tolerations for distributor pods
tolerations: []
# Configuration for the querier
# The querier service handles queries using the LogQL query language, fetching logs both from the ingesters and from long-term storage.
querier:
# -- Number of replicas for the querier
replicas: 1
autoscaling:
# -- Enable autoscaling for the querier, this is only used if `queryIndex.enabled: true`
enabled: true
# -- Minimum autoscaling replicas for the querier
minReplicas: 3
# -- Maximum autoscaling replicas for the querier
maxReplicas: 20
# -- Target CPU utilisation percentage for the querier
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the querier
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the querier image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the querier image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the querier image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for querier pods
priorityClassName: null
# -- Annotations for querier pods
podAnnotations: {}
# -- Labels for querier service
serviceLabels: {}
# -- Additional CLI args for the querier
extraArgs: []
# -- Environment variables to add to the querier pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the querier pods
extraEnvFrom: []
# -- Volume mounts to add to the querier pods
extraVolumeMounts: []
# -- Volumes to add to the querier pods
extraVolumes: []
# -- Resource requests and limits for the querier
resources: {}
# -- Grace period to allow the querier to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for querier pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.querierSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.querierSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for querier pods
nodeSelector: {}
# -- Tolerations for querier pods
tolerations: []
persistence:
# -- Enable creating PVCs for the querier cache
enabled: true
# -- Size of persistent disk
size: 50Gi
# -- Storage class to be used.
# If defined, storageClassName: <storageClass>.
# If set to "-", storageClassName: "", which disables dynamic provisioning.
# If empty or set to null, no storageClassName spec is
# set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
storageClass: null
# Configuration for the query-frontend
queryFrontend:
# -- Number of replicas for the query-frontend
replicas: 1
autoscaling:
# -- Enable autoscaling for the query-frontend
enabled: true
# -- Minimum autoscaling replicas for the query-frontend
minReplicas: 2
# -- Maximum autoscaling replicas for the query-frontend
maxReplicas: 3
# -- Target CPU utilisation percentage for the query-frontend
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the query-frontend
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the query-frontend image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the query-frontend image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the query-frontend image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for query-frontend pods
priorityClassName: null
# -- Annotations for query-frontend pods
podAnnotations: {}
# -- Labels for query-frontend service
serviceLabels: {}
# -- Additional CLI args for the query-frontend
extraArgs: []
# -- Environment variables to add to the query-frontend pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the query-frontend pods
extraEnvFrom: []
# -- Volume mounts to add to the query-frontend pods
extraVolumeMounts: []
# -- Volumes to add to the query-frontend pods
extraVolumes: []
# -- Resource requests and limits for the query-frontend
resources: {}
# -- Grace period to allow the query-frontend to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for query-frontend pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.queryFrontendSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.queryFrontendSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for query-frontend pods
nodeSelector: {}
# -- Tolerations for query-frontend pods
tolerations: []
# Configuration for the table-manager
tableManager:
# -- Specifies whether the table-manager should be enabled
enabled: false
image:
# -- The Docker registry for the table-manager image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the table-manager image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the table-manager image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for table-manager pods
priorityClassName: null
# -- Annotations for table-manager pods
podAnnotations: {}
# -- Labels for table-manager service
serviceLabels: {}
# -- Additional CLI args for the table-manager
extraArgs: []
# -- Environment variables to add to the table-manager pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the table-manager pods
extraEnvFrom: []
# -- Volume mounts to add to the table-manager pods
extraVolumeMounts: []
# -- Volumes to add to the table-manager pods
extraVolumes: []
# -- Resource requests and limits for the table-manager
resources: {}
# -- Grace period to allow the table-manager to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for table-manager pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.tableManagerSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.tableManagerSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for table-manager pods
nodeSelector: {}
# -- Tolerations for table-manager pods
tolerations: []
# Configuration for the gateway
gateway:
# -- Specifies whether the gateway should be enabled
enabled: true
# -- Number of replicas for the gateway
replicas: 1
autoscaling:
# -- Enable autoscaling for the gateway
enabled: false
# -- Minimum autoscaling replicas for the gateway
minReplicas: 1
# -- Maximum autoscaling replicas for the gateway
maxReplicas: 3
# -- Target CPU utilisation percentage for the gateway
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the gateway
targetMemoryUtilizationPercentage:
# -- See `kubectl explain deployment.spec.strategy` for more
# -- ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
deploymentStrategy:
type: RollingUpdate
image:
# -- The Docker registry for the gateway image
registry: docker.io
# -- The gateway image repository
repository: nginxinc/nginx-unprivileged
# -- The gateway image tag
tag: 1.19-alpine
# -- The gateway image pull policy
pullPolicy: IfNotPresent
# -- The name of the PriorityClass for gateway pods
priorityClassName: null
# -- Annotations for gateway pods
podAnnotations: {}
# -- Additional CLI args for the gateway
extraArgs: []
# -- Environment variables to add to the gateway pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the gateway pods
extraEnvFrom: []
# -- Volumes to add to the gateway pods
extraVolumes: []
# -- Volume mounts to add to the gateway pods
extraVolumeMounts: []
# -- The SecurityContext for gateway containers
podSecurityContext:
fsGroup: 101
runAsGroup: 101
runAsNonRoot: true
runAsUser: 101
# -- The SecurityContext for gateway containers
containerSecurityContext:
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
allowPrivilegeEscalation: false
# -- Resource requests and limits for the gateway
resources: {}
# -- Grace period to allow the gateway to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for gateway pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.gatewaySelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.gatewaySelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for gateway pods
nodeSelector: {}
# -- Tolerations for gateway pods
tolerations: []
# Gateway service configuration
service:
# -- Port of the gateway service
port: 80
# -- Type of the gateway service
type: ClusterIP
# -- ClusterIP of the gateway service
clusterIP: null
# -- Node port if service type is NodePort
nodePort: null
# -- Load balancer IPO address if service type is LoadBalancer
loadBalancerIP: null
# -- Annotations for the gateway service
annotations: {}
# -- Labels for gateway service
labels: {}
# Gateway ingress configuration
<TRUNCATED>
}
此时，所有 pod 都已启动并运行。
然后我使用官方helm图表安装了 Promtail 。
Values.yaml 文件（截断未修改的字段）
scrapeConfigs: |
# See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
- job_name: kubernetes-pods-app-kubernetes-io-name
# Kubernets auto discovery pods
kubernetes_sd_configs:
- role: pod
# Replacing kubernetes.io.name with app
relabel_configs:
- action: replace
source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
target_label: app
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
# Dropping logs that does not have app label
- action: drop
regex: ''
source_labels:
- app
# Replace namespace with namespace
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
# Required by promtail config to tail the pod logs
- action: replace
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
# Removing helath and ping logs from the application
pipeline_stages:
- match:
selector: '{app=~".*"} |~ ".*/(health|ping).*"'
action: drop
drop_counter_reason: promtail_noisy_error
- match:
selector: '{namespace="loki"}'
action: drop
drop_counter_reason: dont_like_it
- match:
selector: '{app=~".*"}'
stages:
- template:
source: pod
template: '{ "pod": {{ `"{{ .Value }}"` }}, "app": {{ `"{{ .app }}"` }} ,"pod_log": {{ `{{ .Entry }}` }} }'
- output:
source: pod
- labeldrop:
- filename
- pod
- app
# -- Config file contents for Promtail.
# Must be configured as string.
# It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
# @default -- See `values.yaml`
file: |
server:
log_level: {{ .Values.config.logLevel }}
http_listen_port: {{ .Values.config.serverPort }}
client:
url: {{ tpl .Values.config.lokiAddress . }}
{{- tpl .Values.config.snippets.extraClientConfigs . | nindent 2 }}
positions:
filename: /run/promtail/positions_ash.yaml
我遇到了很多问题：
- 当我尝试在 grafana（网关端点）中添加 Loki 源时。Grafana 抛出错误：
Loki：内部服务器错误。500. rpc 错误：代码 = 不可用 desc = 名称解析器错误：产生零地址
2. 显示时间戳错误的分发器日志：
level=warn ts=2021-10-06T10:41:23.729163833Z caller=grpc_logging.go:38 method=/logproto.Pusher/Push duration=86.664µs err=“rpc error: code = Code(400) desc = entry with时间戳 2021-10-06 10:41:22.689555007 +0000 UTC 被忽略，原因：流的“输入无序”：{namespace=“integration”}，\nentry 时间戳为 2021-10-06 10:41:22.689562602 + 0000 UTC 被忽略，原因：流的“输入无序”：{namespace=“integration”}，\nentry 时间戳为 2021-10-06 10:41:22.689582704 +0000 UTC 被忽略，原因：“输入无序”对于流：{namespace=“integration”}，\n总被忽略：6 个中有 3 个” msg=“gRPC\n”</p>
我不确定我做错了什么。我尝试了值文件中所有可能的更改。
我怀疑 Loki 配置文件有问题。
注意：当我尝试使用默认 values.yaml 文件安装 Loki 和 promtail 时，它可以工作，但我仍然可以看到错误。
原因：'输入乱序'</p>