闲来无事，查看了下Ingress-nginx的文档，发现Ingress-nginx目前已支持Opentelemetry.它是通过opentelemetry-cpp-contrib/nginx 实现.只要Ingress-nginx在代理时注入Trace-Id,然后将该ID通过Header传递给应用,后续应用记录日志时都记录该ID.这样我们就可以从请求到最终响应,实现全链路追踪.

But,How to do it?

查阅官方文档发现,它只一个模块,只要加载这个模块,就可以实现全链路追踪.而在Ingress-nginx中,要启用这个模块,只需要configMap中添加enable-opentelemetry: "true"打开即可.

Step 1: Add configMap

应用以下YAML

1
2
3
4
5
6
7

apiVersion: v1
data:
  X-Trace-Id: "$opentelemetry_trace_id"
kind: ConfigMap
metadata:
  name: custom-headers
  namespace: ingress-nginx

Step 2: Modify the nginx-configuration configmap

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

...
# others
data:
  # Customer http headers, It will send to the upstream
  add-headers: "ingress-nginx/custom-headers"
  proxy-set-headers: "ingress-nginx/custom-headers"
  # 以上两个都要加上
  enable-opentelemetry: 'true'
  allow-snippet-annotations: 'false'
  disable-access-log: 'false'
  error-log-path: rsyslog:server=128.0.255.10:514
  access-log-path: syslog:server=128.0.21.56:514
  log-format-upstream: >-
    {
        "time_msec": "$time_iso8601",
        "host": "$host",
        "uri": "$request_uri",
        "status": $status,
        "size": $bytes_sent",
        "upstream_addr": "$upstream_addr",
        "namespace": "$namespace",
        "service": "$service_name",
        "trace_id": "$opentelemetry_trace_id",
        "spind_id": "$opentelemetry_span_id"
    }    
  proxy-body-size: 600m
  proxy-read-timeout: "60"
  use-gzip: "true"
  worker-cpu-affinity: auto

Step 2: Add vecotor to reveieve the logs

  • docker-compose.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

version: '3'
services:
  vector:
    image: docker.1ms.run/timberio/vector:0.49.0-debian
    container_name: vagent
    restart: always
    network_mode: host
    privileged: true
    volumes:
    - ./vector.yaml:/etc/vector/vector.yaml:ro
    environment:
    - TZ=Asia/Shanghai
  • vector.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

api:
  enabled: false

sources:
  rsyslog:
    type: syslog
    address: 0.0.0.0:514
    mode: udp

transforms:
  to_json:
    type: remap
    inputs:
    - rsyslog
    source: |
      . = parse_json!(.message)
      v = get!(., ["time"])
      # if TimeZone is Asia/Shanghai, format is %Y-%m-%dT%H:%M:%S+08:00
      # Time filed in Clickhouse is DateTime type
      time = parse_timestamp!(v, "%Y-%m-%dT%H:%M:%S+00:00")
      time_str = format_timestamp!(time, "%Y-%m-%d %H:%M:%S")
      . = set!(., ["time"], time_str)      

sinks:
  debug:
    type: console
    buffer:
      type: memory
      max_events: 1
    encoding:
      codec: json
    inputs:
    - to_json
    
  ck:
    type: clickhouse
    inputs:
    - to_json
    endpoint: http://128.0.255.6:8123
    table: access_log
    auth:
      strategy: basic
      user: default
      password: ""
    skip_unknown_fields: true