做这个实验的目的是为了要在客户端请求整个链路上每个节点日志中加个唯一标识，方便排查问题时能根据唯一标识找到异常请求完整链路。实现方案就是打算通过自定义 header 信息，CLB提供 req_id 字段为每次请求生成一个 ID ，这样只要通过自定义 header 并且向后传递，每个节点读取这个 header 信息并记录到本次请求的日志中就可以了。 但是一开始查方案的时候有的描述是说 ingress-nginx-controller 不会传递用户自定义的 header 信息，所以这里打算验证下。
实验思路
- 1、首先写个 python 脚本能监听指定端口响应客户端请求并打印客户端所有 header 信息。
- 2、创建 deployment 资源运行上面 python 脚本验证在 K8S 中能正常工作。
- 3、测试环境找个对外提供服务的域名调整其 ingress 指向上面创建的 deployment 资源。
- 4、公网请求测试域名确认 deployment 资源日志是否能记录 CLB 自定义 header 。
编写能获取 header 信息的脚本
脚本内容
#!/usr/bin/env python
from http.server import BaseHTTPRequestHandler, HTTPServer
class MyHTTPRequestHandler(BaseHTTPRequestHandler):
def _set_response(self):
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
def do_GET(self):
self._set_response()
# 获取客户端的头信息
headers = self.headers
# 打印头信息
print('Headers:')
for header, value in headers.items():
print(f'{header}: {value}')
self.wfile.write(b'Hello, World!')
def run(server_class=HTTPServer, handler_class=MyHTTPRequestHandler, port=8080):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
print(f'Starting server on port {port}...')
httpd.serve_forever()
run()启动脚本
[me@imzcy ~]$ python3 header.py
Starting server on port 8080...客户端请求
curl -H 'name: zcy' -H 'Date: 20230925' localhost:8080确认控制台输出客户单自定义 header
127.0.0.1 - - [25/Sep/2023 17:56:53] "GET / HTTP/1.1" 200 -
Headers:
User-Agent: curl/7.29.0
Host: localhost:8080
Accept: */*
name: zcy
Date: 20230925创建 能获取用户 header 信息的 deploy 资源
创建 configmap
用于将 python-header.py 脚本挂载 pod 中作为容器启动应用监听 80 端口获取客户端 header 信息。
[root@imzcy test]# cat cm-header.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: cm-python-header
namespace: test
data:
python-header.py: |
#!/usr/bin/env python
import sys
from http.server import BaseHTTPRequestHandler, HTTPServer
# 将输出流重定向到 sys.stdout
sys.stdout = sys.stderr
class MyHTTPRequestHandler(BaseHTTPRequestHandler):
def _set_response(self):
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
def do_GET(self):
self._set_response()
# 获取客户端的头信息
headers = self.headers
# 打印头信息
print('Headers:', file=sys.stdout)
for header, value in headers.items():
print(f'{header}: {value}', file=sys.stdout)
self.wfile.write(b'Hello, World!')
def run(server_class=HTTPServer, handler_class=MyHTTPRequestHandler, port=80):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
print(f'Starting server on port {port}...')
httpd.serve_forever()
run()
[root@imzcy test]#[root@imzcy test]# kubectl apply -f cm-header.yaml
configmap/cm-python-header created
[root@imzcy test]#创建 deployment 资源
[root@imzcy test]# cat deployment-header.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deployment-imzcy-ingress-header
namespace: test
spec:
replicas: 1
selector:
matchLabels:
env: imzcy-test
desc: ingress-header
template:
metadata:
labels:
env: imzcy-test
desc: ingress-header
spec:
containers:
- name: python
image: python:3.9.18
command: ["python", "/data/python-header.py"]
volumeMounts:
- name: python-header
mountPath: /data
ports:
- containerPort: 80
name: http
protocol: TCP
volumes:
- name: python-header
configMap:
name: cm-python-header
[root@imzcy test]#[root@imzcy test]# kubectl apply -f deployment-header.yaml
deployment.apps/deployment-imzcy-ingress-header created
[root@imzcy test]#验证客户端请求
查看 pod IP
[root@imzcy test]# kubectl -n test get pods -o wide |grep "NAME\|imzcy"
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deployment-imzcy-ingress-header-856dd767-hgvs8 1/1 Running 0 38s 192.18.23.78 192.18.31.13 <none> <none>
[root@imzcy test]#查看 Pod 当前日志
[root@imzcy test]# kubectl -n test logs --tail 100 -f deployment-imzcy-ingress-header-856dd767-hgvs8
Starting server on port 80...curl 模拟客户端请求
[root@VM-31-8-centos ~]# curl http://192.18.23.78
Hello, World![root@VM-31-8-centos ~]#
[root@VM-31-8-centos ~]#
[root@VM-31-8-centos ~]# curl -H 'Name: zcy' http://192.18.23.78/abc
Hello, World![root@VM-31-8-centos ~]#
[root@VM-31-8-centos ~]#
[root@VM-31-8-centos ~]# curl -H 'Name: zcy' -H 'Date: 2023-09-26' http://192.18.23.78/123
Hello, World![root@VM-31-8-centos ~]#
[root@VM-31-8-centos ~]# 跟踪 Pod 日志确认能看到 header 信息
192.18.31.8 - - [26/Sep/2023 03:31:34] "GET / HTTP/1.1" 200 -
Headers:
User-Agent: curl/7.29.0
Host: 192.18.23.78
Accept: */*
192.18.31.8 - - [26/Sep/2023 03:32:05] "GET /abc HTTP/1.1" 200 -
Headers:
User-Agent: curl/7.29.0
Host: 192.18.23.78
Accept: */*
Name: zcy
192.18.31.8 - - [26/Sep/2023 03:32:32] "GET /123 HTTP/1.1" 200 -
Headers:
User-Agent: curl/7.29.0
Host: 192.18.23.78
Accept: */*
Name: zcy
Date: 2023-09-26
调整测试环境 ingress 指向验证服务
获取测试域名对应服务的 ingress 名称
[root@imzcy ~]# kubectl -n test get ingress |grep "NAME\|signinapp"
NAME CLASS HOSTS ADDRESS PORTS AGE
ing-express-signinapp nginx-wan signinapp-test.imzcy.cn 192.18.31.15 80 63d
[root@imzcy ~]#查看 ingress 当前指向的服务名和 pod 地址
[root@imzcy ~]# /root/kubectl -n test describe ingress ing-express-signinapp
Name: ing-express-signinapp
Namespace: test
Address: 192.18.31.15
Default backend: default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
Host Path Backends
---- ---- --------
signinapp-test.imzcy.cn
/ svc-express-signinapp-v39:80 (192.18.23.60:5000)
Annotations: <none>
Events: <none>
[root@imzcy ~]#创建 svc 资源
创建验证服务 svc 资源匹配上面 deployment 对应 pod
[root@imzcy test]# cat svc-imzcy-ingress-header.yaml
apiVersion: v1
kind: Service
metadata:
name: svc-imzcy-ingress-header
namespace: test
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
env: imzcy-test
desc: ingress-header
type: ClusterIP
[root@imzcy test]# [root@imzcy test]# kubectl apply -f svc-imzcy-ingress-header.yaml
service/svc-imzcy-ingress-header created
[root@imzcy test]#更新 ingress 指向验证服务
备份老的 ingress 配置
[root@imzcy test]# kubectl -n test get ingress ing-express-signinapp -o yaml >bak.ing-express-signinapp.yaml确认新 ingress 配置
[root@imzcy test]# cat new.ing-express-signinapp.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ing-express-signinapp
namespace: test
spec:
ingressClassName: nginx-wan
rules:
- host: signinapp-test.imzcy.cn
http:
paths:
- backend:
service:
name: svc-imzcy-ingress-header
port:
number: 80
path: /
pathType: Prefix
[root@imzcy test]#apply 应用更改
[root@imzcy test]# kubectl apply -f new.ing-express-signinapp.yaml
ingress.networking.k8s.io/ing-express-signinapp configured
[root@imzcy test]#确认 ingress 指向上面跑 python 脚本的 Pod
[root@imzcy test]# kubectl -n test describe ingress ing-express-signinapp
Name: ing-express-signinapp
Namespace: test
Address: 192.18.31.15
Default backend: default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
Host Path Backends
---- ---- --------
signinapp-test.imzcy.cn
/ svc-imzcy-ingress-header:80 (192.18.23.78:80)
Annotations: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 8s (x11 over 63d) nginx-ingress-controller Scheduled for sync
[root@imzcy test]#客户端请求测试环境域名
[root@imzcy ~]# curl -H 'Name: zcy' https://signinapp-test.imzcy.cn/test?name=zcy
Hello, World![root@imzcy ~]#
[root@imzcy ~]#确认日志中 header 信息
192.18.31.15 - - [26/Sep/2023 03:50:03] "GET /test?name=zcy HTTP/1.1" 200 -
Headers:
Host: signinapp-test.imzcy.cn
X-Request-ID: 2b9fce6a463894d93313ef129855c9d2
X-Real-IP: 58.58.58.58
X-Forwarded-For: 58.58.58.58
X-Forwarded-Host: signinapp-test.imzcy.cn
X-Forwarded-Port: 80
X-Forwarded-Proto: http
X-Forwarded-Scheme: http
X-Scheme: http
X-Original-Forwarded-For: 58.58.58.58
X-clb-lbid: lb-xxxxxx9d
Stgw-request-id: a4de98c498fe7850e2a5f30f88603d91
X-Stgw-Time: 1695700203.996
X-Client-Proto: https
X-Client-Proto-Ver: HTTP/1.1
User-Agent: curl/7.29.0
Accept: */*
Name: zcy
本文采用 知识共享署名4.0 国际许可协议进行许可。
本站文章除注明转载/出处外，均为本站原创或翻译，转载前请务必署名。
如果您的问题未解决，欢迎微信扫描右侧二维码与我联系。