我目前正在检查为什么当时间范围设置为超过 24 小时时 Kibana 中的 APM UI 没有向我输出信息。
检查配置时，我确实注意到我们没有为 APM 服务器设置 Kibana 端点。
检查 APM 日志我看不到错误，但是当转到 APM UI 时，我可以在 Kibana 日志中找到此错误：
kibana |{"type":"response","@timestamp":"2020-01-23T09:15:08Z","tags":["access:apm"],"pid":7,"method":"get","statusCode":400,"req":{"url":"/api/apm/settings/agent-configuration/services","method":"get","headers":{"host":"<our hostname>","connection":"keep-alive","sec-fetch-dest":"empty","kbn-version":"7.3.1","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.53 Safari/537.36 Edg/80.0.361.33","content-type":"application/json","accept":"*/*","sec-fetch-site":"same-origin","sec-fetch-mode":"cors","referer":"https://<our hostname>/app/apm","accept-encoding":"gzip, deflate, br","accept-language":"sl,en;q=0.9,en-GB;q=0.8,en-US;q=0.7"},"remoteAddress":"192.168.101.91","userAgent":"192.168.101.91","referer":"https://<our hostname>/app/apm"},"res":{"statusCode":400,"responseTime":30038,"contentLength":9},"message":"GET /api/apm/settings/agent-configuration/services 400 30038ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:15:47Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"65","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":15,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 15ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:15:58Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"30","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":20,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 20ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:17Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"65","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":20,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 20ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:25Z","tags":["access:apm"],"pid":7,"method":"get","statusCode":200,"req":{"url":"/api/apm/settings/agent-configuration","method":"get","headers":{"host":"<our hostname>","connection":"keep-alive","sec-fetch-dest":"empty","kbn-version":"7.3.1","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.53 Safari/537.36 Edg/80.0.361.33","content-type":"application/json","accept":"*/*","sec-fetch-site":"same-origin","sec-fetch-mode":"cors","referer":"https://<our hostname>/app/apm","accept-encoding":"gzip, deflate, br","accept-language":"sl,en;q=0.9,en-GB;q=0.8,en-US;q=0.7"},"remoteAddress":"192.168.101.91","userAgent":"192.168.101.91","referer":"https://<our hostname>/app/apm"},"res":{"statusCode":200,"responseTime":17,"contentLength":9},"message":"GET /api/apm/settings/agent-configuration 200 17ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:29Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"30","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":17,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 17ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:47Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"65","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":21,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 21ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:47Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"65","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":16,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 16ms - 9.0B"}
kibana | Error: Request Timeout after 30000ms
kibana | at /usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:362:15
kibana | at Timeout.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:391:7)
kibana | at ontimeout (timers.js:436:11)
kibana | at tryOnTimeout (timers.js:300:5)
kibana | at listOnTimeout (timers.js:263:5)
kibana | at Timer.processTimers (timers.js:223:10)
kibana | {"type":"response","@timestamp":"2020-01-23T09:16:25Z","tags":["access:apm"],"pid":7,"method":"get","statusCode":400,"req":{"url":"/api/apm/settings/agent-configuration/services","method":"get","headers":{"host":"<our hostname>","connection":"keep-alive","sec-fetch-dest":"empty","kbn-version":"7.3.1","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.53 Safari/537.36 Edg/80.0.361.33","content-type":"application/json","accept":"*/*","sec-fetch-site":"same-origin","sec-fetch-mode":"cors","referer":"https://<our hostname>/app/apm","accept-encoding":"gzip, deflate, br","accept-language":"sl,en;q=0.9,en-GB;q=0.8,en-US;q=0.7"},"remoteAddress":"192.168.101.91","userAgent":"192.168.101.91","referer":"https://<our hostname>/app/apm"},"res":{"statusCode":400,"responseTime":30041,"contentLength":9},"message":"GET /api/apm/settings/agent-configuration/services 400 30041ms - 9.0B"}
kibana | {"type":"response","@timestamp":"2020-01-23T09:17:01Z","tags":["access:apm"],"pid":7,"method":"post","statusCode":404,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"<our hostname>:443","user-agent":"Go-http-client/1.1","content-length":"30","accept":"application/json","content-type":"application/json","kbn-version":"7.3.1","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.18.0.1","userAgent":"172.18.0.1"},"res":{"statusCode":404,"responseTime":41,"contentLength":9},"message":"POST /api/apm/settings/agent-configuration/search 404 41ms - 9.0B"}
从输出看来，Kibana 找不到任何服务器配置，但设置已成功完成，但我也无法访问在 Kibana 中移动的 APM 设置（超时，错误 404）。
其他查询或来自相同索引的查询运行良好，不会增加任何延迟。
APM 代理是最新版本的 Django (Python) 代理。而且集群的资源指标和状态都非常好，所以集群应该足够强大。
我们也在 Kubernetes 中进行了测试部署，服务有时会显示在 APM UI 中，但目前已关闭。
环境和版本：
Elasticsearch 在 Docker 容器中部署在 GCP 上（不同 VM 上的 4 个节点 + 1 个具有 APM 服务器、Kibana 和 Elasticsearch 客户端节点的 VM）。
ES、Kibana、APM-Server 版本：7.3.1